]>
Commit | Line | Data |
---|---|---|
06ee5545 RN |
1 | #!/bin/bash |
2 | # | |
3 | # | |
4 | # DO NOT EDIT! This file is generated from geo-nearest.sh | |
5 | # | |
6 | ||
7 | # | |
8 | # geo-nearest: Fetch list of nearest geocaches. | |
9 | # | |
10 | # Requires: curl; gpsbabel; bash or ksh; | |
11 | # mysql (if using the gpsdrive.sql output option) | |
12 | # | |
cffb6fa9 | 13 | # Donated to the public domain by Rick Richardson <rickrich@gmail.com> |
06ee5545 RN |
14 | # |
15 | # Use at your own risk. Not suitable for any purpose. Not legal tender. | |
16 | # | |
cffb6fa9 | 17 | # $Id: geo-nearest.sh,v 1.49 2013/02/18 21:41:06 rick Exp $ |
06ee5545 RN |
18 | # |
19 | ||
20 | PROGNAME="$0" | |
21 | ||
22 | usage() { | |
23 | cat <<EOF | |
24 | NAME | |
25 | `basename $PROGNAME` - Fetch a list of nearest geocaches | |
26 | ||
27 | SYNOPSIS | |
28 | `basename $PROGNAME` [options] | |
29 | ||
30 | `basename $PROGNAME` [options] latitude longitude | |
31 | ||
cffb6fa9 RN |
32 | `basename $PROGNAME` [options] latitude longitude cache-type |
33 | ||
06ee5545 RN |
34 | `basename $PROGNAME` [options] zipcode |
35 | ||
36 | `basename $PROGNAME` [options] u=<username> | |
37 | ||
cffb6fa9 RN |
38 | `basename $PROGNAME` [options] ul=<username> |
39 | ||
06ee5545 RN |
40 | `basename $PROGNAME` [options] pq=<pocket-query> |
41 | ||
42 | `basename $PROGNAME` [options] tx=<bookmark-id> | |
43 | ||
44 | `basename $PROGNAME` [options] -b bookmark | |
45 | `basename $PROGNAME` [options] guid=<bookmark-id> | |
46 | ||
47 | DESCRIPTION | |
48 | Fetch a list of nearest geocaches. | |
49 | ||
50 | Requires: | |
51 | A premium member (\$30/yr) OR a basic member (free) login at: | |
52 | http://www.geocaching.com | |
53 | Visit a cache page and click the "Download to EasyGPS" link at least | |
54 | once so you can read and agree to the license terms. Otherwise, you | |
55 | will not get any waypoint data. | |
56 | ||
57 | curl http://curl.haxx.se/ | |
58 | gpsbabel http://gpsbabel.sourceforge.net/ | |
59 | ||
60 | EOF | |
61 | gc_usage | |
62 | cat << EOF | |
63 | ||
64 | NOTE | |
65 | A basic member will get caches very slow (20 cache pages per minute) | |
66 | because we have to get the actual cache pages. They will be stored in: | |
67 | ~/.geo/caches/GCXXXX.html. | |
68 | Of course, after running this command, geo-html2gpx could be run. | |
69 | ||
70 | EXAMPLES | |
71 | Nearest 20 caches, display shortnames: | |
72 | ||
73 | geo-nearest -s | |
74 | ||
75 | Search nearest 500 caches for virtual caches not yet found: | |
76 | ||
77 | geo-nearest -n500 -Ivirtual -Xifound | |
78 | ||
79 | Add nearest 50 caches to a GpsDrive SQL database | |
80 | ||
81 | geo-nearest -n50 -f -s -S | |
82 | ||
83 | Purge the existing SQL database of all geocaches, and fetch | |
84 | 200 fresh ones... | |
85 | ||
86 | geo-nearest -S -P -s -n200 | |
87 | ||
88 | 640x480 map of nearest caches using map source 2: | |
89 | ||
90 | geo-nearest -omap,"-a2 -W640 -H480" | |
91 | ||
92 | Copy two cachers: | |
93 | ||
94 | geo-nearest -n200 -Xifound -udyl1231 -pPW | awk '{print \$1}' >1.foo | |
95 | geo-nearest -n200 -Xifound -urickrich -pPW |awk '{print \$1}' >2.foo | |
96 | geo-gid -otabsep \$(comm -12 1.foo 2.foo) >both | |
97 | ||
cffb6fa9 | 98 | Fetch by owner placed: |
06ee5545 RN |
99 | |
100 | geo-nearest u=team-deadhead | |
101 | ||
cffb6fa9 RN |
102 | Fetch by owner found: |
103 | ||
104 | geo-nearest ul="AAA+of+MichigAn&sortdir=asc&sort=placed" | |
105 | ||
06ee5545 RN |
106 | Fetch by tx method: |
107 | ||
108 | # nearby caches of this (puzzle) type, that I haven't found | |
109 | geo-nearest -n500 -f -otabsep tx=40861821-1835-4e11-b666-8d41064d03fe | | |
110 | geo-mystery >> Caches/rick.ts | |
111 | ||
cffb6fa9 RN |
112 | Also, tx=webcam, tx=earth, tx=multi, tx=event, tx=virtual, tx=letter, |
113 | tx=unknown, tx=trad (tx=reg is an alias). | |
114 | ||
115 | Fetch by cache-type method: | |
116 | ||
117 | # nearby caches of this (puzzle) type, that I haven't found | |
118 | geo-nearest -n500 -f -otabsep unknown | | |
119 | geo-mystery >> Caches/rick.ts | |
120 | ||
121 | Also, cache-type is webcam, earth, multi, event, virtual, letter, | |
122 | unknown, trad (reg is an alias). | |
123 | ||
06ee5545 RN |
124 | Fetch a bookmark list: |
125 | ||
126 | geo-nearest -b acro | |
127 | or | |
128 | geo-nearest guid=baae5bf9-4315-4874-b7fb-ac84c9585641 | |
129 | ||
130 | Fetch a PQ query: | |
131 | ||
cffb6fa9 RN |
132 | geo-nearest -q "Needs Maintenance" |
133 | or | |
06ee5545 RN |
134 | geo-nearest pq=08be103b-ffd1-4e27-992f-616e144e24df |
135 | ||
136 | FILES | |
137 | ~/.georc | |
138 | ~/.geo/caches/ | |
139 | ||
140 | SEE ALSO | |
141 | geo-newest, geo-found, geo-placed, geo-keyword, geo-code, geo-map, | |
142 | geo-waypoint, | |
143 | $WEBHOME | |
144 | EOF | |
145 | ||
146 | exit 1 | |
147 | } | |
148 | ||
149 | ############################################################################## | |
150 | # begin #include "geo-common" | |
151 | ############################################################################## | |
152 | ||
153 | # I doubt this stuff will work in other than english | |
154 | LANG=en_US | |
155 | ||
156 | # | |
157 | # Common global constants | |
158 | # | |
159 | UA="Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)" | |
160 | WEBHOME="http://geo.rkkda.com/" | |
161 | ||
162 | # | |
163 | # Common global variables | |
164 | # | |
165 | DEBUG=0 | |
166 | CRUFT= | |
167 | CURL_OPTS= | |
168 | ||
169 | # | |
170 | # Report an error and exit | |
171 | # | |
172 | error() { | |
173 | echo "`basename $PROGNAME`: $1" >&2 | |
174 | exit 1 | |
175 | } | |
176 | ||
177 | debug() { | |
178 | if [ $DEBUG -ge $1 ]; then | |
179 | echo "`basename $PROGNAME`: $2" >&2 | |
180 | fi | |
181 | } | |
182 | ||
183 | verbose() { | |
184 | if [ $VERBOSE -ge $1 ]; then | |
185 | echo "$2" >&2 | |
186 | fi | |
187 | } | |
188 | ||
189 | dbgcmd() { | |
190 | if [ $DEBUG -ge $DBGCMD_LVL ]; then | |
191 | echo "$@" >&2 | |
192 | fi | |
193 | "$@" | |
194 | } | |
195 | DBGCMD_LVL=2 | |
196 | ||
197 | # | |
198 | # procedure to remove cruft files | |
199 | # | |
200 | remove_cruft() { | |
201 | if [ $DEBUG = 0 -a "$CRUFT" != "" ]; then | |
202 | for i in $CRUFT | |
203 | do | |
204 | [ -f $i ] && rm -f $i | |
205 | done | |
206 | fi | |
207 | } | |
208 | ||
209 | trap remove_cruft EXIT | |
210 | ||
211 | # | |
212 | # Convert DegDec, MinDec, or DMS lat/lon to DegDec | |
213 | # | |
214 | latlon() { | |
215 | # Handle NSEW prefixes | |
216 | arg1=`echo "$1" | sed -e 's/^[nNeE]//' -e 's/^[wW]/-/' -e 's/^[sS]/-/'` | |
217 | # If negative, print the sign then take the absolute value | |
218 | case "$arg1" in | |
219 | -*) echo -n "-"; arg1=`echo "$arg1" | sed 's/^-//'`;; | |
220 | esac | |
221 | # Now handle the 3 different formats | |
222 | case $# in | |
223 | 1) | |
224 | case "$arg1" in | |
225 | *.*.*) echo "$arg1" \ | |
226 | | sed -e 's/,//' -e 's#\([^.]*\)\.#\1 #' -e 's#$# 6k 60/+p#' \ | |
227 | | dc | |
228 | ;; | |
229 | lat=*) | |
230 | echo "$arg1" | sed 's/^lat=//' | |
231 | ;; | |
232 | lon=*) | |
233 | echo "$arg1" | sed 's/^lon=//' | |
234 | ;; | |
235 | *) echo $arg1 | |
236 | ;; | |
237 | esac | |
238 | ;; | |
239 | 2) echo "6k $arg1 $2 60/+p" | dc;; | |
240 | 3) echo "6k $arg1 $2 60/ $3 3600/++p" | dc;; | |
241 | esac | |
242 | } | |
243 | ||
244 | # | |
cffb6fa9 | 245 | # Convert DegDec to MinDec with optional NS/EW |
06ee5545 RN |
246 | # |
247 | degdec2mindec() { | |
cffb6fa9 RN |
248 | awk -v v=$1 -v sym=$2 \ |
249 | ' | |
250 | function abs(x) { return (x>=0) ? x : -x } | |
251 | BEGIN{ | |
252 | i=int(v) | |
253 | if (sym == "") | |
254 | printf "%d.%06.3f\n", i, abs(v-i) * 60 | |
255 | else | |
256 | printf "%s%d.%06.3f\n", \ | |
257 | i >= 0.0 ? substr(sym, 1, 1) : substr(sym, 2, 1), \ | |
258 | abs(i), abs(v-i) * 60 | |
259 | }' | |
06ee5545 RN |
260 | } |
261 | ||
262 | # | |
263 | # Read RC file, if there is one | |
264 | # | |
265 | read_rc_file() { | |
266 | if [ -f $HOME/.georc ]; then | |
267 | . $HOME/.georc | |
268 | # Allow LAT/LON in rc file to be in any of the formats that we grok | |
269 | if [ "" != "$LAT" ]; then | |
cffb6fa9 RN |
270 | if ! is_latlon $LAT 0; then |
271 | error "Latitude '$LAT' is not parsed in .georc!" | |
272 | fi | |
06ee5545 RN |
273 | LAT=`latlon $LAT` |
274 | fi | |
275 | if [ "" != "$LON" ]; then | |
cffb6fa9 RN |
276 | if ! is_latlon 0 $LON; then |
277 | error "Longitude '$LON' is not parsed in .georc!" | |
278 | fi | |
06ee5545 RN |
279 | LON=`latlon $LON` |
280 | fi | |
281 | else | |
282 | cat <<-EOF > $HOME/.georc | |
283 | # | |
284 | # These are the default values for the geo-* series of programs | |
285 | # Please edit this file as needed. Setting values for | |
286 | # USERNAME, PASSWORD, LAT/LON, and STATE are required. | |
287 | # | |
288 | ||
289 | ################################# | |
290 | # Login and paid membership status for www.geocaching.com... | |
291 | #USERNAME=name | |
292 | #PASSWORD=pasword | |
293 | #SOC=0 | |
294 | ||
295 | ################################# | |
296 | # Your HOME lat/lon and state... | |
297 | #LAT=N44.55.666 | |
298 | #LON=W93.11.222 | |
299 | #STATE=MN | |
300 | ||
301 | ################################# | |
302 | # Default map scale, font, and source... | |
303 | #MAPSCALE=10K | |
304 | #MAPFONT=helvetica | |
305 | #MAPSRC=2 | |
306 | ||
307 | ################################# | |
308 | # Login for terraserver.com... | |
309 | #TSCOM_EMAIL=xxx@yyy.com | |
310 | #TSCOM_PW=password | |
311 | ||
312 | ################################# | |
313 | # Miscellaneous... | |
314 | #OUTFMT=gpsdrive | |
315 | #GPSDRIVE_VER=2.09 | |
cffb6fa9 RN |
316 | #GEOMYSTERY=$HOME/.geo-mystery |
317 | #OCMYSTERY=$HOME/.oc-mystery | |
318 | #DATEFMT=0 | |
06ee5545 | 319 | #CURL_OPTS="-4" |
cffb6fa9 | 320 | #CURL_OPTS="--sslv3" |
06ee5545 RN |
321 | EOF |
322 | error "First time user: please review and edit $HOME/.georc" | |
323 | fi | |
324 | } | |
325 | ||
326 | if [ `uname` = 'Darwin' ]; then | |
327 | sed=gsed | |
328 | date=gdate | |
329 | touch=gtouch | |
330 | PATH=$PATH:/usr/local/bin:/opt/local/bin | |
331 | export PATH | |
332 | else | |
333 | sed=sed | |
334 | date=date | |
335 | touch=touch | |
336 | fi | |
337 | ||
338 | # | |
339 | # Get the value from a name= value= pair in a file | |
340 | # | |
341 | get_value() { | |
342 | # <input type="hidden" name="__EVENTTARGET" value="" | |
343 | what=$1 | |
344 | where=$2 | |
345 | eval $what=`$sed -n "s/^.*\"$what\" *value=\"\([^\"]*\)\".*/\1/p" < $where` | |
346 | } | |
347 | ||
348 | # | |
349 | # urlencode | |
350 | # | |
351 | # incomplete, just does what we need it to do | |
352 | # | |
353 | urlencode() { | |
354 | echo "$1" | sed -e 's/\+/%2B/g' -e 's/\&/%26/g' #-e 's/\//%2F/' | |
355 | } | |
356 | ||
357 | # | |
358 | # urlencode2 | |
359 | # | |
360 | urlencode2() { | |
361 | echo "$1" | | |
362 | awk ' | |
363 | BEGIN { | |
364 | split("1 2 3 4 5 6 7 8 9 A B C D E F", hextab, " ") | |
365 | hextab[0] = 0 | |
366 | for (i = 1; i <= 255; ++i) ord[ sprintf ("%c", i) "" ] = i + 0 | |
367 | } | |
368 | { | |
369 | encoded = "" | |
370 | for (i = 1; i <= length($0); ++i ) { | |
371 | c = substr ($0, i, 1) | |
372 | val = ord[c] | |
373 | if (val >= 97 && val <= 122) #0x61-0x7A | |
374 | encoded = encoded c | |
375 | else if (val >= 65 && val <= 90) #0x41-0x5A | |
376 | encoded = encoded c | |
377 | else if (val >= 48 && val <= 57) #0x30-0x39 | |
378 | encoded = encoded c | |
379 | else if (val >= 45 && val <= 46) #0x2D-0x2E | |
380 | encoded = encoded c | |
381 | else if (c == " ") | |
382 | encoded = encoded "+" | |
383 | else if (val < 128) { | |
384 | lo = val % 16 | |
385 | hi = int(val / 16); | |
386 | encoded = encoded "%" hextab[hi] hextab[lo] | |
387 | } | |
388 | else { | |
389 | byte = 192 + val/64 | |
390 | lo = byte % 16 | |
391 | hi = int(byte / 16); | |
392 | encoded = encoded "%" hextab[hi] hextab[lo] | |
393 | byte = 128 + val%64 | |
394 | lo = byte % 16 | |
395 | hi = int(byte / 16); | |
396 | encoded = encoded "%" hextab[hi] hextab[lo] | |
397 | } | |
398 | } | |
399 | print encoded | |
400 | } | |
401 | ' | |
402 | } | |
403 | ||
404 | # | |
405 | # return true if current arguments appear to be a lat/lon | |
406 | # | |
407 | is_latlon() { | |
408 | if [ "$#" -lt 2 ]; then | |
409 | return 1 | |
410 | fi | |
411 | case "$1" in | |
412 | lat=*) ;; # cut/paste from GPX file | |
413 | [NS]) return 0;; # cut/paste from gc.com | |
414 | [NSns][0-9]*) ;; | |
415 | [-][0-9]*) ;; | |
416 | [0-9]*) ;; | |
417 | *) return 1;; | |
418 | esac | |
419 | case "$2" in | |
420 | lon=*) return 0;; | |
421 | [EWew][0-9]*) return 0;; | |
422 | [-][0-9]*) return 0;; | |
423 | [0-9]*) return 0;; | |
424 | *) return 1;; | |
425 | esac | |
426 | } | |
427 | ||
428 | # | |
429 | # split lines between two strings | |
430 | # | |
431 | # $1 - string 1 | |
432 | # $2 - string 2 | |
433 | # $3 - null or 'g' | |
434 | # | |
435 | split_lines_between() { | |
436 | sed "s@$1$2@$1\\ | |
437 | $2@$3" | |
438 | } | |
439 | ||
440 | ############################################################################## | |
441 | # end #include "geo-common" | |
442 | ############################################################################## | |
443 | ############################################################################## | |
444 | # begin #include "geo-common-gc" | |
445 | # | |
cffb6fa9 | 446 | # $Id: geo-common-gc,v 1.313 2013/02/01 14:29:33 rick Exp $ |
06ee5545 RN |
447 | ############################################################################## |
448 | ||
449 | # | |
450 | # Common global constants | |
451 | # | |
452 | UA="Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)" | |
453 | UA="Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/XX (KHTML, like Gecko) Version/ZZ Safari/YY" | |
454 | GEO="http://www.geocaching.com" | |
455 | GEOS="https://www.geocaching.com" | |
456 | ||
457 | # | |
458 | # Global variables that can be overridden on command line or rc file | |
459 | # | |
460 | PASSWORD=dummy | |
461 | USERNAME=dummy | |
462 | LAT=44.9472 | |
463 | LON=-93.4914 | |
464 | SOC=${SOC:0} | |
465 | GEOMYSTERY=/dev/null | |
466 | GEODIR=$HOME/.geo | |
467 | case `uname` in | |
468 | CYGWIN*) CURL_OPTS=-k;; | |
469 | *) CURL_OPTS= | |
470 | esac | |
471 | ||
472 | # | |
473 | # Global variables | |
474 | # | |
475 | COOKIE_FILE=$HOME/.geocookies | |
476 | NOCOOKIES=0 | |
477 | FOUND=1 | |
478 | USERFOUND=1 | |
479 | BABELFLAGS= | |
480 | RADIUS= | |
cffb6fa9 RN |
481 | RADIUS_NUM= |
482 | RADIUS_MILES= | |
06ee5545 RN |
483 | OUTFILE= |
484 | OUTFMT=gpsdrive | |
485 | NUM=20 | |
486 | INCLUDE=* | |
487 | EXCLUDE='-unavail' | |
488 | VARTIME=found | |
489 | GEOSLEEP=${GEOSLEEP:-8} | |
490 | ||
491 | # | |
492 | # Common options handling | |
493 | # | |
494 | gc_usage() { | |
495 | cat <<EOF | |
496 | OPTIONS | |
497 | -b bookmark Use list "bookmark" [none] | |
cffb6fa9 RN |
498 | -q query geo-nearest: Use PQ list "query" [none] |
499 | -q search geo-newest: Use search "search" [none] | |
06ee5545 RN |
500 | -c Remove cookie file when done |
501 | -f Do not report any found or unavailable caches | |
502 | -m Do not report any members-only caches | |
503 | -F Report caches found by the login 'username' as unfound | |
504 | -n num Return "num" caches [$NUM] | |
505 | -s Output short names for the caches (gpsbabel option) | |
506 | -I term Include only caches with 'term' [$INCLUDE] | |
507 | -X term Exclude caches with 'term' [$EXCLUDE] | |
cffb6fa9 | 508 | terms: ~ (exclude none), unfound, ifound, soc, unavail, |
06ee5545 RN |
509 | regular, multi, virtual, webcam, event, hybrid, cito |
510 | -r radius Display only caches with radius (e.g. -r 25M) | |
511 | -M mystery Use file 'mystery' for unknown/mystery/puzzle | |
512 | caches [$GEOMYSTERY]. Awk Format: | |
cffb6fa9 RN |
513 | |
514 | gcid lat lon comment | |
515 | ||
516 | i.e.: GC2CBVB n44.45.123 w93.00.321 Final | |
06ee5545 RN |
517 | |
518 | -u username Username for http://www.geocaching.com | |
519 | -p password Password for http://www.geocaching.com | |
520 | -o format Output format, -o? for possibilities [$OUTFMT] | |
521 | plus "gpsdrive.sql" for direct insertion into MySQL DB | |
522 | plus "map[,geo-map-opts]" to display a geo-map. | |
523 | -O filename Output file, if not stdout | |
524 | -S Alias for -o gpsdrive.sql | |
525 | -d For -S, just delete selected records | |
526 | -P For -S, purge all records of type -t $SQLTAG* | |
527 | -t type For -ogpsdrive.sql, the waypoint type [$SQLTAG] | |
528 | -H htmldir Also fetch the printable HTML pages (slowly) | |
529 | -L logdir Also fetch the plain text log entries (slowly) | |
530 | For -H or -L, the limit is 1500 updated caches/day. | |
531 | -! "lpr -Plp" Print HTML pages | |
cffb6fa9 RN |
532 | -E var=val Set environment "var" to "val" |
533 | i.e. DATEFMT=0|1 | |
06ee5545 RN |
534 | -D lvl Debug level [$DEBUG] |
535 | -U Retrieve latest version of this script | |
536 | ||
537 | DEFAULTS | |
538 | Defaults can also be set with variables in file \$HOME/.georc: | |
539 | ||
540 | PASSWORD=password; USERNAME=username; SOC=0|1; | |
541 | LAT=latitude; LON=logitude; GEOMYSTERY=/dev/null; | |
542 | NUM=num; OUTFMT=format; BABELFLAGS=-s; | |
543 | SQLUSER=gast; SQLPASS=gast; SQLDB=geoinfo; | |
cffb6fa9 RN |
544 | DATEFMT=[0|1]; |
545 | DATE FORMATS | |
546 | Geocaching.com date formats that are compatible: | |
547 | ||
548 | GC Format Example Compatible | |
549 | YYYY-MM-DD 2011-07-13 yes | |
550 | YYYY/MM/DD 2011/07/13 yes | |
551 | MM/DD/YYYY 07/13/2011 yes | |
552 | DD/MM/YYYY 13/07/2011 yes if DATEFMT=1 in \$HOME/.georc | |
553 | DD/Mmm/YYYY 13/Jul/2001 no | |
554 | Mmm/DD/YYYY Jul/13/2011 no | |
555 | DD Mmm YY 13 Jul 11 yes (english only) | |
556 | ||
557 | Change them here: | |
558 | ||
559 | http://www.geocaching.com/account/ManagePreferences.aspx | |
06ee5545 RN |
560 | EOF |
561 | } | |
562 | ||
563 | gc_getopts() { | |
564 | # | |
565 | # Defaults for options that cannot be overriden in the RC file | |
566 | # | |
567 | PURGE=0 | |
568 | DELETE=0 | |
569 | SQL=0 | |
570 | MAP=0 | |
571 | HTMLDIR= | |
572 | LOGDIR= | |
573 | CMDPIPE= | |
574 | BOOKMARK= | |
cffb6fa9 | 575 | POCKETQUERY= |
06ee5545 | 576 | |
cffb6fa9 | 577 | while getopts "!:E:H:L:I:X:b:q:cdfFmM:n:o:O:p:Pr:sSt:u:D:Uh?-" opt |
06ee5545 RN |
578 | do |
579 | case $opt in | |
580 | "!") CMDPIPE="$OPTARG";; | |
581 | b) BOOKMARK="$OPTARG";; | |
cffb6fa9 | 582 | q) POCKETQUERY="$OPTARG";; |
06ee5545 RN |
583 | c) NOCOOKIES=1;; |
584 | d) DELETE=1;; | |
cffb6fa9 | 585 | E) eval "$OPTARG";; |
06ee5545 RN |
586 | f) FOUND=0; EXCLUDE="$EXCLUDE|-ifound";; |
587 | m) SOC=0;; | |
588 | M) GEOMYSTERY="$OPTARG";; | |
589 | I) | |
590 | if [ "$INCLUDE" = "*" ]; then | |
591 | INCLUDE= | |
592 | else | |
593 | INCLUDE="$INCLUDE|" | |
594 | fi | |
595 | INCLUDE="$INCLUDE-$OPTARG" | |
596 | ;; | |
597 | X) | |
cffb6fa9 | 598 | if [ "" = "$OPTARG" -o "~" = "$OPTARG" ]; then |
06ee5545 RN |
599 | EXCLUDE=-ExClUdEnOtHiNg |
600 | else | |
601 | EXCLUDE="$EXCLUDE|-$OPTARG" | |
602 | fi | |
603 | ;; | |
604 | F) USERFOUND=0;; | |
605 | P) PURGE=1;; | |
606 | n) NUM="$OPTARG" | |
607 | case "$NUM" in | |
608 | [0-9]*) ;; | |
609 | *) error "Not a number: '$NUM'";; | |
610 | esac | |
611 | if [ "$NUM" -lt 1 -o "$NUM" -gt 999999 ]; then | |
612 | error "NUM is not between 1 and 999999 ($NUM)" | |
613 | fi | |
614 | ;; | |
615 | r) RADIUS="$OPTARG" | |
616 | RADIUS_NUM=`awk -v "N=$RADIUS" 'BEGIN{printf "%d\n", N}'` | |
617 | case "$RADIUS" in | |
cffb6fa9 RN |
618 | *km*|*KM*) |
619 | RADIUS_UNITS=km | |
620 | RADIUS_MILES=`dc -e "6k $RADIUS_NUM 1.609344/p" ` | |
621 | ;; | |
622 | *) | |
623 | RADIUS_MILES=$RADIUS_NUM | |
624 | RADIUS_UNITS=mi | |
625 | ;; | |
06ee5545 RN |
626 | esac |
627 | ;; | |
628 | s) BABELFLAGS="$BABELFLAGS -s"; GEONUKE=",nuke_placer";; | |
629 | S) OUTFMT="gpsdrive.sql";; | |
630 | t) SQLTAG="$OPTARG";; | |
631 | u) USERNAME="$OPTARG";; | |
632 | p) PASSWORD="$OPTARG";; | |
633 | o) OUTFMT="$OPTARG";; | |
634 | O) OUTFILE="$OPTARG";; | |
635 | H) HTMLDIR="$OPTARG";; | |
636 | L) LOGDIR="$OPTARG";; | |
637 | D) DEBUG="$OPTARG";; | |
638 | U) echo "Getting latest version of this script..." | |
639 | curl $CURL_OPTS -o$UPDATE_FILE "$UPDATE_URL" | |
640 | chmod +x "$UPDATE_FILE" | |
641 | echo "Latest version is in $UPDATE_FILE" | |
642 | exit | |
643 | ;; | |
644 | h|\?|-) usage;; | |
645 | esac | |
646 | done | |
647 | ||
648 | shiftamt=`expr $OPTIND - 1` | |
649 | shift $shiftamt | |
650 | ||
651 | case "$SOC" in | |
652 | 0) EXCLUDE="$EXCLUDE|-soc";; | |
653 | esac | |
654 | ||
655 | case "$OUTFMT" in | |
656 | map) | |
657 | OUTFMT=tiger,newmarker=grnpin,iconismarker | |
658 | MAP=1 | |
659 | ;; | |
660 | map,*) | |
cffb6fa9 | 661 | MAPOPTS=`echo "$OUTFMT" | $sed -n 's/map,\(.*\)$/\1/p'` |
06ee5545 RN |
662 | OUTFMT=tiger,newmarker=grnpin,iconismarker |
663 | MAP=1 | |
664 | ;; | |
665 | ||
666 | gpsdrive.sql) | |
667 | OUTFMT=gpsdrive | |
668 | SQL=1 | |
669 | # DEBUG=1 | |
670 | ;; | |
671 | \?) | |
672 | gpsbabel_formats | |
673 | exit | |
674 | ;; | |
675 | esac | |
676 | ||
677 | LOGUSERNAME="$USERNAME" | |
678 | return $shiftamt | |
679 | } | |
680 | ||
681 | # | |
682 | # Get viewstate | |
683 | # | |
684 | gc_getviewstate() { | |
685 | file=$1 | |
686 | get_value __EVENTTARGET $file | |
687 | get_value __EVENTARGUMENT $file | |
688 | get_value __VIEWSTATEFIELDCOUNT $file | |
689 | get_value __VIEWSTATE $file; __VIEWSTATE=`urlencode "$__VIEWSTATE"` | |
690 | get_value __EVENTVALIDATION $file; | |
691 | __EVENTVALIDATION=`urlencode "$__EVENTVALIDATION"` | |
692 | viewstate= | |
693 | viewstate="$viewstate -d __VIEWSTATE=$__VIEWSTATE" | |
694 | if [ "$__VIEWSTATEFIELDCOUNT" != "" ]; then | |
06ee5545 | 695 | viewstate="$viewstate -d __VIEWSTATEFIELDCOUNT=$__VIEWSTATEFIELDCOUNT" |
cffb6fa9 RN |
696 | fc=1 |
697 | while test $fc -lt $__VIEWSTATEFIELDCOUNT; do | |
698 | get_value __VIEWSTATE$fc $file | |
699 | eval xxx=\${__VIEWSTATE$fc} | |
700 | eval __VIEWSTATE$fc=`urlencode "$xxx"` | |
701 | eval xxx=\${__VIEWSTATE$fc} | |
702 | viewstate="$viewstate -d __VIEWSTATE$fc=$xxx" | |
703 | fc=`expr $fc + 1` | |
704 | done | |
06ee5545 RN |
705 | fi |
706 | } | |
707 | ||
708 | # | |
709 | # login to geocaching.com | |
710 | # | |
711 | # Outputs: $__VIEWSTATE | |
712 | # | |
713 | gc_login() { | |
714 | _username=`urlencode "$1"` | |
715 | _password=`urlencode "$2"` | |
716 | ||
717 | [ "$_username" != dummy ] || error "You need a www.geocaching.com username" | |
718 | [ "$_password" != dummy ] || error "You need a www.geocaching.com password" | |
719 | ||
720 | # | |
721 | # Get the login page so we can get a valid viewstate | |
722 | # | |
723 | LOGINPAGE=${TMP}-login.html | |
724 | CRUFT="$CRUFT $LOGINPAGE" | |
725 | ||
726 | URL="$GEOS/login/default.aspx" | |
727 | debug 1 "curl $CURL_OPTS $URL" | |
728 | curl $CURL_OPTS -s -A "$UA" -c$COOKIE_FILE \ | |
729 | -e "$URL;auto" \ | |
730 | -o $LOGINPAGE "$URL" | |
cffb6fa9 RN |
731 | [ -s $LOGINPAGE ] \ |
732 | || error "Unable to connect to gc.com; Try: CURL_OPTS=\"--sslv3\" in ~/.georc" | |
06ee5545 RN |
733 | get_value __VIEWSTATE $LOGINPAGE |
734 | __VIEWSTATE=`urlencode "$__VIEWSTATE"` | |
735 | get_value __EVENTVALIDATION $LOGINPAGE | |
736 | __EVENTVALIDATION=`urlencode "$__EVENTVALIDATION"` | |
737 | ||
738 | sleep 2 | |
739 | ||
740 | # | |
741 | # Now post the login | |
742 | # | |
743 | URL="$GEOS/login/default.aspx" | |
744 | #URL="$URL?RESET=Y" | |
cffb6fa9 RN |
745 | # |
746 | # RER 05/05/12: bug in recent curl/openssl use "--sslv3" for a workaround | |
06ee5545 | 747 | debug 1 "curl $URL" |
cffb6fa9 | 748 | dbgcmd curl $CURL_OPTS --sslv3 -s -A "$UA" -b$COOKIE_FILE -c$COOKIE_FILE \ |
06ee5545 RN |
749 | -d __VIEWSTATE="$__VIEWSTATE" \ |
750 | -d __EVENTVALIDATION="$__EVENTVALIDATION" \ | |
751 | -e "$URL;auto" \ | |
cffb6fa9 RN |
752 | -dctl00%24ContentBody%24tbUsername="$_username" \ |
753 | -dctl00%24ContentBody%24tbPassword="$_password" \ | |
754 | -dctl00%24ContentBody%24btnSignIn=Login \ | |
06ee5545 | 755 | -L "$URL" > $LOGINPAGE |
cffb6fa9 RN |
756 | if [ ! -s $LOGINPAGE ]; then |
757 | error "curl: returned /dev/null for '$LOGINPAGE'" | |
758 | fi | |
759 | if grep -q "FormErrorWidget" $LOGINPAGE; then | |
06ee5545 RN |
760 | error "Login username/password does not match." |
761 | fi | |
cffb6fa9 RN |
762 | if grep -q "If you are human" $LOGINPAGE; then |
763 | error "Have to login this time with a web browser because of Recaptcha" | |
764 | fi | |
765 | if grep -q "temporarily down for maintenance." $LOGINPAGE; then | |
766 | error "Geocaching.com is temporarily down for maintenance." | |
767 | fi | |
768 | # Make case insignificant | |
769 | if ! grep -y -q "ctl00_ContentBody_LoggedInPanel" $LOGINPAGE; then | |
770 | error "BUG: gc.com changed the login page." | |
771 | fi | |
06ee5545 RN |
772 | } |
773 | ||
774 | # | |
775 | # procedure to nag about agreeing to EasyGps download license | |
776 | # | |
777 | easy_warning() { | |
778 | cat <<-EOF | |
779 | You have not agreed to the waypoint download license at $GEO | |
780 | ||
781 | Click one of the waypoint license agreement links at $GEO, | |
782 | read and agree to the license terms, then try this program again. | |
783 | EOF | |
784 | } | |
785 | ||
786 | # | |
787 | # getcids infile cidfile xtrafile archfile number | |
788 | # | |
789 | # Wade thru the HTML and produce lists of found, notfound and new CIDs | |
790 | # | |
791 | getcids() { | |
792 | awk \ | |
793 | -v "CIDFILE=$2" \ | |
794 | -v "XTRAFILE=$3" \ | |
795 | -v "ARCHFILE=$4" \ | |
796 | -v "NUM=$5" \ | |
797 | -v "USERFOUND=$USERFOUND" \ | |
798 | -v "VARTIME=$VARTIME" \ | |
799 | -v "SOC=$SOC" \ | |
800 | -v "DATE=$date" \ | |
cffb6fa9 | 801 | -v "DATEFMT=$DATEFMT" \ |
06ee5545 RN |
802 | -v "DEBUG=$DEBUG" \ |
803 | < $1 \ | |
804 | ' | |
805 | function debug(lvl, text) { | |
806 | if (lvl <= DEBUG) | |
807 | print text > "/dev/stderr" | |
808 | } | |
809 | function hex2dec(x, val) { | |
810 | for (val = 0; length(x); x = substr(x, 2)) | |
811 | val = 16*val + index("0123456789ABCDEF", substr(x, 1, 1)) - 1 | |
812 | return val | |
813 | } | |
814 | # Convert GC0000 to 58913 | |
815 | function wp2id(wp, val) { | |
816 | sub("^GC", "", wp) | |
817 | if (DEBUG > 5) | |
818 | print "wp2id: " wp " ..." > "/dev/stderr" | |
819 | if (length(wp) <= 4 && wp < "G000") | |
820 | { | |
821 | val = hex2dec(wp) | |
822 | if (DEBUG > 5) | |
823 | print "wp2id hex: " val " ..." > "/dev/stderr" | |
824 | return val | |
825 | } | |
826 | set = "0123456789ABCDEFGHJKMNPQRTVWXYZ" | |
827 | val = 0 | |
828 | for (pos = 1; pos <= length(wp); ++pos) | |
829 | { | |
830 | val *= 31; | |
831 | val += index(set, substr(wp, pos, 1)) - 1; | |
832 | } | |
833 | val = val - 411120 | |
834 | if (DEBUG > 5) | |
835 | print "wp2id id: " val " ..." > "/dev/stderr" | |
836 | return val | |
837 | } | |
838 | function id2wp(id, val) { | |
839 | gid = ""; | |
840 | if (id < 0) | |
841 | ; | |
842 | else if (id < 65536) | |
843 | gid = sprintf("GC%04X", id) | |
844 | else | |
845 | { | |
846 | GcOffset = 16 * 31 * 31 * 31 - 65536 | |
847 | GcSet = "0123456789ABCDEFGHJKMNPQRTVWXYZ" | |
848 | id += GcOffset; | |
849 | for (i = 1; i <= 4; ++i) | |
850 | { | |
851 | gid = substr(GcSet, id%31 + 1, 1) gid | |
852 | id = int(id / 31) | |
853 | } | |
854 | tmp = substr(GcSet, id%31 + 1, 1) | |
855 | if (tmp != 0) | |
856 | { | |
857 | gid = tmp gid | |
858 | id = int(id / 31) | |
859 | } | |
860 | if (id) | |
861 | gid = "" | |
862 | else | |
863 | gid = "GC" gid | |
864 | } | |
865 | if (DEBUG >= 5) | |
866 | print "id = ", id, "wp = ", gid > "/dev/stderr" | |
867 | return gid | |
868 | } | |
cffb6fa9 | 869 | |
06ee5545 | 870 | function hash2sdt(text, |
cffb6fa9 RN |
871 | day, cmd, i, value, mod, s0, t0, d0) { |
872 | if (thiscode == "") | |
873 | { | |
874 | code[1] = "PDMTRXBJjZNH9fKW8vetlCgbos24mydknahrcOxYqLwGSE370z65ApVFQ" | |
875 | code[2] = "6JKt4b70rHoeTLqQB2jANwcYSMW9xPk3aplE5hGvyfsZDgFmndOVRC8zX" | |
876 | code[3] = "Rl2QaDdTsnK7rOZoywzx0H58NLCAGqgtp43mvYJVkFEejMWB69cXhfSbP" | |
877 | code[4] = "jShz6gF3OvonKEWkJCbZf4GeDX25VcyPsqwRYpLM8BtATm0x7l9aHQdNr" | |
878 | code[5] = "aJs2OWpoYfN9zvDHCwZrm83b4SKkFhXcAL0QPgBy7tMdRGlqxjEVn65Te" | |
879 | code[6] = "g6aXp7rqvkEl5JLnRDhfWGHTeA9OMQYdwS2BcbK0jCyFx4sPo83tmzZVN" | |
880 | code[7] = "MlocEO5eAgTFfLXzyPDk2qJhjBn98QSdNHR47mGWtY3ZvVxCpr0Ks6abw" | |
881 | code[8] = "p7SPbvzM4VkQ8f25cJL0gBlWjYmFoKhArXRHZntTGe6CNsyE3a9qOdwDx" | |
882 | code[9] = "znlBb8GHNdhCKyOpo53ePX9xfkvjD0aTEAVZ72qSQwrmMYct6JgRFWLs4" | |
883 | code[10] = "KXztRHdrkBwSWEn3AN2Fh8vOL5QMjx6oTgyCD7JYpaGmVcf9Pelb4q0Zs" | |
884 | code[11] = "M3odkDCgQa7GWK9pYh8tLAl6xFecRwPZNbTz0s42rqfOnSmvEjVyB5JXH" | |
885 | code[12] = "gweGxNAYCKfLdrb68X93ahjHMDtTWRFJVZvl0po5ySmOQnscPz4Ekq2B7" | |
886 | code[13] = "p2zelnEox9HqkGF4V5AZ6LbMBsy0dQc8J3fDXvON7jWgrKYCSTaPtRhmw" | |
887 | code[14] = "cy2gjCAJt3KmMxbr5OzQeYsvHpLF8fRVaoNXdGBW0l7ESZDP9qkw4Th6n" | |
888 | code[15] = "9KQZlwfBsCqN6oWkYhrSpDAEP48c5m3eJn2jVHLdybMx0gaOGRXzvFTt7" | |
889 | code[16] = "7j6CJPfqbMygan42HwxFBcrK3RYLstmOXWGeSQZ9kzl0VE8oDTpAhd5Nv" | |
890 | code[17] = "YScWw7gsx49ebLAqMGotHRX3fnCv6FBOkzVE5jdlmPTarDKQJyp8Z0hN2" | |
891 | code[18] = "BgwR6oskAcVPqeYdLKr3ONvflXz2nDSbxMQJEhyZC0pWHm759GFa4tT8j" | |
892 | code[19] = "9QyolrBq37he4nbCTcXEkmGHjYS02RDgMZLfOtavVzKNFxPs58p6JwWAd" | |
893 | code[20] = "8OwsHykXLJ3tqWZmR2pbVagMcTdnSC9FveN5z0xYjGK6frh7Q4EPDBAlo" | |
894 | code[21] = "RVBh8qEaN4Aw2Ob5egSytWjQD6oFmKvXfYnCzcx73kJsHrd9TLPZM0Glp" | |
895 | code[22] = "7MqQGJkKzZvdgmRXBH8CYeAVf3cy29LOW4PNaD0rsnowEpTbF5Sxhtl6j" | |
896 | code[23] = "8C26QBoWjEPcJ3Sf7TYtexXlMNyqArm0KD9HOLvZakGwnVbFsdp45Rzhg" | |
897 | code[24] = "6e5fC2S4Bwcm9pVKyqdFE3WlRDroTLJgGtN8ZQsAabMzHYO0Xvj7Pnxhk" | |
898 | code[25] = "JXHgMGcAVNpDEwWYvyBl025m89dOoTSKPxhstjq6e4QrCk3aZFbzfn7RL" | |
899 | code[26] = "Yp3ARoesjKXB2cadmkHvFQM7SEJtqf0l9TwzbPLrZDxO8h5C6NnGWyVg4" | |
900 | code[27] = "9kJaM4HonPWtcQvsYSEK3Vy5pjCRFqNO6lgr0x7mh2b8wdTGDzfLXeABZ" | |
901 | code[28] = "4n2gSwqzGB7C0MpV3yDY9FTtol8XjkJNcEPr6LxhZQKdRvHOfsA5eWmab" | |
902 | code[29] = "P6G8jsOxBCqf3k74DnXtAvl2ThabeEHJVgZp5wSzF9NWRcYMyL0dromKQ" | |
903 | code[30] = "WVwgM93zjmNcLpHvoSK2Q5bCxaATGsryXn6Bhfd0Z8qtERk7PYOJD4Fle" | |
904 | code[31] = "PZqTVol4HOXz0GWt38Qec59rKMyabfhE2DdsNxpF7RASkBjnYwvCgmJ6L" | |
905 | cmd = sprintf("%s -u +%%-d", DATE) | |
906 | cmd | getline day; close(cmd) | |
907 | thiscode = code[day] | |
06ee5545 | 908 | } |
06ee5545 | 909 | |
cffb6fa9 RN |
910 | value = 0 |
911 | for (i = 1; i <= length(text); ++i) { | |
912 | value = 57 * value + index(thiscode, substr(text, i, 1)) - 1 | |
913 | } | |
914 | #print value | |
06ee5545 | 915 | |
cffb6fa9 | 916 | mod = (value - 131586) % 16777216 |
06ee5545 | 917 | |
cffb6fa9 RN |
918 | s0 = int(mod / 131072); |
919 | t0 = int((mod % 131072) / 256) | |
920 | d0 = int((mod % 131072) % 256) | |
921 | #print s0, t0, d0 | |
06ee5545 RN |
922 | |
923 | # cache size | |
cffb6fa9 RN |
924 | if (s0 == 0) size = "not chosen" |
925 | else if (s0 == 1) size = "Micro" | |
926 | else if (s0 == 2) size = "Regular" | |
927 | else if (s0 == 3) size = "Large" | |
928 | else if (s0 == 4) size = "Virtual" | |
929 | else if (s0 == 5) size = "Unknown" | |
930 | else if (s0 == 7) size = "Small" | |
06ee5545 RN |
931 | else size = "BUG" |
932 | ||
933 | # terrain rating | |
cffb6fa9 RN |
934 | if (t0 == 0) terrain = 1.0 |
935 | else if (t0 == 1) terrain = 1.5 | |
936 | else if (t0 == 2) terrain = 2.0 | |
937 | else if (t0 == 3) terrain = 2.5 | |
938 | else if (t0 == 4) terrain = 3.0 | |
939 | else if (t0 == 5) terrain = 3.5 | |
940 | else if (t0 == 6) terrain = 4.0 | |
941 | else if (t0 == 7) terrain = 4.5 | |
06ee5545 RN |
942 | else terrain = 5.0 |
943 | ||
944 | # difficulty rating | |
cffb6fa9 RN |
945 | if (d0 == 0) difficulty = 1.0 |
946 | else if (d0 == 1) difficulty = 1.5 | |
947 | else if (d0 == 2) difficulty = 2.0 | |
948 | else if (d0 == 3) difficulty = 2.5 | |
949 | else if (d0 == 4) difficulty = 3.0 | |
950 | else if (d0 == 5) difficulty = 3.5 | |
951 | else if (d0 == 6) difficulty = 4.0 | |
952 | else if (d0 == 7) difficulty = 4.5 | |
06ee5545 RN |
953 | else difficulty = 5.0 |
954 | } | |
cffb6fa9 | 955 | |
06ee5545 RN |
956 | function begin_new_entry() { |
957 | # Beginning of a new entry, reset variables to defaults | |
958 | inrecord = 1 | |
959 | avail = 1 | |
960 | archived = 0 | |
961 | bugs = 0 | |
962 | ifound = 0 | |
963 | iplaced = 0 | |
964 | soc = 0 | |
965 | unfound = 1 | |
966 | tdcnt = 0 | |
967 | gcid = "GC0000" | |
968 | foundt = 0 | |
969 | ifoundrec = 0 | |
970 | ifoundt = 0 | |
971 | container = "unknown" | |
972 | sendgps = 0 # Non-subscription members | |
cffb6fa9 | 973 | placer = "" |
06ee5545 RN |
974 | # Bug Fix: FIXME |
975 | #gctype = "Traditional cache" | |
976 | #type = "regular" | |
977 | #difficulty = 0 | |
978 | #terrain = 0 | |
979 | # End Bug Fix: FIXME | |
980 | if (DEBUG >= 5) | |
981 | print "Begin " NUM " ..." > "/dev/stderr" | |
982 | } | |
983 | function parse_dates(text, cmd, val) { | |
984 | sub("[*]$", "", text) | |
cffb6fa9 | 985 | sub("^ *", "", text) # remove leading blanks |
06ee5545 RN |
986 | debug(5, "parse_dates: text=" text) |
987 | if (text ~ /Today/) | |
988 | { | |
989 | cmd = sprintf("%s -d \"%s\" +%%s", DATE, "12am today") | |
990 | cmd | getline val; close(cmd) | |
991 | } | |
992 | else if (text ~ /Yesterday/) | |
993 | { | |
994 | cmd = sprintf("%s -d \"%s\" +%%s", DATE, "12am yesterday") | |
995 | cmd | getline val; close(cmd) | |
996 | } | |
997 | else if (text ~ /ago/) | |
998 | { | |
999 | cmd = sprintf("%s -d \"12am %s\" +%%s", DATE, text) | |
1000 | cmd | getline val; close(cmd) | |
1001 | } | |
1002 | else if (text ~ /[0-9][^<]*[0-9]/ ) | |
1003 | { | |
cffb6fa9 RN |
1004 | # GC Format Example Compatible |
1005 | # YYYY/MM/DD 2011/07/13 yes | |
1006 | # MM/DD/YYYY 07/13/2011 yes | |
1007 | # DD/MM/YYYY 13/07/2011 yes if DATEFMT = 1 | |
1008 | # DD/Mmm/YYYY 13/Jul/2001 no | |
1009 | # Mmm/DD/YYYY Jul/13/2011 no | |
1010 | nf = split(text, fld, "/") | |
1011 | if (nf == 3 && DATEFMT == 1) | |
1012 | { | |
1013 | # DD before MM | |
1014 | debug(5, "parse_dates: fld=" fld[1] "." fld[2] "." fld[3]) | |
1015 | cmd = sprintf("%s -d \"12am %s\" +%%s", DATE, | |
1016 | fld[2] "/" fld[1] "/" fld[3]) | |
1017 | cmd | getline val; close(cmd) | |
1018 | } | |
1019 | else | |
1020 | { | |
1021 | # Regular way | |
1022 | cmd = sprintf("%s -d \"12am %s\" +%%s", DATE, text) | |
1023 | cmd | getline val; close(cmd) | |
1024 | } | |
06ee5545 RN |
1025 | } |
1026 | else | |
1027 | val = 0 | |
1028 | debug(5, "parse_dates: val=" val) | |
1029 | return val | |
1030 | } | |
1031 | BEGIN { | |
1032 | q = sprintf("%c", 39) | |
1033 | ++NUM | |
1034 | gctype = "Traditional cache" | |
1035 | type = "regular" | |
1036 | } | |
1037 | # Test: geo-nearest -s (18) | |
1038 | # Test: geo-newest -s (20) | |
1039 | # Test: geo-nearest -b multi (8) | |
1040 | # | |
1041 | # For geo-nearest AND geo-nearest -b | |
1042 | /class=".* Data BorderTop"/ { | |
1043 | begin_new_entry() | |
1044 | debug(3, "Data BorderTop") | |
1045 | next | |
1046 | } | |
1047 | /<td/ { | |
1048 | ++tdcnt; | |
1049 | } | |
1050 | inrecord && tdcnt >= 1 && tdcnt <= 10 { | |
1051 | if (DEBUG >= 5) | |
1052 | print "tdcnt " tdcnt ", text: " $0 > "/dev/stderr" | |
1053 | } | |
1054 | /<a href="\/seek\/cache_details.aspx?[^>]*>/ { | |
1055 | name = $0 | |
1056 | ||
1057 | # remove strikeout class | |
1058 | sub("<span class[^>]*>", "", name) | |
1059 | sub("</span>", "", name) | |
1060 | sub(".*<span>", "", name) | |
1061 | sub("</span>.*", "", name) | |
1062 | ||
1063 | sub(".*<a href=.../seek/cache_details.aspx?[^>]*>", "", name) | |
1064 | sub("</a>*.", "", name) | |
1065 | sub("</[^>]*>", "", name) | |
1066 | sub("</[^>]*>", "", name) | |
1067 | sub("<[^>]*>", "", name) | |
1068 | sub("<[^>]*>", "", name) | |
cffb6fa9 RN |
1069 | debug(5, "name: " name) |
1070 | } | |
1071 | # bookmarks by guid=... | |
1072 | /<a href="http:..www.geocaching.com\/seek\/cache_details.aspx.guid=[^>]*>/ { | |
1073 | if ($0 ~ /img src/) | |
1074 | { | |
1075 | name = $0 | |
1076 | sub(".*img src[^>]*>", "", name) | |
1077 | sub("</span.*", "", name) | |
1078 | sub(".*Strike.>", "", name) | |
1079 | sub("</a.*", "", name) | |
1080 | debug(5, "bid name: " name) | |
1081 | ||
1082 | gctype = $0 | |
1083 | sub(".* alt=.", "", gctype) | |
1084 | sub(". />.*", "", gctype) | |
1085 | debug(5, "gctype: " gctype) | |
1086 | } | |
1087 | else | |
1088 | { | |
1089 | begin_new_entry() | |
1090 | gcid = $0 | |
1091 | sub(".*<a href=.*/seek/cache_details.aspx?[^>]*>", "", gcid) | |
1092 | sub("<.*", "", gcid) | |
1093 | cid = wp2id(gcid) | |
1094 | debug(5, "gcid: " gcid) | |
1095 | } | |
06ee5545 RN |
1096 | } |
1097 | /"Found It!"/ { | |
1098 | if (USERFOUND) | |
1099 | ifound = 1 | |
1100 | next | |
1101 | } | |
1102 | /Premium Member Only Cache/ { | |
1103 | soc = 1 | |
1104 | } | |
cffb6fa9 | 1105 | tdcnt == (9-0) && /[0-9][0-9]*.[0-9][0-9]*/ { |
06ee5545 RN |
1106 | # RER: May 4, 2011 |
1107 | date = $0 | |
1108 | sub(/^ */, "", date) | |
cffb6fa9 | 1109 | sub(/.*small.>/, "", date) |
06ee5545 RN |
1110 | sub(/ *<.*/, "", date) |
1111 | sub(/\015/, "", date) | |
cffb6fa9 RN |
1112 | debug(5, "placedt text: "date) |
1113 | placedt = parse_dates(date) | |
06ee5545 RN |
1114 | debug(5, "placedt: " placedt) |
1115 | } | |
1116 | # <td> | |
1117 | # Yesterday<strong>*</strong><br /> | |
1118 | # <span class="Success"> | |
1119 | # 3 days ago*</span> | |
1120 | # </td> | |
1121 | #OR | |
1122 | # <td> | |
1123 | # Yesterday<strong>*</strong><br /> | |
1124 | # <span class="Success"> | |
1125 | # </span> | |
1126 | # </td> | |
cffb6fa9 | 1127 | tdcnt == (10-0) && /[a-zA-Z0-9*>]<br .>/ { |
06ee5545 RN |
1128 | # catch dates DD MMM YY, N days ago, Today, Yesterday |
1129 | unfound = 0 | |
1130 | date = $0 | |
1131 | sub(".*<td>", "", date) | |
1132 | sub("<.*", "", date) | |
1133 | lastfoundt = parse_dates(date) | |
1134 | foundt = lastfoundt | |
1135 | debug(5, "lastfoundt: " lastfoundt) | |
1136 | } | |
cffb6fa9 | 1137 | tdcnt == (10-0) && /[a-zA-Z0-9*>]<.span>/ { |
06ee5545 RN |
1138 | unfound = 0 |
1139 | date = $0 | |
1140 | sub(".*Success.>", "", date) | |
1141 | sub("<.*", "", date) | |
1142 | ifoundt = parse_dates(date) | |
1143 | debug(5, "ifoundt: " ifoundt) | |
1144 | } | |
1145 | /<strike><font/ { | |
1146 | # i.e. <font color="red"> | |
1147 | avail = 0; archived = 1 | |
1148 | } | |
1149 | /<\/strike><\/font/ { | |
1150 | # i.e. <font color="red"> | |
1151 | avail = 0; archived = 1 | |
1152 | } | |
1153 | /<strike>[^<]/ { | |
1154 | avail = 0; | |
1155 | } | |
cffb6fa9 RN |
1156 | /class=".*Warning Strike.*">/ { |
1157 | debug(5, "***archived***") | |
1158 | avail = 0; archived = 1 | |
1159 | } | |
06ee5545 RN |
1160 | /class=".*Strike">/ { |
1161 | avail = 0 | |
1162 | } | |
cffb6fa9 RN |
1163 | tdcnt == 6 && /^ *by / { |
1164 | placer = $0 | |
1165 | sub("^ *by ", "", placer) | |
1166 | sub("\015", "", placer) | |
1167 | debug(5, "placer: <" placer ">") | |
06ee5545 | 1168 | } |
cffb6fa9 | 1169 | tdcnt == 6 && /^ *GC/ { |
06ee5545 RN |
1170 | gcid = $0 |
1171 | sub("^ *", "", gcid) | |
1172 | sub("\015", "", gcid) | |
1173 | debug(5, "GCID: <" gcid ">") | |
1174 | cid = wp2id(gcid) | |
1175 | } | |
06ee5545 RN |
1176 | /alt=.Your Geocache/ { |
1177 | # altmy cache | |
1178 | # A mistake in the HTML! should be alt="my cache" | |
1179 | if (USERFOUND) { | |
1180 | iplaced = 1 | |
1181 | ifound = 1 | |
1182 | } | |
1183 | } | |
1184 | /alt="Earthcache"/ { | |
1185 | type = "earth" | |
1186 | gctype = "Earthcache" | |
1187 | } | |
1188 | /alt="Event Cache"/ { | |
1189 | type = "event" | |
1190 | gctype = "Event cache" | |
1191 | } | |
1192 | /alt="Cache In Trash Out Event"/ { | |
1193 | type = "cito" | |
1194 | gctype = "Cache In Trash Out Event" | |
1195 | } | |
1196 | /alt="Traditional Cache"/ { | |
1197 | type = "regular" | |
1198 | gctype = "Traditional cache" | |
1199 | } | |
1200 | /alt="Letterbox Hybrid"/ { | |
1201 | type = "hybrid" | |
1202 | gctype = "Letterbox Hybrid" | |
1203 | } | |
1204 | /alt="Multi-cache"/ { | |
1205 | type = "multi" | |
1206 | gctype = "Multi-Cache" | |
1207 | } | |
1208 | /alt="Unknown Cache"/ { | |
1209 | type = "unknown" | |
1210 | gctype = "Unknown Cache" | |
1211 | } | |
1212 | /alt="Virtual Cache"/ { | |
1213 | type = "virtual" | |
1214 | gctype = "Virtual cache" | |
1215 | } | |
1216 | /alt="Webcam Cache"/ { | |
1217 | type = "webcam" | |
1218 | gctype = "Webcam Cache" | |
1219 | } | |
1220 | /alt="Wherigo Cache"/ { | |
1221 | type = "wherigo" | |
1222 | gctype = "Wherigo Cache" | |
1223 | } | |
1224 | /alt="Mega-Event Cache"/ { | |
1225 | type = "mega" | |
1226 | gctype = "Mega-Event Cache" | |
1227 | } | |
1228 | /name=.CID. / { | |
1229 | i = match($0, "value=.") | |
1230 | cid = substr($0, i+7, 99) + 0 | |
1231 | } | |
1232 | /name=.BID. / { | |
1233 | i = match($0, "value=.") | |
1234 | bid = substr($0, i+7, 99) + 0 | |
1235 | } | |
1236 | #/ImgGen.seek.CacheInfo.ashx?v=/ { | |
cffb6fa9 | 1237 | archived && /[?]v=/ { |
06ee5545 RN |
1238 | text = $0 |
1239 | debug(5, "sdt text: " text) | |
1240 | sub(".*?v=", "", text) | |
1241 | sub(". .*", "", text) | |
1242 | debug(5, "sdt text: " text) | |
1243 | hash2sdt(text) | |
1244 | debug(5, "size: " size " difficulty: " difficulty " terrain: " terrain) | |
1245 | container = size | |
1246 | } | |
1247 | /Unapproved cache/ && inrecord { | |
1248 | avail = 0; archived = 1 | |
1249 | } | |
1250 | /class=.Checkbox NoBolding./ { | |
1251 | sendgps = 1 # Subscription members | |
1252 | } | |
1253 | /<\/[tT][rR]>/ && inrecord { | |
1254 | # RER mod 11/22/10: if (0 && .. | |
1255 | if (0 && sendgps == 0) | |
1256 | next | |
1257 | inrecord = 0 | |
1258 | debug(5, "End") | |
1259 | ||
1260 | strtype = "Geocache" | |
1261 | ||
1262 | if (soc) strtype = strtype "-soc" | |
1263 | if (unfound) strtype = strtype "-unfound" | |
1264 | else if (ifound) strtype = strtype "-ifound" | |
1265 | if (!avail) strtype = strtype "-unavail" | |
1266 | if (archived) strtype = strtype "-archived" | |
1267 | ||
1268 | strtype = strtype "-" type | |
1269 | ||
1270 | # gpsbabel only allows one time in the DB, figure out what | |
1271 | # time to use for this, but always carry all three times | |
1272 | # in the .xtra file | |
1273 | if (iplaced) ifoundt = placedt | |
1274 | vartime=1234 | |
1275 | if (VARTIME == "placed") vartime = placedt | |
1276 | else if (VARTIME == "ifound") vartime = ifoundt | |
cffb6fa9 | 1277 | #else if (foundt > 0) vartime = foundt |
06ee5545 RN |
1278 | else vartime = placedt |
1279 | ||
1280 | # date -d 1970-01-01 1237093200 sec +%Y%m%d | |
1281 | # 20090315 | |
1282 | cmd = sprintf("%s -d \"1970-01-01 %d sec\" +%%Y%%m%%d", | |
1283 | DATE, lastfoundt) | |
1284 | cmd | getline lastfound; close(cmd) | |
1285 | ||
1286 | # avail=1 is the choice right now (8/19/05) | |
1287 | # archived=0 is the choice right now (11/14/06) | |
cffb6fa9 RN |
1288 | # sendgps=1 Remove non-soc caches from basic member. (12/18/2012) |
1289 | debug(5, gcid "\tsendgps: " sendgps " archived: " archived \ | |
1290 | " SOC: " SOC " soc: " soc " bid: " bid) | |
1291 | if (!archived && (SOC || !soc) && (sendgps == 1 || bid) ) | |
06ee5545 RN |
1292 | { |
1293 | if (bid != 0) { | |
1294 | printf("-dBID=%d\n", bid) > CIDFILE | |
1295 | gcid = id2wp(bid) | |
1296 | } else if (cid != 0) { | |
1297 | printf("-dCID=%d\n", cid) > CIDFILE | |
1298 | } | |
1299 | # GCID type vartime ifound soc iplaced tPLACED tFOUND tIFOUND | |
1300 | #"%.1f\t%.1f\t%s\t" | |
1301 | #difficulty, terrain, container, | |
1302 | printf "%s\t%s\t%d\t%s\t%s\t%s\t%d\t%d\t%d\t" \ | |
cffb6fa9 RN |
1303 | "%s\t%s\t%s\t%s\n", |
1304 | gcid, strtype, vartime, | |
1305 | ifound, soc, iplaced, placedt, foundt, ifoundt, | |
1306 | gctype, placer, "hint", lastfound >> XTRAFILE | |
06ee5545 RN |
1307 | if (--NUM == 0) |
1308 | exit | |
1309 | } | |
1310 | else if (archived) | |
1311 | { | |
1312 | if (DEBUG > 5) | |
1313 | print "Archived ", gcid, " " name > "/dev/stderr" | |
1314 | if (name == "") | |
1315 | name = "error-no-name!" | |
cffb6fa9 RN |
1316 | if (placer == "") |
1317 | placer = "unknown-placer" | |
06ee5545 | 1318 | # Use "merge" file output, NOT XTRAFILE! |
cffb6fa9 RN |
1319 | printf "%s\t%s\t0.0\t0.0\t%s%s%s\t" \ |
1320 | "%.1f\t%.1f\t%s\t" \ | |
1321 | "%s\t%d\t%d\t%d\t%d\t%d\t%d\t%s\t%s\t%s\t%s\t%s\n", | |
06ee5545 RN |
1322 | gcid, name, |
1323 | "http://www.geocaching.com/seek/cache_details.aspx", | |
1324 | "?log=y&wp=", gcid, | |
1325 | difficulty, terrain, container, | |
1326 | strtype, vartime, | |
1327 | ifound, soc, iplaced, placedt, foundt, ifoundt, | |
cffb6fa9 | 1328 | gctype, placer, "hint", lastfound >> ARCHFILE |
06ee5545 RN |
1329 | if (--NUM == 0) |
1330 | exit | |
1331 | } | |
1332 | } | |
1333 | ' | |
1334 | } | |
1335 | ||
1336 | # | |
1337 | # A temporary style we can use for merging the loc data with | |
1338 | # the scraped html data. This is a dual purpose hack. We | |
1339 | # use it as an output format to convert the .loc data to a | |
1340 | # record-per-line format. We use it as an input format to | |
1341 | # read up the merged data. | |
1342 | # | |
1343 | make_scrape_style() { | |
1344 | cat <<-EOF | |
1345 | ENCODING UTF-8 | |
1346 | FIELD_DELIMITER TAB | |
1347 | RECORD_DELIMITER NEWLINE | |
1348 | BADCHARS TAB | |
1349 | IFIELD SHORTNAME, "", "%s" | |
1350 | IFIELD DESCRIPTION, "", "%s" | |
cffb6fa9 RN |
1351 | IFIELD LAT_DECIMAL, "", "%.6f" |
1352 | IFIELD LON_DECIMAL, "", "%.6f" | |
06ee5545 RN |
1353 | IFIELD URL, "", "%s" |
1354 | IFIELD GEOCACHE_DIFF, "", "%3.1f" #difficulty | |
1355 | IFIELD GEOCACHE_TERR, "", "%3.1f" #terrain | |
1356 | IFIELD GEOCACHE_CONTAINER,"", "%s" #container (not set) | |
1357 | IFIELD ICON_DESCR, "", "%s" #strtype (Geocache-*) | |
1358 | IFIELD TIMET_TIME, "", "%ld" #variable time | |
1359 | IFIELD IGNORE, "", "%s" #ifound | |
1360 | IFIELD IGNORE, "", "%s" #soc | |
1361 | IFIELD IGNORE, "", "%s" #iplaced | |
1362 | IFIELD IGNORE, "", "%s" #placed time | |
1363 | IFIELD IGNORE, "", "%s" #found time | |
1364 | IFIELD IGNORE, "", "%s" #ifound time | |
1365 | IFIELD GEOCACHE_TYPE, "", "%s" #gc.com type | |
cffb6fa9 | 1366 | IFIELD GEOCACHE_PLACER, "", "%s" #gc.com placer |
06ee5545 RN |
1367 | IFIELD GEOCACHE_HINT, "", "%s" #gc.com hint |
1368 | IFIELD GEOCACHE_LAST_FOUND, "","%s" #gc.com last found | |
1369 | OFIELD SHORTNAME, "", "%s" | |
1370 | OFIELD DESCRIPTION, "", "%s" | |
cffb6fa9 RN |
1371 | OFIELD LAT_DECIMAL, "", "%.6f" |
1372 | OFIELD LON_DECIMAL, "", "%.6f" | |
06ee5545 RN |
1373 | OFIELD URL, "", "%s" |
1374 | OFIELD GEOCACHE_DIFF, "", "%3.1f" #difficulty | |
1375 | OFIELD GEOCACHE_TERR, "", "%3.1f" #terrain | |
1376 | OFIELD GEOCACHE_CONTAINER,"", "%s" #container (not set) | |
1377 | EOF | |
1378 | # OFIELD ICON_DESCR, "", "%s" | |
1379 | } | |
1380 | ||
1381 | # | |
cffb6fa9 | 1382 | # csv2csv geo-mystery-file |
06ee5545 RN |
1383 | # |
1384 | csv2csv() { | |
1385 | if [ ! -e "$1" ]; then | |
1386 | error "Don't have a -M '$1' file" | |
1387 | fi | |
1388 | ||
1389 | awk -v GEOMYSTERY=$1 ' | |
1390 | function latlon ( val ) { | |
1391 | if (val ~ ".[.]..*[.].*") | |
1392 | { | |
1393 | if (val ~ "[-wWsS]") | |
1394 | { | |
1395 | val = substr(val, 2) | |
1396 | neg = 1 | |
1397 | } | |
1398 | else if (val ~ "[nNeE]") | |
1399 | { | |
1400 | val = substr(val, 2) | |
1401 | neg = 0 | |
1402 | } | |
1403 | else | |
1404 | neg = 0 | |
1405 | dd = int(val) | |
1406 | sub("[^.]*[.]", "", val) | |
1407 | dd += (val+0.0) / 60.0 | |
1408 | return neg ? -dd : dd | |
1409 | } | |
1410 | return val | |
1411 | } | |
1412 | BEGIN { | |
1413 | while (getline <GEOMYSTERY > 0) | |
1414 | { | |
1415 | if ($0 ~ "^#") | |
1416 | continue | |
1417 | if ($0 ~ "^[ ]*$") | |
1418 | continue | |
cffb6fa9 RN |
1419 | gc=$1 |
1420 | sub(".*/", "", gc) # remove stuff like http://coord.info/ | |
06ee5545 RN |
1421 | if ($2 ~ "unk") |
1422 | { | |
cffb6fa9 RN |
1423 | lat[gc] = 48 |
1424 | lon[gc] = -88 | |
06ee5545 RN |
1425 | } |
1426 | else | |
1427 | { | |
cffb6fa9 RN |
1428 | lat[gc] = $2 |
1429 | lon[gc] = $3 | |
06ee5545 RN |
1430 | } |
1431 | } | |
1432 | FS=" " | |
1433 | } | |
1434 | { | |
1435 | if (lat[$1] != 0) | |
1436 | { | |
1437 | deglat = latlon( lat[$1] ) | |
1438 | deglon = latlon( lon[$1] ) | |
1439 | # GC# Desc lat lon URL diff terr size | |
cffb6fa9 RN |
1440 | printf "%s\t", $1 |
1441 | printf "+%s\t", $2 | |
06ee5545 RN |
1442 | printf "%.6f\t%.6f\t", deglat, deglon |
1443 | for (i = 5; i < NF; ++i) | |
1444 | printf "%s\t", $i | |
1445 | printf "%s\n", $NF | |
1446 | } | |
1447 | else | |
1448 | ||
1449 | } | |
1450 | ' | |
1451 | } | |
1452 | ||
cffb6fa9 RN |
1453 | # |
1454 | # Fetch the cache patch | |
1455 | # | |
1456 | gc_fetch_cache_page(){ | |
1457 | _page=$1 #out | |
1458 | _gcid=$2 #in | |
1459 | _url=$3 #in | |
1460 | _extra=$4 #in | |
1461 | ||
1462 | # Fetch the basic cache page | |
1463 | echo "<BASE HREF=http://www.geocaching.com/seek/>" > $_page | |
1464 | debug 1 "${_extra}curl $_url #cache fetch $_gcid.html" | |
1465 | curl $CURL_OPTS -L -s -b $COOKIE_FILE -A "$UA" \ | |
1466 | $_url \ | |
1467 | | tr -d "\001\007\010\013\017\020\031" \ | |
1468 | | sed -e '/<\/html>/d' \ | |
1469 | >> $_page | |
1470 | ||
1471 | size=$(ls -l $_page | awk '{print $5}') | |
1472 | if [ $size -lt 1000 ]; then | |
1473 | debug 0 "Could not retrieve web page for cache $_gcid" | |
1474 | return 1 | |
1475 | fi | |
1476 | ||
1477 | # JSON for logbooks | |
1478 | JSON_NUM=100 | |
1479 | tkn=`grep "userToken = '" $_page | sed -e "s/userToken = '//" -e "s/'.*//" ` | |
1480 | jsurl="http://www.geocaching.com/seek/geocache.logbook" | |
1481 | jsurl="$jsurl?tkn=$tkn" | |
1482 | jsurl="$jsurl&idx=1&num=$JSON_NUM&sp=false&sf=false&decrypt=false" | |
1483 | debug 1 "curl $jsurl" >&2 | |
1484 | curl $CURL_OPTS -s -A "$UA" -b $COOKIE_FILE \ | |
1485 | "$jsurl" >> $_page | |
1486 | echo "" >> $_page | |
1487 | echo "</html>" >> $_page | |
1488 | return 0 | |
1489 | } | |
1490 | ||
06ee5545 RN |
1491 | # |
1492 | # Query the gc website | |
1493 | # | |
1494 | gc_query() { | |
1495 | if [ $USERFOUND = 0 ]; then | |
1496 | FOUND=1 | |
1497 | fi | |
1498 | if [ $FOUND = 0 ]; then | |
1499 | SEARCH="$SEARCH&f=1" | |
1500 | fi | |
cffb6fa9 RN |
1501 | if [ "$RADIUS_NUM" != "" ]; then |
1502 | SEARCH="$SEARCH&dist=$RADIUS_MILES" | |
1503 | fi | |
06ee5545 RN |
1504 | |
1505 | if [ $DEBUG -gt 0 ]; then | |
1506 | TMP=/tmp/geo | |
1507 | else | |
1508 | TMP=/tmp/geo$$ | |
1509 | fi | |
1510 | ||
1511 | HTMLPAGE=$TMP.page | |
1512 | CIDFILE=$TMP.cids | |
1513 | LOCFILE=$TMP.loc | |
1514 | LOCTMPFILE=$TMP.tmp | |
1515 | XTRAFILE=$TMP.xtra | |
1516 | CSVFILE=$TMP.csv | |
1517 | CSVFILE2=$TMP.csv2 | |
1518 | JOINFILE=$TMP.join | |
1519 | MERGEFILE=$TMP.merge | |
1520 | ARCHFILE=$TMP.arch | |
1521 | OUTWAY=$TMP.way | |
1522 | STYLE=$TMP.newstyle | |
1523 | ||
1524 | CRUFT="$CRUFT $HTMLPAGE" | |
1525 | CRUFT="$CRUFT $CIDFILE" | |
1526 | CRUFT="$CRUFT $LOCFILE" | |
1527 | CRUFT="$CRUFT $LOCTMPFILE" | |
1528 | CRUFT="$CRUFT $XTRAFILE" | |
1529 | CRUFT="$CRUFT $CSVFILE" | |
1530 | CRUFT="$CRUFT $CSVFILE2" | |
1531 | CRUFT="$CRUFT $JOINFILE" | |
1532 | CRUFT="$CRUFT $MERGEFILE" | |
1533 | CRUFT="$CRUFT $ARCHFILE" | |
1534 | CRUFT="$CRUFT $OUTWAY" | |
1535 | CRUFT="$CRUFT $STYLE" | |
1536 | if [ $NOCOOKIES = 1 ]; then | |
1537 | CRUFT="$CRUFT $COOKIE_FILE" | |
1538 | fi | |
1539 | ||
1540 | # | |
1541 | # Login to gc.com | |
1542 | # | |
1543 | gc_login "$USERNAME" "$PASSWORD" | |
1544 | ||
1545 | # | |
1546 | # Find the bookmark | |
1547 | # | |
1548 | if [ "$BOOKMARK" != "" ]; then | |
1549 | URL="$GEO/bookmarks" | |
1550 | debug 1 "$start: curl $URL #bookmark" | |
1551 | SEARCH=` | |
1552 | curl $CURL_OPTS -L -s -b $COOKIE_FILE -A "$UA" "$URL" \ | |
1553 | | grep -y ">$BOOKMARK<" \ | |
cffb6fa9 RN |
1554 | | $sed \ |
1555 | -e 's@. title.*@@' \ | |
1556 | -e 's@^.*href=.http://www.geocaching.com/@@' -e 's/.>.*$//' | |
06ee5545 RN |
1557 | ` |
1558 | case "$SEARCH" in | |
1559 | *bookmarks*) ;; | |
1560 | *) error "No bookmark with the name '$BOOKMARK'.";; | |
1561 | esac | |
1562 | fi | |
1563 | ||
cffb6fa9 RN |
1564 | # |
1565 | # Find the pocket query | |
1566 | # | |
1567 | if [ "$POCKETQUERY" != "" ]; then | |
1568 | URL="$GEO/pocket/default.aspx" | |
1569 | debug 1 "$start: curl $URL #pocket-query" | |
1570 | SEARCH=` | |
1571 | curl $CURL_OPTS -L -s -b $COOKIE_FILE -A "$UA" "$URL" \ | |
1572 | | grep -y "title=\"$POCKETQUERY\"" \ | |
1573 | | $sed \ | |
1574 | -e 's@.*guid=@@' -e 's@".*@@' | |
1575 | ` | |
1576 | case "$SEARCH" in | |
1577 | *-*-*-*-*) SEARCH="?pq=$SEARCH";; | |
1578 | *) error "No PQ with the name '$POCKETQUERY'.";; | |
1579 | esac | |
1580 | fi | |
1581 | ||
06ee5545 RN |
1582 | # |
1583 | # We might combine one or more pages into a single XML, so cobble | |
1584 | # up a header with the ?xml and loc tags. | |
1585 | # | |
1586 | cat <<-EOF > $LOCFILE | |
1587 | <?xml version="1.0" encoding="UTF-8"?> | |
1588 | <loc version="1.0" src="EasyGPS"> | |
1589 | EOF | |
1590 | ||
1591 | # | |
1592 | # Loop, getting at least "NUM" locations | |
1593 | # | |
1594 | ((NUM=NUM-1)) | |
1595 | if [ $DEBUG -gt 0 ]; then | |
1596 | filter2="tee $TMP.bulk" | |
1597 | else | |
1598 | filter2=cat | |
1599 | fi | |
1600 | ||
1601 | subscriber=1 | |
1602 | > $XTRAFILE | |
1603 | > $ARCHFILE | |
1604 | ((start=0)) | |
1605 | while ((start <= NUM)); do | |
1606 | sleep $GEOSLEEP | |
1607 | ||
1608 | # | |
1609 | # Fetch the page of closest caches and scrape the cache ID's | |
1610 | # | |
1611 | case "$SEARCH" in | |
1612 | *bookmark*) | |
1613 | URL="$GEO/$SEARCH" | |
1614 | ;; | |
1615 | *) | |
1616 | URL="$GEO/seek/nearest.aspx" | |
1617 | URL="$URL$SEARCH" | |
1618 | ;; | |
1619 | esac | |
1620 | debug 1 "$start: curl $URL #list" | |
1621 | if ((start > 0)); then | |
1622 | # "postback"... grab the "next" button | |
1623 | case "$SEARCH" in | |
1624 | *bookmark*) | |
cffb6fa9 RN |
1625 | # __EVENTTARGET="ListInfo\$pgrBMItems\$_ctl08" |
1626 | # Get the last TGT (== ctl08) | |
1627 | TGT=$($sed -n "s/^.*__doPostBack('.*pgrBMItems\$\(.*\)','.*/\1/p" \ | |
1628 | < $HTMLPAGE) | |
1629 | if [ "$TGT" = "" ]; then | |
1630 | error "TGT is blank!" | |
1631 | fi | |
1632 | __EVENTTARGET="ctl00%24ContentBody%24ListInfo%24pgrBMItems%24$TGT" | |
06ee5545 RN |
1633 | ;; |
1634 | *) | |
cffb6fa9 | 1635 | TGT=$($sed -n "s/^.*__doPostBack('.*pgrTop\$\(.*\)','.*/\1/p" \ |
06ee5545 RN |
1636 | < $HTMLPAGE) |
1637 | if [ "$TGT" = "" ]; then | |
1638 | error "TGT is blank!" | |
1639 | fi | |
1640 | __EVENTTARGET="ctl00%24ContentBody%24pgrTop%24$TGT" | |
1641 | ;; | |
1642 | esac | |
1643 | curl $CURL_OPTS -L -s -b $COOKIE_FILE -A "$UA" \ | |
1644 | -d __EVENTTARGET="$__EVENTTARGET" \ | |
1645 | $viewstate \ | |
1646 | "$URL" \ | |
cffb6fa9 | 1647 | | $sed -e "s/'/'/g" -e "s/\r//" > $HTMLPAGE |
06ee5545 RN |
1648 | else |
1649 | curl $CURL_OPTS -L -s -b $COOKIE_FILE -A "$UA" \ | |
1650 | "$URL" \ | |
cffb6fa9 | 1651 | | $sed -e "s/'/'/g" -e "s/\r//" > $HTMLPAGE |
06ee5545 RN |
1652 | if [ "$DEBUG" -ge 1 ]; then |
1653 | grep "Total Records:.*Top.*" $HTMLPAGE | | |
1654 | sed -e "s/<.b>.*//" -e "s/^.*span>//" -e "s/<b>//" 1>&2 | |
1655 | fi | |
1656 | fi | |
1657 | rc=$?; if [ $rc != 0 ]; then | |
1658 | error "curl: fetch $URL" | |
1659 | fi | |
1660 | if grep -s -q "We encountered an error when requesting that page!" \ | |
1661 | $HTMLPAGE; then | |
1662 | error "searching error (1) on $start" | |
1663 | fi | |
1664 | if grep -s -q "has resulted in an error" \ | |
1665 | $HTMLPAGE; then | |
1666 | error "searching error (2) on $start" | |
1667 | fi | |
1668 | if grep -s -q "By State" $HTMLPAGE; then | |
1669 | error "searching gave up on $start" | |
1670 | fi | |
1671 | if grep -s -q ">Advanced Search<" $HTMLPAGE; then | |
1672 | error "need a country AND a state!" | |
1673 | fi | |
1674 | ||
1675 | # | |
1676 | # Grab a few important values from the page | |
1677 | # | |
1678 | gc_getviewstate $HTMLPAGE | |
cffb6fa9 | 1679 | # cp $HTMLPAGE /tmp/view$start.html |
06ee5545 RN |
1680 | |
1681 | # | |
1682 | # Grab the CIDs into two categories: found and notfound | |
1683 | # | |
1684 | > $CIDFILE | |
cffb6fa9 | 1685 | debug 3 "getcids $HTMLPAGE $CIDFILE $XTRAFILE $ARCHFILE $((NUM-start))" |
06ee5545 RN |
1686 | getcids $HTMLPAGE $CIDFILE $XTRAFILE $ARCHFILE $((NUM-start)) |
1687 | ||
1688 | # | |
1689 | # Fetch the waypoints, rip out the ?xml and loc tags, and | |
1690 | # append to the $LOCFILE file. | |
1691 | # | |
1692 | if [ -s "$CIDFILE" ]; then | |
1693 | sleep $GEOSLEEP | |
1694 | case "$SEARCH" in | |
1695 | *bookmark*) | |
1696 | URL="$GEO/$SEARCH" | |
1697 | ;; | |
1698 | *) | |
1699 | URL="$GEO/seek/nearest.aspx" | |
1700 | URL="$URL$SEARCH" | |
1701 | ;; | |
1702 | esac | |
cffb6fa9 RN |
1703 | # |
1704 | # Fetch the .loc file | |
1705 | # 1) -d "Download=Download+Waypoints"... geo-nearest | |
1706 | # 2) -d "ListInfo:btnDownload= ... old | |
1707 | # 3) -d 'ctl00$ContentBody ... -b acro | |
1708 | # | |
06ee5545 RN |
1709 | debug 2 "$start: curl $URL #loc" |
1710 | curl $CURL_OPTS -s -b $COOKIE_FILE -A "$UA" \ | |
cffb6fa9 RN |
1711 | $viewstate \ |
1712 | `cat $CIDFILE` \ | |
1713 | -d "Download=Download+Waypoints" \ | |
1714 | -d "ListInfo:btnDownload=Download+to+.Loc" \ | |
1715 | -d 'ctl00$ContentBody$ListInfo$btnDownload=Download+to+.Loc' \ | |
1716 | "$URL" \ | |
06ee5545 | 1717 | | $filter2 \ |
cffb6fa9 | 1718 | | $sed -e 's/^<?xml [^>]*>//' \ |
06ee5545 RN |
1719 | -e 's/>[gG]eocache</>Geocache</g' \ |
1720 | -e 's/<loc [^>]*>//' \ | |
1721 | -e 's#</loc>##' \ | |
cffb6fa9 | 1722 | > $LOCTMPFILE |
06ee5545 RN |
1723 | rc=$?; if [ $rc != 0 ]; then |
1724 | error "curl: fetch the waypoints" | |
1725 | fi | |
1726 | if grep -s -q "you are not logged in" $LOCTMPFILE; then | |
1727 | error "you are not logged in on $start" | |
1728 | fi | |
1729 | if grep -s -q "has resulted in an error" $LOCTMPFILE; then | |
1730 | error "searching error (3) on $start" | |
1731 | fi | |
1732 | if grep -s -q "Geocaching > Search for Geocaches" $LOCTMPFILE; then | |
1733 | error "searching error (4) on $start" | |
1734 | fi | |
1735 | if grep -s -q "recaptcha_challenge_field" $LOCTMPFILE; then | |
1736 | # | |
1737 | # Basic members: do it the SLOW way | |
1738 | # | |
1739 | if [ "$GEODIR" != "" -a ! -d "$GEODIR" ]; then | |
1740 | mkdir "$GEODIR" || error "Couldn't mkdir $GEODIR" | |
1741 | fi | |
1742 | if [ "$GEODIR/caches" != "" -a ! -d "$GEODIR/caches" ]; then | |
1743 | mkdir "$GEODIR/caches" \ | |
1744 | || error "Couldn't mkdir $GEODIR/caches" | |
1745 | fi | |
1746 | ||
1747 | if [ $subscriber = 1 ]; then | |
1748 | debug 0 "Basic Member: pay the \$30/yr fee for Premium Member!" | |
1749 | debug 0 "Basic Member: doing it the REAL SLOW way!!!" | |
1750 | timestamp="$GEODIR/caches/.timestamp" | |
1751 | $touch -d "1 day ago" $timestamp | |
1752 | fi | |
1753 | subscriber=0 | |
1754 | ||
1755 | # | |
1756 | # Pull 20 GCIDs... | |
1757 | # | |
1758 | bstart=$start | |
1759 | grep "^ *GC.*" $HTMLPAGE \ | |
cffb6fa9 | 1760 | | $sed -e "s/ //g" \ |
06ee5545 RN |
1761 | | while read gcid; do |
1762 | # | |
1763 | # Get the cache page | |
1764 | # | |
1765 | cachepage="$GEODIR/caches/$gcid.html" | |
1766 | if [ ! -s $cachepage -o $timestamp -nt $cachepage ]; then | |
1767 | # cache page doesn't exist or out of date | |
1768 | sleep 3 #$GEOSLEEP | |
cffb6fa9 | 1769 | |
06ee5545 RN |
1770 | urlpage="http://www.geocaching.com" |
1771 | urlpage="$urlpage/seek/cache_details.aspx" | |
1772 | urlpage="$urlpage?log=y&wp=$gcid" | |
cffb6fa9 RN |
1773 | gc_fetch_cache_page $cachepage $gcid $urlpage \ |
1774 | "$bstart: " | |
1775 | if [ $? != 0 ]; then | |
1776 | continue | |
1777 | fi | |
1778 | fi | |
1779 | ||
1780 | # 01/07/12: remove soc's! | |
1781 | if grep -q 'alt="Premium Members only"' $cachepage; then | |
1782 | continue | |
1783 | fi | |
1784 | # 12/18/12: remove soc's | |
1785 | if grep -q 'uxPremiumSubmitBottom' $cachepage; then | |
1786 | continue | |
1787 | fi | |
1788 | # 02/04/12: removed archived! | |
1789 | if grep -q '>This cache has been archived' $cachepage; then | |
1790 | continue | |
06ee5545 RN |
1791 | fi |
1792 | ||
1793 | # | |
1794 | # Cobble up the .loc file entry | |
1795 | # | |
1796 | name=` grep -A1 "<head><title>" $cachepage \ | |
1797 | | tail -1 | tr -d "\r" \ | |
cffb6fa9 | 1798 | | $sed -e 's/[ ]*GC[^ ]* //' -e 's/ (.*by/ by/' ` |
06ee5545 | 1799 | diff=` grep ctl00_ContentBody_uxLegendScale $cachepage \ |
cffb6fa9 | 1800 | | $sed -e 's/.*alt="//' -e 's/ .*//' ` |
06ee5545 | 1801 | terr=` grep ctl00_ContentBody_Localize6 $cachepage \ |
cffb6fa9 | 1802 | | $sed -e 's/.*alt="//' -e 's/ .*//' ` |
06ee5545 | 1803 | size=` grep "alt=.Size:" $cachepage \ |
cffb6fa9 | 1804 | | $sed -e 's/.*alt="Size: //' -e 's/" .*//' ` |
06ee5545 RN |
1805 | case $size in |
1806 | Unknown) cont=1;; | |
1807 | Micro) cont=2;; | |
1808 | Regular) cont=3;; | |
1809 | Large) cont=4;; | |
1810 | Virtual) cont=5;; | |
1811 | Other) cont=6;; | |
1812 | Small) cont=8;; | |
1813 | *) cont=1;; | |
1814 | esac | |
1815 | ||
1816 | echo "<waypoint>" | |
1817 | echo "<name id=\"$gcid\"><![CDATA[$name]]></name>" | |
1818 | grep "lnkConversions" $cachepage \ | |
cffb6fa9 | 1819 | | $sed -e 's/.*lat=/<coord lat="/' \ |
06ee5545 RN |
1820 | -e 's/&lon=/" lon="/' -e 's@&.*@"/>@' |
1821 | echo "<type>Geocache</type>" | |
1822 | echo -n "<link text=\"Cache Details\">" | |
1823 | urlpage="http://www.geocaching.com" | |
1824 | urlpage="$urlpage/seek/cache_details.aspx" | |
1825 | urlpage="$urlpage?wp=$gcid" | |
1826 | echo -n "$urlpage" | |
1827 | echo "</link>" | |
1828 | echo "<difficulty>$diff</difficulty>" | |
1829 | echo "<terrain>$terr</terrain>" | |
1830 | echo "<container>$cont</container>" | |
1831 | echo "</waypoint>" | |
1832 | ||
1833 | debug 3 "bstart: $bstart" | |
1834 | if [ $bstart == $NUM ]; then | |
1835 | break | |
1836 | fi | |
1837 | ((++bstart)) | |
1838 | #break | |
1839 | done >> $LOCFILE | |
1840 | else | |
1841 | cat $LOCTMPFILE >> $LOCFILE | |
1842 | fi | |
1843 | fi | |
1844 | ||
1845 | # | |
1846 | # Check to see if the user hasn't agreed to license terms | |
1847 | # | |
1848 | if grep -s -q "lblAgreementText" $LOCFILE; then | |
1849 | easy_warning >&2 | |
1850 | remove_cruft | |
1851 | exit | |
1852 | fi | |
1853 | ||
1854 | ((start=start+20)) | |
1855 | ||
1856 | # If the Next button is disabled, break this loop | |
1857 | # grep "Records: <b>[1-9].*disabled.><b>Next<" $HTMLPAGE | |
1858 | if grep -s -q "Records: <b>[1-9].*disabled.><b>Next " $HTMLPAGE; then | |
1859 | # echo "$start: dis" | |
1860 | break; | |
1861 | fi | |
1862 | done | |
1863 | ||
1864 | # | |
1865 | # Finish off the .loc file | |
1866 | # | |
1867 | echo "</loc>" >> $LOCFILE | |
1868 | ||
1869 | # | |
1870 | # Convert the .loc data to .csv format and join it with | |
1871 | # the extra data scraped from the HTML page. Filter out | |
1872 | # the data according to the -I and -X options. | |
1873 | # | |
1874 | # http://www.geocaching.com/seek/cache_details.aspx?wp=GCG2H4 | |
1875 | # http://www.geocaching.com/seek/cache_details.aspx?log=y&wp=GCG2H4 | |
1876 | # http://www.geocaching.com/seek/cache_details.aspx?ID=92117&log=y | |
1877 | # | |
1878 | # The joined .csv format looks like this: | |
1879 | # GCH636 Jidana 3 by rickrich 44.94520 -93.47540 \ | |
1880 | # http://www.geocaching.com/seek/cache_details.aspx?log=y&wp=GCH636 \ | |
1881 | # Geocache-ifound-regular 1070285077 1 0 1 \ | |
1882 | # 1067925600 1070285077 0 | |
1883 | # | |
1884 | make_scrape_style > $STYLE | |
1885 | ||
1886 | dbgcmd gpsbabel -i geo$GEONUKE -f $LOCFILE \ | |
1887 | -o xcsv,style=$STYLE -F $CSVFILE2 | |
1888 | if [ $? != 0 ]; then | |
1889 | cp $LOCFILE /tmp/geo.err | |
1890 | error "gpsbabel returned error code [1]" | |
1891 | fi | |
1892 | ||
1893 | # Convert CSV into CSV with puzzles... | |
1894 | csv2csv $GEOMYSTERY < $CSVFILE2 > $CSVFILE | |
1895 | ||
cffb6fa9 | 1896 | # 01/07/12: NO --nocheck-order; it is wrong! |
06ee5545 RN |
1897 | join -t ' ' $CSVFILE $XTRAFILE > $JOINFILE |
1898 | if [ $? != 0 ]; then | |
1899 | cp $CSVFILE /tmp/geo.err.csv | |
1900 | cp $XTRAFILE /tmp/geo.err.xtra | |
1901 | error "joined file error: see join /tmp/geo.err.{csv,xtra}" | |
1902 | fi | |
1903 | egrep -- "$INCLUDE" < $JOINFILE \ | |
1904 | | egrep -v -- "$EXCLUDE" \ | |
cffb6fa9 | 1905 | | $sed -e 's/wp=/log=y\&&/' > $MERGEFILE |
06ee5545 RN |
1906 | |
1907 | if [ $FOUND = 1 ]; then | |
1908 | cat $ARCHFILE \ | |
1909 | | egrep -- "$INCLUDE" | egrep -v -- "$EXCLUDE" \ | |
cffb6fa9 | 1910 | | $sed -e 's/wp=/log=y\&&/' >> $MERGEFILE |
06ee5545 RN |
1911 | fi |
1912 | ||
1913 | if [ $DEBUG -ge 2 ]; then | |
1914 | # First two of these should be the same number of lines! | |
1915 | wc -l $CSVFILE >&2 | |
1916 | wc -l $XTRAFILE >&2 | |
1917 | wc -l $MERGEFILE >&2 | |
1918 | fi | |
1919 | ||
1920 | # | |
1921 | # Convert to the desired format | |
1922 | # | |
1923 | BABELFILT= | |
1924 | if [ "$RADIUS" != "" ]; then | |
1925 | BABELFILT="-x radius,distance=$RADIUS,lat=$LAT,lon=$LON" | |
1926 | fi | |
1927 | ||
1928 | if [ $SQL = 1 ]; then | |
1929 | # | |
1930 | # add it via mysql | |
1931 | # | |
1932 | if [ "$OUTFILE" != "" ]; then | |
1933 | >"$OUTFILE" | |
1934 | fi | |
1935 | ||
1936 | if [ $PURGE = 1 ]; then | |
1937 | gpsdrive_purge | gpsdrive_mysql | |
1938 | PURGE=2 | |
1939 | fi | |
1940 | ||
1941 | dbgcmd gpsbabel $BABELFLAGS \ | |
1942 | -i xcsv,style=$STYLE -f $MERGEFILE \ | |
1943 | $BABELFILT -o "$OUTFMT" -F $OUTWAY | |
1944 | if [ $? != 0 ]; then | |
1945 | error "gpsbabel returned error code [2]" | |
1946 | fi | |
1947 | gpsdrive_add <$OUTWAY $SQLTAG | gpsdrive_mysql | |
1948 | elif [ $MAP = 1 ]; then | |
1949 | dbgcmd gpsbabel $BABELFLAGS \ | |
1950 | -i xcsv,style=$STYLE -f $MERGEFILE \ | |
1951 | $BABELFILT -o "$OUTFMT" -F $OUTWAY | |
1952 | if [ $? != 0 ]; then | |
1953 | error "gpsbabel returned error code [3]" | |
1954 | fi | |
1955 | if [ "$OUTFILE" = "" ]; then | |
1956 | dbgcmd geo-map -s0 $MAPOPTS -t$OUTWAY | |
1957 | else | |
1958 | dbgcmd geo-map -s0 $MAPOPTS -t$OUTWAY -o"$OUTFILE" | |
1959 | fi | |
1960 | else | |
1961 | # | |
1962 | # output to stdout or to a file | |
1963 | # | |
1964 | if [ "$OUTFILE" = "" ]; then | |
1965 | OUTTMP="$TMP.way"; CRUFT="$CRUFT $OUTTMP" | |
1966 | dbgcmd gpsbabel $BABELFLAGS \ | |
1967 | -i xcsv,style=$STYLE -f $MERGEFILE \ | |
1968 | $BABELFILT -o "$OUTFMT" -F $OUTTMP | |
1969 | if [ $? != 0 ]; then | |
1970 | error "gpsbabel returned error code [4]" | |
1971 | fi | |
1972 | cat $OUTTMP | |
1973 | else | |
1974 | dbgcmd gpsbabel $BABELFLAGS \ | |
1975 | -i xcsv,style=$STYLE -f $MERGEFILE \ | |
1976 | $BABELFILT -o "$OUTFMT" -F $OUTFILE | |
1977 | if [ $? != 0 ]; then | |
1978 | error "gpsbabel returned error code [5]" | |
1979 | fi | |
1980 | fi | |
1981 | fi | |
1982 | ||
1983 | # | |
1984 | # Optionally, print the HTML pages | |
1985 | # | |
1986 | if [ "$CMDPIPE" != "" ]; then | |
1987 | OIFS="$IFS" | |
1988 | IFS=" " | |
1989 | while read id desc lat lon url diff terr container strtype \ | |
1990 | vartime ifound soc iplaced placedt foundt ifoundt extra; do | |
1991 | url="$url&decrypt=y" | |
1992 | echo "Print: $url" | |
1993 | HTMLPAGE2=$TMP.html | |
1994 | CRUFT="$CRUFT $HTMLPAGE2" | |
1995 | debug 1 "curl $url" >&2 | |
1996 | dbgcmd curl $CURL_OPTS -s -A "$UA" -b $COOKIE_FILE "$url" > $HTMLPAGE2 | |
1997 | htmldoc --quiet -t ps --nup 2 --fontsize 14 --webpage $HTMLPAGE2 \ | |
1998 | | psselect -q -p1-1 | eval $CMDPIPE | |
1999 | # exit | |
2000 | done < $MERGEFILE | |
2001 | IFS="$OIFS" | |
2002 | fi | |
2003 | ||
2004 | # | |
2005 | # Optionally, fetch printable HTML pages | |
2006 | # | |
2007 | if [ "$HTMLDIR" != "" -o "$LOGDIR" != "" ]; then | |
2008 | if [ "$HTMLDIR" != "" -a ! -d "$HTMLDIR" ]; then | |
2009 | mkdir "$HTMLDIR" || error "Couldn't mkdir $HTMLDIR" | |
2010 | fi | |
2011 | if [ "$LOGDIR" != "" -a ! -d "$LOGDIR" ]; then | |
2012 | mkdir "$LOGDIR" || error "Couldn't mkdir $LOGDIR" | |
2013 | fi | |
2014 | ||
2015 | HTMLPAGE2=$TMP.html | |
2016 | CRUFT="$CRUFT $HTMLPAGE2" | |
2017 | TIMESTAMP=$TMP.time | |
2018 | CRUFT="$CRUFT $TIMESTAMP" | |
2019 | ||
2020 | fetchcnt=0 | |
2021 | fetchmax=1000 | |
2022 | if [ $DEBUG -ge 3 ]; then | |
2023 | fetchmax=3 | |
2024 | fi | |
2025 | OIFS="$IFS" | |
2026 | IFS=" " | |
2027 | while read id desc lat lon url diff terr container strtype \ | |
2028 | vartime ifound soc iplaced placedt foundt ifoundt extra; do | |
2029 | # | |
2030 | # Don't fetch page if we already have a current version | |
2031 | # | |
cffb6fa9 | 2032 | #debug 1 "placedt=<$placedt> <$foundt>" |
06ee5545 RN |
2033 | if [ $placedt -gt $foundt ]; then |
2034 | filetime="$placedt" | |
2035 | else | |
2036 | filetime="$foundt" | |
2037 | fi | |
2038 | $touch -d "1/1/70 $filetime seconds last second" $TIMESTAMP | |
2039 | if [ "$filetime" -gt 0 -a "$HTMLDIR/$id.html" -nt $TIMESTAMP ]; then | |
2040 | continue | |
2041 | fi | |
2042 | ||
2043 | # Basic member... | |
2044 | if [ "$subscriber" == 0 -a "$HTMLDIR" != "" ]; then | |
2045 | debug 2 "cp $GEODIR/caches/$id.html $HTMLDIR/$id.html" | |
2046 | cp "$GEODIR/caches/$id.html" "$HTMLDIR/$id.html" | |
2047 | # debug 2 "f: $filetime" | |
2048 | if [ "$filetime" -gt 0 ]; then | |
2049 | $touch -d "1/1/70 $filetime seconds" "$HTMLDIR/$id.html" | |
2050 | fi | |
2051 | continue | |
2052 | fi | |
2053 | ||
2054 | # Limit to 1000 caches/day | |
2055 | ((fetchcnt=fetchcnt+1)) | |
2056 | if [ $fetchcnt -gt $fetchmax ]; then | |
2057 | error "Fetch count exceeded $fetchmax. Try tomorrow!" | |
2058 | fi | |
2059 | ||
2060 | # Be kind to the server. Do not remove this sleep | |
2061 | sleep $GEOSLEEP | |
2062 | ||
cffb6fa9 | 2063 | gc_fetch_cache_page $HTMLPAGE $id $url "" |
06ee5545 | 2064 | if [ $? != 0 ]; then |
06ee5545 RN |
2065 | continue |
2066 | fi | |
2067 | ||
2068 | if [ "$HTMLDIR" != "" ]; then | |
2069 | cp $HTMLPAGE "$HTMLDIR/$id.html" || | |
2070 | error "Couldn't copy $id cache page" | |
2071 | if [ "$filetime" -gt 0 ]; then | |
2072 | $touch -d "1/1/70 $filetime seconds" "$HTMLDIR/$id.html" | |
2073 | fi | |
2074 | fi | |
2075 | ||
2076 | if [ "$LOGDIR" != "" ]; then | |
cffb6fa9 RN |
2077 | $sed -e '1,/Logged Visits/d' \ |
2078 | -e 's/<strong><img src=/\ | |
2079 | &/g' \ | |
2080 | < $HTMLPAGE | | |
2081 | egrep ">$LOGUSERNAME<|strong> \($LOGUSERNAME\) \(" | | |
2082 | $sed -e 's#<a href="log.aspx?LUID.*##' \ | |
2083 | -e 's#<a href=./profile[^ ]*##' \ | |
2084 | > $HTMLPAGE2 | |
06ee5545 RN |
2085 | lynx -dump $HTMLPAGE2 > $LOGDIR/$id.log |
2086 | if [ "$filetime" -gt 0 ]; then | |
2087 | $touch -d "1/1/70 $filetime seconds" "$LOGDIR/$id.log" | |
2088 | fi | |
2089 | fi | |
2090 | done < $MERGEFILE | |
2091 | IFS="$OIFS" | |
2092 | ||
2093 | if [ "$LOGDIR" != "" ]; then | |
cffb6fa9 | 2094 | # This is a hack, and might be inaccurate |
06ee5545 RN |
2095 | echo -n "Finds:" >&2 |
2096 | MONTHS=" January | February | March | April | May | June " | |
2097 | MONTHS="$MONTHS| July | August | September | October " | |
2098 | MONTHS="$MONTHS| November | December " | |
2099 | egrep "$MONTHS" $LOGDIR/*.log | | |
cffb6fa9 | 2100 | egrep "Found it |icon_smile|icon_happy|icon_camera" | wc -l >&2 |
06ee5545 RN |
2101 | fi |
2102 | ||
2103 | fi | |
2104 | } | |
2105 | ||
2106 | ############################################################################## | |
2107 | # end #include "geo-common-gc" | |
2108 | ############################################################################## | |
2109 | ############################################################################## | |
2110 | # begin #include "geo-common-gpsdrive" | |
2111 | ############################################################################## | |
2112 | ||
2113 | # | |
2114 | # default MySQL global options... | |
2115 | # | |
2116 | SQLUSER=gast | |
2117 | SQLPASS=gast | |
2118 | SQLDB=geoinfo | |
2119 | SQLITEDB=$HOME/.gpsdrive/waypoints.db | |
2120 | SQLTAG=Geocache | |
2121 | ||
2122 | # | |
2123 | # procedures for updating gpsdrive database via MySQL | |
2124 | # | |
2125 | # Global Vars: $SQLDB, $SQLTAG $OUTFILE | |
2126 | # | |
2127 | gpsdrive_create_210() { | |
2128 | echo "CREATE TABLE poi (" | |
cffb6fa9 | 2129 | echo "poi_id INTEGER PRIMARY KEY AUTOINCREMENT," |
06ee5545 RN |
2130 | echo "name VARCHAR(80) NOT NULL default 'not specified'," |
2131 | echo "poi_type VARCHAR(160) NOT NULL default 'unknown'," | |
2132 | echo "lat DOUBLE NOT NULL default '0'," | |
2133 | echo "lon DOUBLE NOT NULL default '0'," | |
2134 | echo "alt DOUBLE default '0'," | |
2135 | echo "comment VARCHAR(255) default NULL," | |
2136 | echo "last_modified DATETIME NOT NULL default '0000-00-00'," | |
2137 | echo "source_id INTEGER NOT NULL default '1'," | |
2138 | echo "private CHAR(1) default NULL);" | |
2139 | echo "CREATE TABLE poi_extra (" | |
2140 | echo "poi_id INTEGER NOT NULL default '0'," | |
2141 | echo "field_name VARCHAR(160) NOT NULL default '0'," | |
2142 | echo "entry VARCHAR(8192) default NULL);" | |
2143 | } | |
2144 | ||
2145 | gpsdrive_purge() { | |
2146 | case "$GPSDRIVE_VER" in | |
2147 | ""|"2.09") | |
2148 | delcmd="delete from waypoints" | |
2149 | echo "use $SQLDB;" | |
2150 | echo "$delcmd where type like '$SQLTAG%';" | |
2151 | ;; | |
2152 | "2.10") | |
2153 | if [ ! -s $SQLITEDB ]; then | |
2154 | gpsdrive_create_210 | |
2155 | fi | |
cffb6fa9 | 2156 | SQLTAG=`echo $SQLTAG | tr A-Z a-z` |
06ee5545 RN |
2157 | delcmd="delete from poi" |
2158 | echo "$delcmd where poi_type like '$SQLTAG%';" | |
2159 | ;; | |
2160 | esac | |
2161 | } | |
2162 | ||
2163 | gpsdrive_add_209() { | |
2164 | delcmd="delete from waypoints" | |
2165 | addcmd="replace into waypoints (name,lat,lon,type)" | |
cffb6fa9 | 2166 | addcmd="insert into waypoints (name,lat,lon,type)" |
06ee5545 RN |
2167 | sqltag="$1" |
2168 | echo "use $SQLDB;" | |
2169 | while read name lat lon type extra | |
2170 | do | |
2171 | name=`echo "$name" | tr -d "'"` | |
2172 | # Primary key is autoincrementing id number, so delete | |
2173 | # the old record (if any) by name and type | |
2174 | if [ $PURGE = 0 ]; then | |
2175 | echo "$delcmd where name='$name' and type like '$SQLTAG%';" | |
2176 | fi | |
2177 | ||
2178 | if [ $DELETE = 0 ]; then | |
2179 | # Add the new record | |
2180 | if [ "$sqltag" = "Geocache" ]; then | |
2181 | tag="$type" | |
2182 | else | |
2183 | tag="$sqltag" | |
2184 | fi | |
2185 | echo "$addcmd values ('$name','$lat','$lon','$tag');" | |
2186 | fi | |
2187 | done | |
2188 | } | |
2189 | ||
2190 | gpsdrive_add_210(){ | |
2191 | if [ ! -s $SQLITEDB ]; then | |
2192 | gpsdrive_create_210 | |
2193 | fi | |
cffb6fa9 | 2194 | echo "PRAGMA synchronous=OFF;" |
06ee5545 | 2195 | delcmd="delete from poi" |
cffb6fa9 RN |
2196 | addcmd="replace into poi (name,poi_type,lat,lon,alt,comment,last_modified)" |
2197 | addcmd="insert into poi (name,poi_type,lat,lon,alt,comment,last_modified)" | |
06ee5545 RN |
2198 | sqltag="$1" |
2199 | sqltag=`echo "$sqltag" | tr A-Z a-z` | |
2200 | OIFS="$IFS" | |
2201 | IFS="|" | |
2202 | #while read name lat lon type extra | |
2203 | poi_id=0 | |
cffb6fa9 RN |
2204 | egrep -v 'Geocache Found|-ifound' \ |
2205 | | tr -d "|" | tr ' ' '|' | tr -d "'" | | |
06ee5545 RN |
2206 | while read index shortname description notes url urltext icon lat lon \ |
2207 | lat32 lon32 latdecdir londecdir latdirdec londirdec latdir londir \ | |
2208 | altfeet altmeters excel timet diff terr container type \ | |
2209 | pathmiles pathkm placer yyyymmmdd hint lastfound extra | |
2210 | do | |
2211 | if [ "$placer" = "$USERNAME" ]; then | |
2212 | continue | |
2213 | fi | |
2214 | if [ "$placer" = "dyl1231" ]; then | |
2215 | continue | |
2216 | fi | |
cffb6fa9 RN |
2217 | |
2218 | #name=`echo "$description" | tr -d "'"` | |
2219 | #For Performance... | |
2220 | name=${description//\'/} | |
2221 | ||
06ee5545 RN |
2222 | # Primary key is autoincrementing id number, so delete |
2223 | # the old record (if any) by name and type | |
2224 | if [ $PURGE = 0 ]; then | |
2225 | echo "$delcmd where name='$name' and poi_type like '%$sqltag%';" | |
2226 | fi | |
2227 | ||
2228 | ((poi_id=poi_id+1)) | |
2229 | if [ $DELETE = 0 ]; then | |
2230 | # Add the new record | |
2231 | if [ "$sqltag" = "geocache" ]; then | |
2232 | tag="$type" | |
2233 | else | |
2234 | tag="$sqltag" | |
2235 | fi | |
2236 | case "$type" in | |
2237 | "Earthcache") poi_type="geocache.geocache_earth";; | |
2238 | "Event Cache") poi_type="geocache.geocache_event";; | |
2239 | "Mega-Event Cache") poi_type="geocache.geocache_event";; | |
2240 | "Cache In Trash Out Event") poi_type="geocache.geocache_event";; | |
2241 | "found") poi_type="geocache.geocache_found";; | |
2242 | "Multi-cache") poi_type="geocache.geocache_multi";; | |
2243 | "Unknown Cache") poi_type="geocache.geocache_mystery";; | |
2244 | "night") poi_type="geocache.geocache_night";; | |
2245 | "Traditional Cache") poi_type="geocache.geocache_traditional";; | |
2246 | "Virtual Cache") poi_type="geocache.geocache_virtual";; | |
2247 | "Webcam Cache") poi_type="geocache.geocache_webcam";; | |
2248 | *) poi_type="geocache";; | |
2249 | esac | |
cffb6fa9 | 2250 | echo -n "$addcmd values ('$name','$poi_type'," |
06ee5545 RN |
2251 | echo -n "'$lat','$lon'," |
2252 | echo -n "'0.0'," #alt | |
2253 | echo -n "'$type $container $diff/$terr. Last Found: $lastfound. $hint'," | |
2254 | echo "'0');" | |
2255 | fi | |
2256 | done | |
2257 | IFS="$OIFS" | |
2258 | } | |
2259 | ||
2260 | gpsdrive_add() { | |
2261 | case "$GPSDRIVE_VER" in | |
2262 | ""|"2.09") gpsdrive_add_209 "$1";; | |
2263 | "2.10") gpsdrive_add_210 "$1";; | |
2264 | esac | |
2265 | } | |
2266 | ||
2267 | gpsdrive_mysql() { | |
2268 | if [ "$OUTFILE" != "" ]; then | |
2269 | cat >> $OUTFILE | |
2270 | elif [ $DEBUG -gt 0 ]; then | |
2271 | cat | |
2272 | else | |
2273 | mysql -u$SQLUSER -p$SQLPASS | |
2274 | fi | |
2275 | } | |
2276 | ||
2277 | gpsdrive_sqlite3() { | |
2278 | if [ "$OUTFILE" != "" ]; then | |
2279 | cat >> $OUTFILE | |
2280 | elif [ $DEBUG -gt 0 ]; then | |
2281 | cat | |
2282 | else | |
2283 | sqlite3 $HOME/.gpsdrive/waypoints.db | |
2284 | fi | |
2285 | } | |
2286 | ||
2287 | # | |
2288 | # Extended list of gpsbabel output formats | |
2289 | # | |
2290 | gpsbabel_formats() { | |
2291 | gpsbabel -? | sed -e '1,/File Types/d' -e '/Supported data filters/,$d' | |
2292 | echo " gpsdrive.sql " \ | |
2293 | "GpsDrive direct MySQL database insertion" | |
2294 | echo " map[,geo-map-opts] " \ | |
2295 | "Display map of waypoints using geo-map" | |
2296 | } | |
2297 | ||
2298 | ############################################################################## | |
2299 | # end #include "geo-common-gpsdrive" | |
2300 | ############################################################################## | |
2301 | ||
2302 | # | |
2303 | # Set default options, can be overriden on command line or in rc file | |
2304 | # | |
2305 | UPDATE_URL=$WEBHOME/geo-nearest | |
2306 | UPDATE_FILE=geo-nearest.new | |
2307 | ||
2308 | read_rc_file | |
2309 | ||
2310 | # | |
2311 | # Process the options | |
2312 | # | |
2313 | ||
2314 | gc_getopts "$@" | |
2315 | shift $? | |
2316 | ||
2317 | # | |
2318 | # Main Program | |
2319 | # | |
2320 | case "$#" in | |
2321 | 6) | |
2322 | # Cut and paste from geocaching.com cache page | |
2323 |