Commit 83739963420642ad8021b66ab2f68cedaa6a99e6
Get message downloads working
Charles Lehner committed on 9/12/2014, 6:16:32 PMParent: 1f4de2eeab8c60b37f065c88bd87aa3ba6c51e0d
Files changed
| ggscrape | changed |
| ggscrape | ||
|---|---|---|
| @@ -7,9 +7,9 @@ | ||
| 7 | 7 … | # Copyright (c) 2014 Charles Lehner |
| 8 | 8 … | # Released under the terms of the MIT License. |
| 9 | 9 … | |
| 10 | 10 … | VERSION=0.0.1 |
| 11 | -BASE_URL='https://groups.google.com/forum/?_escaped_fragment_=' | |
| 11 … | +BASE_URL='https://groups.google.com/forum/' | |
| 12 | 12 … | |
| 13 | 13 … | EX_USAGE=64 |
| 14 | 14 … | topic_range=100 |
| 15 | 15 … | |
| @@ -28,24 +28,28 @@ | ||
| 28 | 28 … | fi |
| 29 | 29 … | } |
| 30 | 30 … | |
| 31 | 31 … | req() { |
| 32 | - debug_print query: "$1" | |
| 32 … | + debug_print req: "$1" | |
| 33 | 33 … | curl -sN -b "$cookie_str" "$BASE_URL$@" |
| 34 | 34 … | } |
| 35 | 35 … | |
| 36 … | +req_fragment() { | |
| 37 … | + req "?_escaped_fragment_=$@" | |
| 38 … | +} | |
| 39 … | + | |
| 36 | 40 … | test_permission() { |
| 37 | 41 … | debug_print testing permission |
| 38 | - req "forum/${group_id}%5B1-1-false%5D" -I | grep -q '200 OK' | |
| 42 … | + req_fragment "forum/${group_id}%5B1-1-false%5D" -I | grep -q '200 OK' | |
| 39 | 43 … | } |
| 40 | 44 … | |
| 41 | 45 … | get_topics_single() { |
| 42 | 46 … | local start=$1 |
| 43 | 47 … | local end=$2 |
| 44 | 48 … | |
| 45 | 49 … | debug_print "get topics $group_id [$start-$end]" |
| 46 | 50 … | |
| 47 | - req "forum/${group_id}%5B${start}-${end}-false%5D" | sed -n \ | |
| 51 … | + req_fragment "forum/${group_id}%5B${start}-${end}-false%5D" | sed -n \ | |
| 48 | 52 … | "s/^<i>Showing [^<]* of 0 topics<\/i>$//p; |
| 49 | 53 … | s/<tr>/\0\n/; /lastPostDate/ { |
| 50 | 54 … | s/.*lastPostDate\">\([^<]*\).*$/\1/m; P; D; |
| 51 | 55 … | }; |
| @@ -64,9 +68,9 @@ | ||
| 64 | 68 … | local start=$(($1)) |
| 65 | 69 … | local end=$(($2)) |
| 66 | 70 … | local temp_end |
| 67 | 71 … | |
| 68 | - if [[ -z "$start" ]]; then | |
| 72 … | + if ((start==0)); then | |
| 69 | 73 … | start=1 |
| 70 | 74 … | fi |
| 71 | 75 … | |
| 72 | 76 … | debug_print "get all topics $group_id [$start-$end]" |
| @@ -79,9 +83,9 @@ | ||
| 79 | 83 … | |
| 80 | 84 … | # get message ids in a topic |
| 81 | 85 … | get_messages() { |
| 82 | 86 … | local topic_id="$1" |
| 83 | - req "topic/${group_id}/${topic_id}" | sed -n\ | |
| 87 … | + req_fragment "topic/${group_id}/${topic_id}" | sed -n\ | |
| 84 | 88 … | 's/.*<td class="subject"><a href="[^"]*'$topic_id'\/\([^"]*\)".*/\1/p' |
| 85 | 89 … | } |
| 86 | 90 … | |
| 87 | 91 … | download_message() { |
| @@ -89,11 +93,19 @@ | ||
| 89 | 93 … | local msg_id="$2" |
| 90 | 94 … | debug_print download topic $topic_id message $msg_id |
| 91 | 95 … | path="${dest_dir}/${group_id}${topic_id}${msg_id}.eml" |
| 92 | 96 … | if [[ -s "$path" ]]; then |
| 93 | - echo "file for message ${msg_id} already exists. skipping." | |
| 97 … | + echo "message ${topic_id}${msg_id} already downloaded. skipping." | |
| 94 | 98 … | else |
| 95 | - req "${group_id}/${topic_id}/${msg_id}" > "$path" | |
| 99 … | + echo "message ${topic_id}${msg_id} downloading." | |
| 100 … | + temp=$(mktemp) | |
| 101 … | + if req "message/raw?msg=${group_id}/${topic_id}/${msg_id}" -o "$temp" | |
| 102 … | + then | |
| 103 … | + mv "$temp" "$path" | |
| 104 … | + else | |
| 105 … | + echo "message ${topic_id}${msg_id} failed to download." >&2 | |
| 106 … | + rm "$temp" | |
| 107 … | + fi | |
| 96 | 108 … | fi |
| 97 | 109 … | } |
| 98 | 110 … | |
| 99 | 111 … | download_messages() { |
Built with git-ssb-web