git ssb

1
#!/bin/bash
2
# vi: ts=4 sw=4 et
3
#
4
# ggscrape(1)
5
# Download emails from a Google Groups
6
#
7
# Copyright (c) 2014 Charles Lehner
8
# Released under the terms of the MIT License.
9
10
VERSION=1.0.0
11
BASE_URL='https://groups.google.com/forum/'
12
13
EX_USAGE=64
14
topic_range=100
15
16
opt_verbose=
17
group_id=
18
dest_dir=
19
ln_dir=
20
cookie_str=
21
22
if [[ -n "$GG_COOKIE" ]]; then
23
    cookie_str="$GG_COOKIE"
24
fi
25
26
debug_print() {
27
    if [[ -n $opt_verbose ]]; then
28
        echo $@ >&2
29
    fi
30
}
31
32
req() {
33
    debug_print req: "$1"
34
    curl -sN -b "$cookie_str" "$BASE_URL$@"
35
}
36
37
req_fragment() {
38
    req "?_escaped_fragment_=$@"
39
}
40
41
check_permission() {
42
    debug_print testing permission
43
    if ! req_fragment "forum/${group_id}%5B1-1-false%5D" -I | grep -q '200 OK'
44
    then
45
        echo 'Unable to access group.' >&2
46
        return 1
47
    fi
48
}
49
50
get_topics_single() {
51
    local start=$1
52
    local end=$2
53
54
    debug_print "get topics $group_id [$start-$end]"
55
56
    req_fragment "forum/${group_id}%5B${start}-${end}-false%5D" | sed -n \
57
        "s/^<i>Showing [^<]* of 0 topics<\/i>$//p;
58
        s/<tr>/\0\n/; /lastPostDate/ {
59
            s/.*lastPostDate\">\([^<]*\).*$/\1/m; P; D;
60
        };
61
        /subject/{
62
            s/.*href=\"[^\"]*\/\([^\"]*\)\"[^>]*>\([^<]*\).*/\1\n\2/; p;
63
        }" | sed '/^$/q99; N; N; s/\(.*\)\n\(.*\)\n\(.*\)/\1\t\3\t\2/'
64
    # return status 99 if no matches found
65
}
66
67
unknown_option() {
68
    echo "Unknown option $1" >&2
69
    exit $EX_USAGE
70
}
71
72
get_topics() {
73
    local start=$(($1))
74
    local end=$(($2))
75
    local temp_end
76
77
    if ((start==0)); then
78
        start=1
79
    fi
80
81
    check_permission || return 1
82
83
    debug_print "get all topics $group_id [$start-$end]"
84
85
    for ((i = start; i <= end || end == 0; i += topic_range)); do
86
        ((temp_end = (end && i+topic_range-1 > end) ? end : i+topic_range-1))
87
        get_topics_single $i $temp_end || break
88
    done
89
}
90
91
# get message ids in a topic
92
get_messages() {
93
    local topic_id="$1"
94
    req_fragment "topic/${group_id}/${topic_id}" | sed -n\
95
        's/.*<td class="subject"><a href="[^"]*'$topic_id'\/\([^"]*\)".*/\1/p'
96
}
97
98
fix_message() {
99
    local file="$1"
100
    # Some messages have two duplicate sets of headers, the second set starting
101
    # with some X-Google stuff. Remove the second set of headers if they are
102
    # present.
103
    if sed -n '/^\r$/{ n; /^X-Google-Groups:/q; q 1; }' "$file"; then
104
        debug_print 'Removing duplicate headers'
105
        sed -i~ '/^X-Google-Groups:/,/^\r$/d' "$file" && rm "$file~"
106
    fi
107
}
108
109
download_message() {
110
    local topic_id="$1"
111
    local msg_id="$2"
112
    debug_print download topic $topic_id message $msg_id
113
    path="${dest_dir}/${group_id}${topic_id}${msg_id}.eml"
114
    if [[ -s "$path" ]]; then
115
        echo "message ${topic_id}${msg_id} already downloaded. skipping."
116
    else
117
        echo "message ${topic_id}${msg_id} downloading."
118
        temp=$(mktemp)
119
        if req "message/raw?msg=${group_id}/${topic_id}/${msg_id}" -o "$temp"
120
        then
121
            fix_message "$temp"
122
            mv "$temp" "$path"
123
            if [[ -n "$ln_dir" ]]; then
124
                ln "$path" "$ln_dir"
125
            fi
126
        else
127
            echo "message ${topic_id}${msg_id} failed to download." >&2
128
            rm "$temp"
129
        fi
130
    fi
131
}
132
133
download_messages() {
134
    local topic_id="$1"
135
    debug_print download topic $topic_id
136
    get_messages "$topic_id" | while read msg_id _; do
137
        download_message "$topic_id" "$msg_id"
138
    done
139
}
140
141
download_mails() {
142
    local start=$1
143
    local end=$2
144
145
    get_topics "$start" "$end" | while read topic_id topic_title; do
146
        download_messages $topic_id
147
    done
148
}
149
150
show_version() {
151
    echo ggscrape $VERSION
152
}
153
154
show_help() {
155
    cat << EOF
156
ggscrape. Download emails from a Google Group
157
158
Usage:
159
  ggscrape <group_id> test
160
  ggscrape <group_id> topics
161
  ggscrape <group_id> messages <topic_id>
162
  ggscrape <group_id> download <dest_dir>
163
  ggscrape fix_message <file>
164
165
Options:
166
  -h, --help            Show this screen
167
  --version             Show version
168
  -v, --verbose         Show debug info
169
  -c, --cookie <cookie> Use the given cookie string
170
  -b, --begin <topicnum> Topic number at which to begin downloading
171
  -e, --end <topicnum>  Topic number at which to stop downloading
172
  -l, --ln <ln_dir>     Hard link email files into this directory
173
174
Environmental variables:
175
176
  GG_COOKIE             use as value for --cookie
177
178
EOF
179
}
180
181
cmd=
182
topic_id=
183
topic_begin=
184
topic_end=
185
fix_file=
186
187
while [[ "$#" -gt 0 ]]; do
188
    case "$1" in
189
        -h|--help) show_help; exit;;
190
        --version) show_version; exit;;
191
        -v|--verbose) opt_verbose=1;;
192
        -c|--cookie) cookie_str="$2"; shift;;
193
        -b|--begin) topic_begin="$2"; shift;;
194
        -e|--end) topic_end="$2"; shift;;
195
        -l|--ln) ln_dir="$2"; shift;;
196
        topics) cmd=topics;;
197
        test) cmd=test;;
198
        messages) cmd=messages; topic_id="$2"; shift;;
199
        download) cmd=download; dest_dir="$2"; shift;;
200
        fix_message) cmd=fix_message; fix_file="$2"; shift;;
201
        *) if [[ -z "$group_id" ]]; then
202
            group_id="$1"
203
        else
204
            unknown_option "$1"
205
        fi;;
206
    esac
207
    shift
208
done
209
210
if [[ $cmd==fix_message ]]; then
211
    if [[ -z "$fix_file" ]]; then
212
        show_help
213
        exit 1
214
    fi
215
    fix_message "$fix_file"
216
    exit
217
fi
218
219
if [[ -z "$group_id" ]]; then
220
    show_help
221
    exit 1
222
fi
223
224
case "$cmd" in
225
    '') show_help; exit;;
226
    test) check_permission && echo Success;;
227
    topics) get_topics "$topic_begin" "$topic_end";;
228
    download) download_mails "$topic_begin" "$topic_end";;
229
    messages) get_messages "$topic_id";;
230
    *) echo "Unknown command $cmd" >&2;;
231
esac
232