1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
|
"""
Fetches papers.
"""
import re
import os
import json
import random
import requests
import lxml.etree
from StringIO import StringIO
import modules.scihub
import urllib
import traceback
import pdfparanoia
logchannel = os.environ.get("LOGGING", None)
PROXY = 'http://ec2-54-218-13-46.us-west-2.compute.amazonaws.com:8500/plsget'
USER_AGENT = 'Mozilla/5.0 (X11; Linux i686 (x86_64)) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11'
ARCHIVE_DIR = '/home/bryan/public_html/papers2/paperbot/'
ARCHIVE_BASE = 'http://diyhpl.us/~bryan/papers2/paperbot/'
IEEE_EXPLORE_BASE = 'http://ieeexplore.ieee.org/xpl/articleDetails.jsp?arnumber='
HEADERS_TM_1 = {"User-Agent": "time-machine/1.0"}
HEADERS_TM_11 = {"User-Agent": "time-machine/1.1"}
HEADERS_TM_2 = {"User-Agent": "time-machine/2.0"}
HEADERS_TEAPOT = {"User-Agent": "pdf-teapot"}
HEADERS_DEFENSE = {"User-Agent": "pdf-defense-force"}
LIBGEN_FORM = "http://libgen.info/scimag/librarian/form.php"
URL_REGEX = 'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
proxy_list = [
{
'proxy_url': None,
'proxy_type': 'normal'},
# {
# 'proxy_url': PROXY,
# 'proxy_type': 'custom_flask_json'
# }
]
def nullLog(msg):
pass
def make_jstor_url(document_id):
"""Return the url to a document from its ID."""
PREFIX = 'http://www.jstor.org/stable/pdfplus/'
SUFFIX = '.pdf?acceptTC=true'
return PREFIX + document_id + SUFFIX
class paperbot_download_request(object):
_log = nullLog
def get(self, pdf_url, use_generator=False, **kwargs):
proxies_left_to_try = len(proxy_list)
extension = ".txt"
request_iteration = 0
proxy_url_index = 0
user_agent = USER_AGENT
headers = {
"User-Agent": user_agent,
}
_log = self._log
_log('paperbot_download_request pdf_url: %s' % pdf_url)
while proxies_left_to_try:
proxy_url = proxy_list[proxy_url_index]['proxy_url']
proxy_type = proxy_list[proxy_url_index]['proxy_type']
_log('proxies_left_to_try: %d proxy_url_index %d'
% (proxies_left_to_try, proxy_url_index))
_log('request_iteration: %d' % request_iteration)
# perform default behaviour if proxy is None
if proxy_list[proxy_url_index]['proxy_url'] is None:
if pdf_url.startswith("https://"):
response = requests.get(pdf_url, verify=False, **kwargs)
else:
response = requests.get(pdf_url, **kwargs)
else:
# check type of proxy
if proxy_type == 'custom_flask_json':
data = {
'pdf_url': pdf_url,
'headers': kwargs.get('headers', None),
'request_iteration': request_iteration
}
headers["Content-Type"] = "application/json"
_log('trying custom_flask_json, proxy_url %s' % proxy_url)
response = requests.get(proxy_url, data=json.dumps(data),
headers=headers)
elif proxy_type == 'normal':
# i'm not even checking if http or https is in the pdf_url,
# since the default proxy of None is already being tried in
# this loop
proxies = {
"http": proxy_url,
"https": proxy_url,
}
headers = kwargs.get('headers', None)
# I don't know if passing None or {} for headers is bad, so
# I put this if:
if headers is not None:
response = requests.get(pdf_url, headers=headers,
proxies=proxies)
else:
response = requests.get(pdf_url, proxies=proxies)
if use_generator:
yield response
else:
_log('checking \'PDF\' in response.headers')
if "pdf" in response.headers["content-type"]:
extension = ".pdf"
_log('yielding tuply with PDF in response')
# yield (response, extension)
proxies_left_to_try = 0
break
# return
if 'proxies_remaining' in response.headers:
remaining = response.headers['proxies_remaining']
_log('proxies_remaining in headers: %s' % remaining)
# decrement the index if the custom proxy doesn't have any more
# internal proxies to try
if response.headers['proxies_remaining'] == 0 or \
response.headers['proxies_remaining'] == '0':
proxies_left_to_try -= 1
request_iteration = 0
proxy_url_index += 1
else:
_log('request_iteration+=1')
request_iteration += 1
else:
# decrement the index to move on to the next proxy in our
# proxy_list
proxies_left_to_try -= 1
request_iteration = 0
proxy_url_index += 1
if use_generator:
return
_log('last yield in paperbot_download_request')
yield (response, extension)
def download(phenny, input, verbose=True):
"""
Downloads a paper.
"""
if logchannel:
_log = lambda x: phenny.msg("#%s" % logchannel, x)
else:
_log = lambda x: None
# only accept requests in a channel
if not input.sender.startswith('#'):
# unless the user is an admin, of course
if not input.admin:
phenny.say("i only take requests in the ##hplusroadmap channel.")
return
else:
# just give a warning message to the admin.. not a big deal.
phenny.say("okay i'll try, but please send me requests in ##hplusroadmap in the future.")
# get the input
line = input.group()
# was this an explicit command?
explicit = False
if line.startswith(phenny.nick):
explicit = True
line = line[len(phenny.nick):]
if line.startswith(",") or line.startswith(":"):
line = line[1:]
if line.startswith(" "):
line = line.strip()
# don't bother if there's nothing there
if len(line) < 5 or ("http://" not in line and "https://" not in line) or \
not line.startswith("http"):
return
for line in re.findall(URL_REGEX, line):
# fix an UnboundLocalError problem
shurl = None
line = filter_fix(line)
# fix for login.jsp links to ieee xplore
line = fix_ieee_login_urls(line)
line = fix_jstor_pdf_urls(line)
translation_url = "http://localhost:1969/web"
headers = {
"Content-Type": "application/json",
}
data = {
"url": line,
"sessionid": "what"
}
data = json.dumps(data)
response = requests.post(translation_url, data=data, headers=headers)
if response.status_code == 200 and response.content != "[]":
# see if there are any attachments
content = json.loads(response.content)
item = content[0]
title = item["title"]
if "DOI" in item:
_log("Translator DOI")
lgre = requests.post(LIBGEN_FORM,
data={"doi": item["DOI"]})
tree = parse_html(lgre.content)
if tree.xpath("//h1")[0].text != "No file selected":
phenny.say("http://libgen.info/scimag/get.php?doi=%s"
% urllib.quote_plus(item["DOI"]))
return
if "attachments" in item:
pdf_url = None
for attachment in item["attachments"]:
if "mimeType" in attachment and \
"application/pdf" in attachment["mimeType"]:
pdf_url = attachment["url"]
break
if pdf_url:
user_agent = USER_AGENT
paperbot_download_request_obj = paperbot_download_request()
paperbot_download_request_obj._log = _log
gen = paperbot_download_request_obj.get(pdf_url,
use_generator=False,
headers=headers)
# this is stupidly ugly
for genresponse in gen:
response, extension = genresponse
# detect failure
if response.status_code != 200:
shurl, _ = modules.scihub.scihubber(pdf_url)
if shurl:
if "libgen" in shurl:
phenny.say("http://libgen.info/scimag/get.php?doi=%s" % urllib.quote_plus(item["DOI"]))
elif "pdfcache" not in shurl:
phenny.say(shurl)
else:
pdfstr = modules.scihub.scihub_dl(shurl)
phenny.say(modules.scihub.libgen(pdfstr, item["DOI"]))
return
data = response.content
if "pdf" in response.headers["content-type"]:
try:
data = pdfparanoia.scrub(StringIO(data))
try:
_log('after pdfparanoia.scrub')
requests.get('http://localhost:8500/remoteprint',
headers={'msg': 'after pdfparanoia.scrub'})
except:
pass
break
except:
# this is to avoid a PDFNotImplementedError
pass
if "DOI" in item:
phenny.say(modules.scihub.libgen(data, item["DOI"]))
return
# grr..
title = title.encode("ascii", "ignore")
path = os.path.join(ARCHIVE_DIR, title + ".pdf")
file_handler = open(path, "w")
file_handler.write(data)
file_handler.close()
filename = requests.utils.quote(title)
# Remove an ending period, which sometimes happens when the
# title of the paper has a period at the end.
if filename[-1] == ".":
filename = filename[:-1]
url = "http://diyhpl.us/~bryan/papers2/paperbot/" + filename + ".pdf"
phenny.say(url)
continue
elif verbose and explicit:
_log("Translation server PDF fail")
shurl, doi = modules.scihub.scihubber(line)
continue
elif verbose and explicit:
_log("Translation server PDF fail")
shurl, doi = modules.scihub.scihubber(line)
phenny.say(download_url(line, _log))
continue
elif verbose and explicit:
_log("Translation server fail")
shurl, doi = modules.scihub.scihubber(line)
_log("Scihubber -> (%s, %s)" % (shurl, doi))
if shurl:
if "pdfcache" in shurl:
if doi:
pdfstr = modules.scihub.scihub_dl(shurl)
phenny.say(modules.scihub.libgen(pdfstr, doi))
else:
phenny.say(download_url(shurl, _log,
cookies=modules.scihub.shcookie))
else:
phenny.say(shurl)
elif verbose and explicit:
_log("All approaches failed")
phenny.say(download_url(line, _log))
return
download.commands = ["fetch", "get", "download"]
download.priority = "high"
download.rule = r'(.*)'
def download_ieee(url):
"""
Downloads an IEEE paper. The Zotero translator requires frames/windows to
be available. Eventually translation-server will be fixed, but until then
it might be nice to have an IEEE workaround.
"""
# url = "http://ieeexplore.ieee.org:80/xpl/freeabs_all.jsp?reload=true&arnumber=901261"
# url = "http://ieeexplore.ieee.org/iel5/27/19498/00901261.pdf?arnumber=901261"
raise NotImplementedError
def download_url(url, _log=nullLog, **kwargs):
paperbot_download_request_obj = paperbot_download_request()
paperbot_download_request_obj._log = _log
response_generator = paperbot_download_request_obj.get(url,
use_generator=True,
headers={"User-Agent": "origami-pdf"})
cc = 0
for response in response_generator:
_log('using generator for %s time' % cc)
cc += 1
paperbot_download_request_obj2 = paperbot_download_request()
paperbot_download_request_obj2._log = _log
content = response.content
# response = requests.get(url, headers={"User-Agent": "origami-pdf"}, **kwargs)
# content = response.content
# just make up a default filename
title = "%0.2x" % random.getrandbits(128)
# default extension
extension = ".txt"
if "pdf" in response.headers["content-type"]:
extension = ".pdf"
elif check_if_html(response):
# parse the html string with lxml.etree
tree = parse_html(content)
# extract some metadata with xpaths
citation_pdf_url = find_citation_pdf_url(tree, url)
citation_title = find_citation_title(tree)
# aip.org sucks, citation_pdf_url is wrong
if citation_pdf_url and "link.aip.org/" in citation_pdf_url:
citation_pdf_url = None
if citation_pdf_url and "ieeexplore.ieee.org" in citation_pdf_url:
content = requests.get(citation_pdf_url).content
tree = parse_html(content)
# citation_title = ...
# wow, this seriously needs to be cleaned up
if citation_pdf_url and citation_title and \
"ieeexplore.ieee.org" not in citation_pdf_url:
citation_title = citation_title.encode("ascii", "ignore")
response = requests.get(citation_pdf_url,
headers=HEADERS_DEFENSE)
content = response.content
if "pdf" in response.headers["content-type"]:
extension = ".pdf"
title = citation_title
else:
if "sciencedirect.com" in url and "ShoppingCart" not in url:
_log('download_url got a sciencedirect URL')
try:
try:
title_xpath = "//h1[@class='svTitle']"
title = tree.xpath(title_xpath)[0].text
pdf_url = tree.xpath("//a[@id='pdfLink']/@href")[0]
except IndexError:
title = tree.xpath("//title")[0].text
pdf_url = tree.xpath("//a[@id='pdfLink']/@href")[0]
if 'http' not in pdf_url:
main_url_split = response.url.split('//')
http_prefix = main_url_split[0]
if 'http' in http_prefix:
domain_url = main_url_split[1].split('/')[0]
slash = '/' if pdf_url[0] != '/' else ''
pdf_url = http_prefix + '//' + domain_url + slash + pdf_url
gen = paperbot_download_request_obj2.get(pdf_url,
use_generator=False,
headers={"User-Agent": "sdf-macross"})
# this is stupidly ugly
for genresponse in gen:
new_response, extension = genresponse
new_content = new_response.content
_log('paperbot_download_request_obj2 content-type: %s'
% new_response.headers["content-type"])
if "pdf" in new_response.headers["content-type"]:
extension = ".pdf"
break
except Exception as e:
_log(traceback.format_exc())
pass
else:
content = new_content
response = new_response
elif "jstor.org/" in url:
# clean up the url
if "?" in url:
url = url[0:url.find("?")]
# not all pages have the <input type="hidden" name="ppv-title"> element
try:
title = tree.xpath("//div[@class='hd title']")[0].text
except Exception:
try:
input_xpath = "//input[@name='ppv-title']/@value"
title = tree.xpath(input_xpath)[0]
except Exception:
pass
# get the document id
document_id = None
if url[-1] != "/":
# if "stable/" in url:
# elif "discover/" in url:
# elif "action/showShelf?candidate=" in url:
# elif "pss/" in url:
document_id = url.split("/")[-1]
if document_id.isdigit():
try:
pdf_url = make_jstor_url(document_id)
new_response = requests.get(pdf_url,
headers=HEADERS_TM_11)
new_content = new_response.content
if "pdf" in new_response.headers["content-type"]:
extension = ".pdf"
except Exception:
pass
else:
content = new_content
response = new_response
elif ".aip.org/" in url:
try:
title = tree.xpath("//title/text()")[0].split(" | ")[0]
pdf_url = [link for link in tree.xpath("//a/@href")
if "getpdf" in link][0]
new_response = requests.get(pdf_url,
headers=HEADERS_TM_1)
new_content = new_response.content
if "pdf" in new_response.headers["content-type"]:
extension = ".pdf"
except Exception:
pass
else:
content = new_content
response = new_response
elif "ieeexplore.ieee.org" in url:
try:
pdf_url = [url for url in tree.xpath("//frame/@src")
if "pdf" in url][0]
new_response = requests.get(pdf_url,
headers=HEADERS_TM_2)
new_content = new_response.content
if "pdf" in new_response.headers["content-type"]:
extension = ".pdf"
except Exception:
pass
else:
content = new_content
response = new_response
elif "h1 class=\"articleTitle" in content:
try:
title_xpath = "//h1[@class='articleTitle']"
title = tree.xpath(title_xpath)[0].text
title = title.encode("ascii", "ignore")
url_xpath = "//a[@title='View the Full Text PDF']/@href"
pdf_url = tree.xpath(url_xpath)[0]
except:
pass
else:
if pdf_url.startswith("/"):
url_start = url[:url.find("/", 8)]
pdf_url = url_start + pdf_url
response = requests.get(pdf_url,
headers=HEADERS_TEAPOT)
content = response.content
if "pdf" in response.headers["content-type"]:
extension = ".pdf"
# raise Exception("problem with citation_pdf_url or citation_title")
# well, at least save the contents from the original url
pass
# make the title again just in case
if not title:
title = "%0.2x" % random.getrandbits(128)
# can't create directories
title = title.replace("/", "_")
path = os.path.join(ARCHIVE_DIR, title + extension)
if extension in [".pdf", "pdf"]:
try:
content = pdfparanoia.scrub(StringIO(content))
except:
# this is to avoid a PDFNotImplementedError
pass
file_handler = open(path, "w")
file_handler.write(content)
file_handler.close()
title = title.encode("ascii", "ignore")
url = ARCHIVE_BASE + requests.utils.quote(title) + extension
return url
def parse_html(content):
if not isinstance(content, StringIO):
content = StringIO(content)
parser = lxml.etree.HTMLParser()
tree = lxml.etree.parse(content, parser)
return tree
def check_if_html(response):
return "text/html" in response.headers["content-type"]
def find_citation_pdf_url(tree, url):
"""
Returns the <meta name="citation_pdf_url"> content attribute.
"""
citation_pdf_url = extract_meta_content(tree, "citation_pdf_url")
if citation_pdf_url and not citation_pdf_url.startswith("http"):
if citation_pdf_url.startswith("/"):
url_start = url[:url.find("/", 8)]
citation_pdf_url = url_start + citation_pdf_url
else:
raise Exception("unhandled situation (citation_pdf_url)")
return citation_pdf_url
def find_citation_title(tree):
"""
Returns the <meta name="citation_title"> content attribute.
"""
citation_title = extract_meta_content(tree, "citation_title")
return citation_title
def extract_meta_content(tree, meta_name):
try:
content = tree.xpath("//meta[@name='" + meta_name + "']/@content")[0]
except:
return None
else:
return content
def filter_fix(url):
"""
Fixes some common problems in urls.
"""
if ".proxy.lib.pdx.edu" in url:
url = url.replace(".proxy.lib.pdx.edu", "")
return url
def fix_ieee_login_urls(url):
"""
Fixes urls point to login.jsp on IEEE Xplore. When someone browses to the
abstracts page on IEEE Xplore, they are sometimes sent to the login.jsp
page, and then this link is given to paperbot. The actual link is based on
the arnumber.
example:
http://ieeexplore.ieee.org/xpl/login.jsp?tp=&arnumber=806324&url=http%3A%2F%2Fieeexplore.ieee.org%2Fxpls%2Fabs_all.jsp%3Farnumber%3D806324
"""
if "ieeexplore.ieee.org/xpl/login.jsp" in url:
if "arnumber=" in url:
parts = url.split("arnumber=")
# i guess the url might not look like the example in the docstring
if "&" in parts[1]:
arnumber = parts[1].split("&")[0]
else:
arnumber = parts[1]
return IEEE_EXPLORE_BASE + arnumber
# default case when things go wrong
return url
def fix_jstor_pdf_urls(url):
"""
Fixes urls pointing to jstor pdfs.
"""
if "jstor.org/" in url:
if ".pdf" in url and "?acceptTC=true" not in url:
url += "?acceptTC=true"
return url
|