diff --git a/.gitignore b/.gitignore index e22ac668..b56e5055 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.py[cod] *.log -.* +.docs __pycache__/ +config.py diff --git a/README.md b/README.md index a1e635fb..46edf88c 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,32 @@ # Citer -A citation generator tool for Wikipedia. Currently accessible from: -http://tools.wmflabs.org/citer/ (the English version) -http://tools.wmflabs.org/yadfa/ (the Persian version) +A citation generator tool for Wikipedia. Currently accessible from:\ +[https://citer.toolforge.org/](https://citer.toolforge.org/) (the English version)\ +[https://yadfa.toolforge.org/](https://yadfa.toolforge.org/) (the Persian version) ## What does it do? -Citer is specially useful for generating citations from Google Books URLs, DOIs (Any Digital object Identifiers) and ISBNs (International Standard Book Numbers). -Additionally URL of many major news websites are supported, including: -The New York Times, BBC, Daily Mail, Daily Mirror, The Daily Telegraph, The Huffington Post, The Washington Post, The Boston Globe, Bloomberg Businessweek, Financial Times, and The Times of India. Sepecial support for the URLs of the [Wayback Machine](https://en.wikipedia.org/wiki/Wayback_Machine) is also implemented. - -Some other tested and supported Persian web-sites: -* http://www.noormags.com (نورمگز) +Citer is especially useful for generating citations from Google Books URLs, DOIs (Any Digital object Identifiers) and ISBNs (International Standard Book Numbers). +Additionally, URLs of many major news websites are supported, including: + +* The New York Times +* BBC +* Daily Mail +* Daily Mirror +* The Daily Telegraph +* The Huffington Post +* The Washington Post +* The Boston Globe +* Bloomberg Businessweek +* Financial Times +* The Times of India + +Special support for the URLs of the [Wayback Machine](https://en.wikipedia.org/wiki/Wayback_Machine) is also implemented. + +Some other tested and supported Persian websites: +* http://www.noormags.ir (نورمگز) * http://www.noorlib.ir (کتابخانه دیجیتال نور) -* http://www.adinebook.com (آدینه‌بوک) +* http://www.ketab.ir (خانه كتاب) * http://socialhistory.ihcs.ac.ir/ (تحقیقات تاریخ اجتماعی) @@ -21,15 +34,20 @@ Some other tested and supported Persian web-sites: To run Citer on your local computer: -1. Install Python 3.6+. -2. Clone the project. -3. Install the dependencies using `pip install -r requirements.txt`. -3. Make sure that `flup` is __not__ installed in your environment. -4. Run Citer by calling `main.py`. +1. Install Python 3.9+ +2. Clone the project +3. Install the dependencies using `pip install --user -r requirements.txt` +4. Copy `config.py.example` to `config.py` (You might want to get an NCBI API key and add it to the config file if you're going to use its services) +5. Run `python3 app.py` -If everything goes fine, the main page will be accessible from: - http://127.0.0.1:5000/ +If there are no warnings or error messages (and no HTML is displayed), the main page will be accessible from:\ + [http://localhost:5000/](http://localhost:5000/) +If you experience any problems or have questions, please open an issue on this repo. ## Language Setting -The default language is English and can be change to Persian using the setting in config.py file. +The default language is English and can be changed to Persian using the setting in the config.py file. + + +## Known issues +* The bookmarklet does not work on archive.org (issue #26) or any other website that does not allow opening external links. One needs to use Citer directly in such cases. diff --git a/app.py b/app.py index 93ec3d28..912d030f 100644 --- a/app.py +++ b/app.py @@ -1,58 +1,76 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- - from collections import defaultdict from html import unescape from logging import getLogger, Formatter, WARNING, INFO from logging.handlers import RotatingFileHandler +from os.path import dirname, abspath from urllib.parse import parse_qs, urlparse, unquote from wsgiref.headers import Headers -from requests import ConnectionError as RequestsConnectionError +from requests import ConnectionError as RequestsConnectionError, \ + JSONDecodeError from config import LANG -from lib.adinebook import adinehbook_sfn_cit_ref -from lib.commons import uninum2en, sfn_cit_ref_to_json -from lib.doi import doi_sfn_cit_ref, DOI_SEARCH -from lib.googlebooks import googlebooks_sfn_cit_ref -from lib.isbn_oclc import ( - ISBN_10OR13_SEARCH, IsbnError, isbn_sfn_cit_ref, oclc_sfn_cit_ref) -from lib.noorlib import noorlib_sfn_cit_ref -from lib.noormags import noormags_sfn_cit_ref -from lib.pubmed import pmcid_sfn_cit_ref, pmid_sfn_cit_ref -from lib.urls import urls_sfn_cit_ref -from lib.waybackmachine import waybackmachine_sfn_cit_ref +from lib.ketabir import url_to_dict as ketabir_url_to_dict +from lib.commons import uninum2en, scr_to_json, ISBN_10OR13_SEARCH, \ + dict_to_sfn_cit_ref, ReturnError +from lib.doi import doi_to_dict, DOI_SEARCH +from lib.googlebooks import url_to_dict as google_books_dict +from lib.isbn_oclc import IsbnError, isbn_to_dict, oclc_dict +from lib.jstor import url_to_dict as jstor_url_to_dict +from lib.noorlib import url_to_dict as noorlib_url_to_dict +from lib.noormags import url_to_dict as noormags_url_to_dict +from lib.pubmed import pmcid_dict, pmid_dict +from lib.urls import url_to_dict as urls_url_to_dict +from lib.waybackmachine import url_to_dict as archive_url_to_dict if LANG == 'en': from lib.html.en import ( - DEFAULT_SFN_CIT_REF, - UNDEFINED_INPUT_SFN_CIT_REF, - HTTPERROR_SFN_CIT_REF, - OTHER_EXCEPTION_SFN_CIT_REF, - sfn_cit_ref_to_html, + DEFAULT_SCR, + UNDEFINED_INPUT_SCR, + HTTPERROR_SCR, + OTHER_EXCEPTION_SCR, + scr_to_html, CSS, CSS_HEADERS, JS, JS_HEADERS) else: from lib.html.fa import ( - DEFAULT_SFN_CIT_REF, - UNDEFINED_INPUT_SFN_CIT_REF, - HTTPERROR_SFN_CIT_REF, - OTHER_EXCEPTION_SFN_CIT_REF, - sfn_cit_ref_to_html, + DEFAULT_SCR, + UNDEFINED_INPUT_SCR, + HTTPERROR_SCR, + OTHER_EXCEPTION_SCR, + scr_to_html, CSS, CSS_HEADERS) +def google_encrypted_dict(url, parsed_url, date_format) -> dict: + if parsed_url[2][:7] in {'/books', '/books/'}: + # sample urls: + # https://encrypted.google.com/books?id=6upvonUt0O8C + # https://www.google.com/books?id=bwfoCAAAQBAJ&pg=PA32 + # https://www.google.com/books/edition/_/bwfoCAAAQBAJ?gbpv=1&pg=PA32 + return google_books_dict(parsed_url, date_format) + return urls_url_to_dict(url, date_format) + + TLDLESS_NETLOC_RESOLVER = { - 'adinebook': adinehbook_sfn_cit_ref, - 'adinehbook': adinehbook_sfn_cit_ref, - 'noorlib': noorlib_sfn_cit_ref, - 'noormags': noormags_sfn_cit_ref, - 'web.archive': waybackmachine_sfn_cit_ref, - 'web-beta.archive': waybackmachine_sfn_cit_ref, - 'books.google.co': googlebooks_sfn_cit_ref, - 'books.google': googlebooks_sfn_cit_ref, + 'ketab': ketabir_url_to_dict, + + 'noorlib': noorlib_url_to_dict, + 'noormags': noormags_url_to_dict, + + 'web.archive': archive_url_to_dict, + 'web-beta.archive': archive_url_to_dict, + + 'books.google.co': google_books_dict, + 'books.google.com': google_books_dict, + 'books.google': google_books_dict, + + 'google': google_encrypted_dict, + 'encrypted.google': google_encrypted_dict, + + 'jstor': jstor_url_to_dict, }.get RESPONSE_HEADERS = Headers([('Content-Type', 'text/html; charset=UTF-8')]) @@ -65,13 +83,14 @@ def get_root_logger(): custom_logger = getLogger() custom_logger.setLevel(INFO) + srcdir = dirname(abspath(__file__)) handler = RotatingFileHandler( - filename='citer.log', + filename=f'{srcdir}/citer.log', mode='a', maxBytes=20000, backupCount=0, encoding='utf-8', - delay=0) + ) handler.setLevel(INFO) handler.setFormatter( Formatter('\n%(asctime)s\n%(levelname)s\n%(message)s\n')) @@ -82,7 +101,7 @@ def get_root_logger(): LOGGER = get_root_logger() -def url_doi_isbn_to_sfn_cit_ref(user_input, date_format) -> tuple: +def input_to_dict(user_input, date_format, /) -> dict: en_user_input = unquote(uninum2en(user_input)) # Checking the user input for dot is important because # the use of dotless domains is prohibited. @@ -90,110 +109,121 @@ def url_doi_isbn_to_sfn_cit_ref(user_input, date_format) -> tuple: if '.' in en_user_input: # Try predefined URLs # Todo: The following code could be done in threads. - if not user_input.startswith('http'): + if not (url_input := user_input.startswith('http')): url = 'http://' + user_input else: url = user_input + parsed_url = urlparse(url) # TLD stands for top-level domain - tldless_netloc = urlparse(url)[1].rpartition('.')[0] - resolver = TLDLESS_NETLOC_RESOLVER( + tldless_netloc = parsed_url[1].rpartition('.')[0] + # todo: make lazy? + if (to_dict := TLDLESS_NETLOC_RESOLVER( tldless_netloc[4:] if tldless_netloc.startswith('www.') - else tldless_netloc) - if resolver: - return resolver(url, date_format) + else tldless_netloc + )) is not None: + if to_dict is google_books_dict: + return to_dict(parsed_url, date_format) + elif to_dict is google_encrypted_dict: + return to_dict(url, parsed_url, date_format) + return to_dict(url, date_format) + # DOIs contain dots - m = DOI_SEARCH(unescape(en_user_input)) - if m: - return doi_sfn_cit_ref(m.group(1), True, date_format) - return urls_sfn_cit_ref(url, date_format) + if (m := DOI_SEARCH(unescape(en_user_input))) is not None: + try: + return doi_to_dict(m[0], True, date_format) + except JSONDecodeError: + if url_input is False: + raise + # continue with urls_scr + + return urls_url_to_dict(url, date_format) else: # We can check user inputs containing dots for ISBNs, but probably is - # error prone. - m = ISBN_10OR13_SEARCH(en_user_input) - if m: + # error-prone. + if (m := ISBN_10OR13_SEARCH(en_user_input)) is not None: try: - return isbn_sfn_cit_ref(m.group(), True, date_format) + return isbn_to_dict(m[0], True, date_format) except IsbnError: pass - return UNDEFINED_INPUT_SFN_CIT_REF - + return UNDEFINED_INPUT_SCR -def app(environ, start_response): - query_dict_get = parse_qs(environ['QUERY_STRING']).get +def app(environ: dict, start_response: callable) -> tuple: path_info = environ['PATH_INFO'] if '/static/' in path_info: - if path_info.endswith('.css'): + if path_info[-4:] == '.css': start_response('200 OK', CSS_HEADERS) - return [CSS] + return CSS, else: # path_info.endswith('.js') and config.lang == 'en' start_response('200 OK', JS_HEADERS) - return [JS] + return JS, + query_dict_get = parse_qs(environ['QUERY_STRING']).get date_format = query_dict_get('dateformat', [''])[0].strip() - input_type = query_dict_get('input_type', [''])[0] # Warning: input is not escaped! - user_input = query_dict_get('user_input', [''])[0].strip() - if not user_input: - response_body = sfn_cit_ref_to_html( - DEFAULT_SFN_CIT_REF, date_format, input_type + if not (user_input := query_dict_get('user_input', [''])[0].strip()): + response_body = scr_to_html( + DEFAULT_SCR, date_format, input_type ).encode() RESPONSE_HEADERS['Content-Length'] = str(len(response_body)) start_response('200 OK', RESPONSE_HEADERS.items()) - return [response_body] + return response_body, output_format = query_dict_get('output_format', [''])[0] # apiquery - resolver = input_type_to_resolver[input_type] + to_dict = input_type_to_resolver[input_type] # noinspection PyBroadException try: - response = resolver(user_input, date_format) + d = to_dict(user_input, date_format) except RequestsConnectionError: status = '500 ConnectionError' LOGGER.exception(user_input) if output_format == 'json': - response_body = sfn_cit_ref_to_json(HTTPERROR_SFN_CIT_REF) + response_body = scr_to_json(HTTPERROR_SCR) else: - response_body = sfn_cit_ref_to_html( - HTTPERROR_SFN_CIT_REF, date_format, input_type) - except Exception: + response_body = scr_to_html( + HTTPERROR_SCR, date_format, input_type) + except Exception as e: status = '500 Internal Server Error' - LOGGER.exception(user_input) + + if isinstance(e, ReturnError): + scr = e.args + else: + LOGGER.exception(user_input) + scr = OTHER_EXCEPTION_SCR + if output_format == 'json': - response_body = sfn_cit_ref_to_json(OTHER_EXCEPTION_SFN_CIT_REF) + response_body = scr_to_json(scr) else: - response_body = sfn_cit_ref_to_html( - OTHER_EXCEPTION_SFN_CIT_REF, date_format, input_type) + response_body = scr_to_html(scr, date_format, input_type) else: status = '200 OK' + scr = dict_to_sfn_cit_ref(d) if output_format == 'json': - response_body = sfn_cit_ref_to_json(response) + response_body = scr_to_json(scr) else: - response_body = sfn_cit_ref_to_html( - response, date_format, input_type) + response_body = scr_to_html(scr, date_format, input_type) response_body = response_body.encode() RESPONSE_HEADERS['Content-Length'] = str(len(response_body)) start_response(status, RESPONSE_HEADERS.items()) - return [response_body] + return response_body, input_type_to_resolver = defaultdict( - lambda: url_doi_isbn_to_sfn_cit_ref, { - 'url-doi-isbn': url_doi_isbn_to_sfn_cit_ref, - 'pmid': pmid_sfn_cit_ref, - 'pmcid': pmcid_sfn_cit_ref, - 'oclc': oclc_sfn_cit_ref}) + lambda: input_to_dict, { + 'url-doi-isbn': input_to_dict, # todo: can be removed? + 'pmid': pmid_dict, + 'pmcid': pmcid_dict, + 'oclc': oclc_dict}) if __name__ == '__main__': # note that app.py is not run as '__main__' in kubernetes - try: - from flup.server.fcgi import WSGIServer - WSGIServer(app).run() - except ImportError: # on local computer - from wsgiref.simple_server import make_server - httpd = make_server('localhost', 5000, app) - httpd.serve_forever() + # only for local computer + from wsgiref.simple_server import make_server + httpd = make_server('localhost', 5000, app) + print('serving on http://localhost:5000') + httpd.serve_forever() diff --git a/citer-cli b/citer-cli new file mode 100755 index 00000000..19cf29a7 --- /dev/null +++ b/citer-cli @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 + +import argparse +import html + +import app + +def main(): + ap = argparse.ArgumentParser() + types = list(app.input_type_to_resolver.keys()) + ap.add_argument('-t', '--type', choices=types, metavar='TYPE', default='url-doi-isbn', help=str.join(' or ', map(repr, types))) + ap.add_argument('-d', '--date-format', metavar='DATE-FMT', default='%Y-%m-%d') + ap.add_argument('ident', metavar='URL-OR-ID') + options = ap.parse_args() + resolver = app.input_type_to_resolver[options.type] + d = resolver(options.ident, options.date_format) + scr = app.dict_to_sfn_cit_ref(d) + for item in scr: + print(html.unescape(item), end='\n\n') + +if __name__ == '__main__': + main() diff --git a/config.py b/config.py.example similarity index 90% rename from config.py rename to config.py.example index 7d0c3e7f..41f77881 100644 --- a/config.py +++ b/config.py.example @@ -1,4 +1,5 @@ LANG = 'en' +STATIC_PATH = './static/' + LANG USER_AGENT = 'https://github.com/5j9/citer' SPOOFED_USER_AGENT = '' diff --git a/dev/googlebooksdomains.py b/dev/googlebooksdomains.py new file mode 100644 index 00000000..8eaab5d1 --- /dev/null +++ b/dev/googlebooksdomains.py @@ -0,0 +1,13 @@ +from requests import get +from re import findall, MULTILINE + +github_content = get('https://github.com/SebastianJ/fiber-freeze/raw/master/data/https_urls.txt').content +github_domains = set(findall(rb'(?<=//)books\.google\.[^/\n]*', github_content, MULTILINE)) +assert len(github_domains) == 16 + +# Most referenced domains on the English Wikipedia (2015-05-15) (T96927) +# https://phabricator.wikimedia.org/P587 +phab_content = get('https://phab.wmfusercontent.org/file/data/nw6aboiuwxgb4mytb45u/PHID-FILE-perbg6gmtj55dgenca5h/Most_referenced_domains_on_the_English_Wikipedia_%282015-05-15%29_%28T96927%29').content +phab_domains = set(findall(rb'(?<=//)books\.google\.[^/\n]*', phab_content, MULTILINE)) + +assert 'books.google.co' not in phab_domains | github_domains diff --git a/install.py b/install.py index b90260fa..33af4791 100644 --- a/install.py +++ b/install.py @@ -23,18 +23,15 @@ def set_file_permissions(): def copy_config(): committer_date = check_output([ - 'git', '-C', HOME + '/www/python/src', 'log', '-1', '--format=%cI' - ]).partition(b'T')[0].replace(b'-', b'.') + 'git', '-C', HOME + '/www/python/src', 'log', '-1', '--format=%cd', + '--date=short']).rstrip().replace(b'-', b'.') with open(HOME + '/.citer_config', 'rb') as home_config: with open(HOME + '/www/python/src/config.py', 'wb') as src_config: - src_config.write( - sub( - b"(USER_AGENT = '.*)'\n", - br"\1 v" + committer_date + b"'\n", - home_config.read(), - 1, - ) - ) + src_config.write(sub( + b"(USER_AGENT = '.*)'\n", + br"\1 v" + committer_date + b"'\n", + home_config.read(), + 1)) def main(): diff --git a/lib/adinebook.py b/lib/adinebook.py deleted file mode 100644 index d8462166..00000000 --- a/lib/adinebook.py +++ /dev/null @@ -1,109 +0,0 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- - -"""All things that are specifically related to adinebook website""" - -from collections import defaultdict -import logging - -from langid import classify -from regex import compile as regex_compile -from requests import RequestException - -from lib.commons import first_last, dict_to_sfn_cit_ref, fetch - -ISBN_SEARCH = regex_compile( - r'\s*+' - r'(?:(?\d\d?)? (?[^،]*+)، )?(?\d{4})' -).search -PUBLISHER_SEARCH = regex_compile( - r'ناشر:(?:<[^>]++>\s*)++([^<\n]++)' -).search -TITLE_SEARCH = regex_compile( - r' tuple: - """Return the response namedtuple.""" - dictionary = url2dictionary(url) - dictionary['date_format'] = date_format - if 'language' not in dictionary: - # Assume that language is either fa or en. - # Todo: give warning about this assumption? - dictionary['language'] = \ - classify(dictionary['title'])[0] - return dict_to_sfn_cit_ref(dictionary) - - -def isbn2url(isbn: str): - """Convert isbn to AdinebookURL. Return the url as string.""" - # Apparently adinebook uses 10 digit codes (without hyphens) for its - # book-urls. If it's an isbn13 then the first 3 digits are excluded - isbn = isbn.replace('-', '').replace(' ', '') - if len(isbn) == 13: - isbn = isbn[3:] - url = 'http://www.adinebook.com/gp/product/' + isbn - return url - - -def url2dictionary(adinebook_url: str): - """Get adinebook_url and return the result as a dict.""" - try: - # Try to see if adinebook is available, - # ottobib should continoue its work in isbn.py if it is not. - r = fetch(adinebook_url) - adinebook_html = r.content.decode('utf-8') - except RequestException: - logger.exception(adinebook_url) - return - if 'صفحه مورد نظر پبدا نشد.' in adinebook_html: - return - else: - d = defaultdict(lambda: None, cite_type='book') - d['title'] = TITLE_SEARCH(adinebook_html)[1] - # initiating name lists: - others = [] - authors = [] - editors = [] - translators = [] - # building lists: - for name in AUTHORS_SEARCH(adinebook_html)[1].strip().split('،'): - if '(به اهتمام)' in name: - authors.append(first_last(name.partition('(به اهتمام)')[0])) - elif '(ویراستار)' in name: - editors.append(first_last(name.partition('(ویراستار)')[0])) - elif '(مترجم)' in name: - translators.append(first_last(name.partition('(مترجم)')[0])) - elif '(' in name: - others.append(('', name)) - else: - authors.append(first_last(name)) - if authors: - d['authors'] = authors - if others: - d['others'] = others - if editors: - d['editors'] = editors - if translators: - d['translators'] = translators - m = PUBLISHER_SEARCH(adinebook_html) - if m: - d['publisher'] = m.group(1) - m = DATE_SEARCH(adinebook_html) - if m: - d['month'] = m.group('month') - d['year'] = m.group('year') - m = ISBN_SEARCH(adinebook_html) - if m: - d['isbn'] = m.group(1) - return d - - -logger = logging.getLogger(__name__) diff --git a/lib/bibtex.py b/lib/bibtex.py index 90336ad5..890c72ad 100644 --- a/lib/bibtex.py +++ b/lib/bibtex.py @@ -1,6 +1,3 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- - """This module is used for parsing BibTeX entries. The goal of this code is to parse BibTeX entries from a number of known sites @@ -8,26 +5,26 @@ incorrectly or incompletely as TeX system is very complex and this module is not intended to parse TeX. -Some of the known issues: +Known issues: * Currently it does not detect special symbols and many TeX escape sequences (more information: http://www.bibtex.org/SpecialSymbols/) - * String concatinatins are not recognized. (e.g. "str1" # "str2") + * String concatenations are not recognized. (e.g. "str1" # "str2") * Abbreviations are not supported (e.g. @string { foo = "Mrs. Foo" }) """ from collections import defaultdict -import regex as regex +from regex import compile as rc from lib.commons import first_last # To remove Texts like {APA} from input. -WORDS_IN_BRACES_SUB = regex.compile(r'(? defaultdict: @@ -43,12 +40,10 @@ def parse(bibtex): bibtex = special_sequence_cleanup(bibtex) d = search_for_tag(bibtex) # cite_type: book, journal, incollection, etc. - m = TYPE_SEARCH(bibtex) - if m: - d['cite_type'] = m.group(1).strip().lower() + if (m := TYPE_SEARCH(bibtex)) is not None: + d['cite_type'] = m[1].strip().lower() # author - author = d['author'] - if author: + if author := d['author']: d['authors'] = names = [] names_append = names.append for author in author.split(' and '): @@ -59,8 +54,7 @@ def parse(bibtex): names_append(first_last(author)) del d['author'] # editor, not tested, just a copy of author - editor = d['editor'] - if editor: + if editor := d['editor']: d['editors'] = names = [] names_append = names.append for editor in editor.split(' and '): @@ -70,8 +64,7 @@ def parse(bibtex): continue names_append(first_last(editor)) del d['editor'] - pages = d['pages'] - if pages: + if pages := d['pages']: d['page'] = \ pages.replace(' ', '').replace('--', '–').replace('-', '–') return d diff --git a/lib/commons.py b/lib/commons.py index 689ffa12..587435e6 100644 --- a/lib/commons.py +++ b/lib/commons.py @@ -1,11 +1,6 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- - -"""Common variables, functions, and classes used in string conversions, etc.""" - from calendar import month_abbr, month_name -from datetime import datetime -from datetime import date as datetime_date +from datetime import datetime, date as datetime_date +from functools import partial from json import dumps as json_dumps from isbnlib import mask as isbn_mask, NotValidISBNError @@ -21,21 +16,39 @@ from lib.generator_fa import sfn_cit_ref +# The regex is from: +# http://stackoverflow.com/questions/27910/finding-a-doi-in-a-document-or-page +DOI_SEARCH = regex_compile( + r''' + \b + 10\.[0-9]{4,}+ + (?:\.[0-9]++)*+ + /[^"&\'\s]++ + \b + ''', + VERBOSE, +).search + + b_TO_NUM = {name.lower(): num for num, name in enumerate(month_abbr) if num} B_TO_NUM = {name.lower(): num for num, name in enumerate(month_name) if num} -# jB_TO_NUM contains entries for both ی and ي jB_TO_NUM = { 'فروردین': 1, + 'فروردين': 1, 'اردیبهشت': 2, + 'ارديبهشت': 2, 'خرداد': 3, 'تیر': 4, + 'تير': 4, 'مرداد': 5, 'شهریور': 6, + 'شهريور': 6, 'مهر': 7, 'آبان': 8, 'آذر': 9, 'دی': 10, + 'دي': 10, 'بهمن': 11, 'اسفند': 12} @@ -57,10 +70,7 @@ (?:(?:(?:Sept|Nov|Dec)em)|Octo)ber)) ''') # فروردین|اردیبهشت|خرداد... -jB = ( - '(?>(?' - + '|'.join([jm for jm in jB_TO_NUM]).replace('ی', '[یي]') - + '))') +jB = f"(?>(?{'|'.join([jm for jm in jB_TO_NUM]).replace('ی', '[یي]')}))" # Month abbreviations: b = r'(?>(?Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)).?' # Month numbers 0?1-12 @@ -74,23 +84,49 @@ # Gregorian year pattern 1900-2099 Y = r'(?(?:19|20)\d\d)' ANYDATE_PATTERN = ( - '(?:(?:' + B + '|' + b + r')\ ' + d + r',?\ ' + Y - + '|' + d + r'\ (?:' + B + '|' + b + r')\ ' + Y - + '|' + Y + '(?[-/])' + zm + '(?P=sep)' + zd - + '|' + r'(?\d\d?)\ ' + jB + r'\ (?\d\d\d\d)' - + r'|\b' + Y + zm + zd - + ')') + fr'(?:(?:{B}|{b})\ {d},?\ {Y}|{d}\ (?:{B}|{b})\ {Y}|{Y}(?[-/]){zm}' + fr'(?P=sep){zd}|(?\d\d?)\ {jB}\ (?\d\d\d\d))') ANYDATE_SEARCH = regex_compile(ANYDATE_PATTERN, VERBOSE).search DIGITS_FINDALL = regex_compile(r'\d').findall MC_SUB = regex_compile(r'MC(\w)', IGNORECASE).sub +LAST_FIRST = partial(regex_compile(r'[,،]').split, maxsplit=1) AGENT_HEADER = { 'User-Agent': USER_AGENT, # Not required but recommended by # https://meta.wikimedia.org/wiki/User-Agent_policy - 'Api-User-Agent': NCBI_TOOL + '/' + NCBI_EMAIL} -SPOOFED_AGENT_HEADER = {'User-Agent': SPOOFED_USER_AGENT} + 'Api-User-Agent': f'{NCBI_TOOL}/{NCBI_EMAIL}'} +SPOOFED_AGENT_HEADER = { + 'User-Agent': SPOOFED_USER_AGENT, + 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", +} +REQUEST = partial(Session().request, timeout=10) + +# original regex from: +# https://www.debuggex.com/r/0Npla56ipD5aeTr9 +# https://www.debuggex.com/r/2s3Wld3CVCR1wKoZ +ISBN_10OR13_SEARCH = regex_compile( + r'97[89]([ -]?+)(?=\d{1,5}\1?+\d{1,7}\1?+\d{1,6}\1?+\d)(?:\d\1*){9}\d' + r'|(?=\d{1,5}([ -]?+)\d{1,7}\2?+\d{1,6}\2?+\d)(?:\d\2*+){9}[\dX]' +).search + +ISBN10_SEARCH = regex_compile( + r'(?=\d{1,5}([ -]?+)\d{1,7}\1?+\d{1,6}\1?+\d)(?:\d\1*+){9}[\dX]' +).search + +ISBN13_SEARCH = regex_compile( + r'97[89]([ -]?+)(?=\d{1,5}\1?+\d{1,7}\1?+\d{1,6}\1?+\d)(?:\d\1*+){9}\d' +).search + + +# original regex from: http://stackoverflow.com/a/14260708/2705757 +# ISBN_REGEX = regex_compile( +# r'(?=[-0-9 ]{17}|[-0-9X ]{13}|[0-9X]{10})(?:97[89][- ]?)' +# r'?[0-9]{1,5}[- ]?(?:[0-9]+[- ]?){2}[0-9X]' +# ) + +FOUR_DIGIT_NUM = regex_compile(r'\d\d\d\d').search class InvalidNameError(ValueError): @@ -100,28 +136,27 @@ class InvalidNameError(ValueError): class NumberInNameError(InvalidNameError): - """Raise when a RawName() contains digits..""" + """Raise when a RawName() contains digits.""" -def fetch(url, spoof=False, **kwargs): - with Session() as session: - return session.request( - 'get', url, timeout=10, - headers=SPOOFED_AGENT_HEADER if spoof else AGENT_HEADER, - **kwargs) +class ReturnError(RuntimeError): + """Raise to display message to end user. -def dict_to_sfn_cit_ref(dictionary) -> tuple: - """Return (sfn, cite, ref) strings. - - dictionary should be ready before calling this function. - The dictionary will be cleaned up (empty values will be removed) and - all values will be encoded using encode_for_template() function. - ISBN (if exist) will be hyphenated. + Pass sfn, cit, and ref fields as positional args. """ - value_encode(dictionary) - isbn = dictionary['isbn'] - if isbn: + + +def request(url, spoof=False, method='get', **kwargs): + headers = SPOOFED_AGENT_HEADER if spoof else AGENT_HEADER + if 'headers' in kwargs: + headers |= kwargs.pop('headers') + return REQUEST(method, url, headers=headers, **kwargs) + + +def dict_to_sfn_cit_ref(dictionary) -> tuple: + # Return (sfn, cite, ref) strings. + if isbn := dictionary.get('isbn'): try: dictionary['isbn'] = isbn_mask(isbn) except NotValidISBNError: @@ -130,7 +165,7 @@ def dict_to_sfn_cit_ref(dictionary) -> tuple: return sfn_cit_ref(dictionary) -def sfn_cit_ref_to_json(response) -> str: +def scr_to_json(response) -> str: """Generate api JSON response containing sfn, cite and ref.""" return json_dumps({ 'reference_tag': response.ref, @@ -153,7 +188,10 @@ def first_last(fullname, separator=None) -> tuple: >>> first_last('DeBolt, V.', ',') ('V.', 'DeBolt') - >>> first_last('BBC', None) + The function is more strict if the separator is None: + + >>> first_last('BBC', None) # InvalidNameError + >>> first_last('BBC', ',') ('', 'BBC') """ fullname = fullname.strip() @@ -169,29 +207,26 @@ def first_last(fullname, separator=None) -> tuple: fullname = fullname[:-4] else: suffix = None - if not separator: - if ',' in fullname: - separator = ',' - elif '،' in fullname: - separator = '،' - if separator: + if separator is None: + try: + lastname, firstname = LAST_FIRST(fullname) + except ValueError: # not enough values to unpack, use whitespace + sname = fullname.split() + if len(sname) == 1: # single word first-last with None separator + raise InvalidNameError + lastname = sname.pop() + firstname = ' '.join(sname) + else: if separator in fullname: lastname, _, firstname = fullname.partition(separator) else: lastname, firstname = fullname, '' - else: - sname = fullname.split() - lastname = sname.pop() - firstname = ' '.join(sname) firstname = firstname.strip() if (firstname.isupper() and lastname.isupper()) or \ (firstname.islower() and lastname.islower()): firstname = firstname.title() lastname = lastname.title() - lastname = MC_SUB( - lambda mtch: 'Mc' + mtch.group(1).upper(), - lastname, - ) + lastname = MC_SUB(lambda m: 'Mc' + m[1].upper(), lastname) if suffix: firstname += suffix.title() return firstname, lastname @@ -227,72 +262,29 @@ def find_any_date(str_or_match) -> datetime.date or None: groupdict = match.groupdict() day = int(groupdict['d']) year = int(groupdict['Y']) - month = groupdict.get('jB') today = datetime.today().date() - if month: + get = groupdict.get + + if (month := get('jB')) is not None: date = jdate(year, jB_TO_NUM[month], day).togregorian() if date <= today: return date return - month = groupdict.get('B') - if month: + + if (month := get('B')) is not None: date = datetime_date(year, B_TO_NUM[month.lower()], day) if date <= today: return date return - month = groupdict.get('b') - if month: + + if (month := get('b')) is not None: date = datetime_date(year, b_TO_NUM[month.lower()], day) if date <= today: return date return - month = groupdict.get('m') - if month: + + if (month := get('m')) is not None: date = datetime_date(year, int(month), day) if date <= today: return date return - - -def bidi_pop(string) -> str: - """Makes sure all LRE, RLE, LRO, or RLO chars are terminated with PDF.""" - # Pop isolations - isolates = [ - '\u2066', # LRI - '\u2067', # RLI - '\u2068', # FSI - ] - diff = sum(string.count(c) for c in isolates) - \ - string.count('\u2069') # PDI - string += '\u2069' * diff - # Pop embeddings and overrides - diff = sum( - string.count(c) for c in ( - '\u202A', # LRE - '\u202B', # RLE - '\u202D', # LRO - '\u202E', # RLO - ) - ) - string.count('\u202C') # PDF - return string + '\u202C' * diff - - -def value_encode(dictionary) -> None: - """Cleanup dictionary values. - - * Remove any key with False bool value. - * Replace special characters in dictionary values with their respective - HTML entities. - * Strip all values. - """ - for k, v in dictionary.items(): - if isinstance(v, str): - v = ( - bidi_pop(v.strip()) - .replace('|', '&#124;') - .replace('[', '&#91;') - .replace(']', '&#93;') - .replace('\r\n', ' ') - .replace('\n', ' ') - ) - dictionary[k] = v diff --git a/lib/doi.py b/lib/doi.py index dfc51ebe..03e09495 100644 --- a/lib/doi.py +++ b/lib/doi.py @@ -1,99 +1,77 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- - """Codes related to DOI inputs.""" from collections import defaultdict -from datetime import date as datetime_date -from urllib.parse import unquote +from datetime import datetime +from typing import Any +from urllib.parse import unquote_plus from html import unescape from langid import classify -from regex import compile as regex_compile, VERBOSE -from lib.commons import dict_to_sfn_cit_ref, fetch +from lib.commons import request, DOI_SEARCH from config import LANG -# The regex is from: -# http://stackoverflow.com/questions/27910/finding-a-doi-in-a-document-or-page -DOI_SEARCH = regex_compile( - r''' - \b( - 10\.[0-9]{4,}+ - (?:\.[0-9]++)*+ - /[^"&\'\s]++ - )\b - ''', - VERBOSE, -).search - - -def doi_sfn_cit_ref(doi_or_url, pure=False, date_format='%Y-%m-%d') -> tuple: - """Return the response namedtuple.""" +def doi_to_dict(doi_or_url, pure=False, date_format='%Y-%m-%d', /) -> dict: if pure: doi = doi_or_url else: # unescape '&', '<', and '>' in doi_or_url # decode percent encodings - decoded_url = unquote(unescape(doi_or_url)) - doi = DOI_SEARCH(decoded_url)[1] + decoded_url = unquote_plus(unescape(doi_or_url)) + doi = DOI_SEARCH(decoded_url)[0] dictionary = get_crossref_dict(doi) dictionary['date_format'] = date_format if LANG == 'fa': dictionary['language'] = classify(dictionary['title'])[0] - return dict_to_sfn_cit_ref(dictionary) + return dictionary def get_crossref_dict(doi) -> defaultdict: """Return the parsed data of crossref.org for the given DOI.""" - # See https://github.com/CrossRef/rest-api-doc/blob/master/api_format.md - # for documentation. - # Force using the version 1 of the API to prevent breakage. See: - # https://github.com/CrossRef/rest-api-doc/blob/master/rest_api.md#how-to-manage-api-versions - j = fetch('http://api.crossref.org/v1/works/' + doi).json() - assert j['status'] == 'ok' - d = defaultdict( - lambda: None, {k.lower(): v for k, v in j['message'].items()}) - - d['cite_type'] = d.pop('type') - - for field in ('title', 'container-title', 'issn', 'isbn'): - value = d[field] - if value: - d[field] = value[0] - - date = d['issued']['date-parts'][0] - date_len = len(date) - if date_len == 3: - d['date'] = datetime_date(*date) - elif date_len == 2: - d['year'], d['month'] = str(date[0]), str(date[1]) - else: - year = date[0] - # date can be of the form [None] - # https://github.com/CrossRef/rest-api-doc/issues/169 - if year: - d['year'] = str(date[0]) - - authors = d['author'] - if authors: - d['authors'] = \ - [(name['given'], name['family']) for name in authors] - - editors = d['editor'] - if editors: - d['editors'] = \ - [(name['given'], name['family']) for name in editors] - - translators = d['translator'] - if translators: - d['translators'] = \ - [(name['given'], name['family']) for name in translators] - - page = d['page'] - if page: + # See https://citation.crosscite.org/docs.html for documentation. + j = request( + f'https://doi.org/{doi}', + headers={"Accept": "application/vnd.citationstyles.csl+json"} + ).json() + + d : defaultdict[str, Any] = defaultdict( + lambda: None, {k.lower(): v for k, v in j.items()}) + + d['cite_type'] = d['type'] + + if (author := d['author']) is not None: + d['authors'] = [ + (a['given'], a['family']) for a in author if 'given' in a + ] + + if (issn := d['issn']) is not None: + d['issn'] = issn[0] + + if (published := d['published']) is not None: + date = published['date-parts'][0] + if len(date) == 3: + d['date'] = datetime(*date) + else: # todo: better handle the case where len == 2 + d['year'] = f'{date[0]}' + + if (page := d['page']) is not None: d['page'] = page.replace('-', '–') + if (isbn := d['isbn']) is not None: + d['isbn'] = isbn[0] + return d + + +def extract_names(d: dict, from_key: str, to_key: str): + if (from_values := d[from_key]) is None: + return + to_values = d[to_key] = [] + authors_append = to_values.append + for from_value in from_values: + try: + authors_append((from_value['given'], from_value['family'])) + except KeyError: + pass diff --git a/lib/generator_en.py b/lib/generator_en.py index 0c025c4e..0f158546 100644 --- a/lib/generator_en.py +++ b/lib/generator_en.py @@ -1,22 +1,24 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- - """Codes required to create English Wikipedia citation templates.""" -import re from datetime import date as datetime_date +from functools import partial from collections import defaultdict from logging import getLogger +from regex import compile as regex_compile + from lib.language import TO_TWO_LETTER_CODE # Includes ShortDOIs (See: http://shortdoi.org/) and # https://www.crossref.org/display-guidelines/ -DOI_URL_MATCH = re.compile( - r'https?://(dx\.)?doi\.org/' -).match +DOI_URL_MATCH = regex_compile(r'https?://(dx\.)?doi\.org/').match +DIGITS_TO_EN = str.maketrans('۰۱۲۳۴۵۶۷۸۹', '0123456789') + +refless = partial(regex_compile( + r'( \| ref=({{.*?}}|harv))(?P \| |}})' +).sub, r'\g') TYPE_TO_CITE = { # BibTex types. Descriptions are from @@ -35,6 +37,7 @@ 'manual': 'book', # An article from a journal or magazine. 'article': 'journal', + 'article-journal': 'journal', # The same as INPROCEEDINGS, included for Scribe compatibility. 'conference': 'conference', # An article in a conference proceedings. @@ -65,6 +68,7 @@ 'reference-entry': '', 'proceedings-article': 'conference', 'journal': 'journal', + 'jour': 'journal', # https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id=22368089&retmode=json&tool=my_tool&email=my_email@example.com 'Journal Article': 'journal', 'component': '', @@ -80,20 +84,21 @@ 'book-series': 'book', 'edited-book': 'book', 'standard-series': '', + 'rprt': 'report', }.get def sfn_cit_ref(d: defaultdict) -> tuple: - """Create citation templates according to the given dictionary.""" + """Return sfn, citation, and ref.""" date_format = d['date_format'] - cite_type = TYPE_TO_CITE(d['cite_type']) - if not cite_type: + if not (cite_type := TYPE_TO_CITE(d['cite_type'])): logger.warning('Unknown citation type: %s, d: %s', cite_type, d) cite_type = '' - cit = '* {{cite ' + cite_type + cit = '* {{cite' + else: + cit = '* {{cite ' + cite_type sfn = '{{sfn' - authors = d['authors'] publisher = d['publisher'] website = d['website'] title = d['title'] @@ -103,7 +108,7 @@ def sfn_cit_ref(d: defaultdict) -> tuple: else: journal = d['journal'] - if authors: + if authors := d['authors']: cit += names2para(authors, 'first', 'last', 'author') # {{sfn}} only supports a maximum of four authors for first, last in authors[:4]: @@ -112,26 +117,23 @@ def sfn_cit_ref(d: defaultdict) -> tuple: # the same order should be used in citation_template: sfn += ' | ' + ( publisher or - "''" + journal + "''" if journal else - "''" + website + "''" if website else + f"''{journal}''" if journal else + f"''{website}''" if website else title or 'Anon.' ) - editors = d['editors'] - if editors: + if editors := d['editors']: cit += names2para(editors, 'editor-first', 'editor-last', 'editor') - translators = d['translators'] - if translators: + if translators := d['translators']: for i, (first, last) in enumerate(translators): - translators[i] = first, last + ' (مترجم)' + translators[i] = first, f'{last} (مترجم)' # Todo: add a 'Translated by ' before name of translators? others = d['others'] if others: others.extend(d['translators']) else: d['others'] = d['translators'] - others = d['others'] - if others: + if others := d['others']: cit += names1para(others, 'others') if cite_type == 'book': @@ -140,99 +142,89 @@ def sfn_cit_ref(d: defaultdict) -> tuple: booktitle = None if booktitle: - cit += ' | title=' + booktitle - if title: - cit += ' | chapter=' + title + cit += f' | title={booktitle}' + if title: + cit += f' | chapter={title}' elif title: - cit += ' | title=' + title + cit += f' | title={title}' if journal: - cit += ' | journal=' + journal + cit += f' | journal={journal}' elif website: - cit += ' | website=' + website + cit += f' | website={website}' - chapter = d['chapter'] - if chapter: - cit += ' | chapter=' + chapter + if chapter := d['chapter']: + cit += f' | chapter={chapter}' - publisher = d['publisher'] or d['organization'] - if publisher: - cit += ' | publisher=' + publisher + if publisher := (d['publisher'] or d['organization']): + cit += f' | publisher={publisher}' - address = d['address'] or d['publisher-location'] - if address: - cit += ' | publication-place=' + address + if address := (d['address'] or d['publisher-location']): + cit += f' | publication-place={address}' - edition = d['edition'] - if edition: - cit += ' | edition=' + edition + if edition := d['edition']: + cit += f' | edition={edition}' - series = d['series'] - if series: - cit += ' | series=' + series + if series := d['series']: + cit += f' | series={series}' - volume = d['volume'] - if volume: - cit += ' | volume=' + volume + if volume := d['volume']: + cit += f' | volume={volume.translate(DIGITS_TO_EN)}' - issue = d['issue'] or d['number'] - if issue: - cit += ' | issue=' + issue + if issue := (d['issue'] or d['number']): + cit += f' | issue={issue}' - date = d['date'] - if date: + if date := d['date']: if not isinstance(date, str): date = date.strftime(date_format) - cit += ' | date=' + date + cit += f' | date={date}' - year = d['year'] - if year: + if year := d['year']: + year = str(int(year)) # convert any non-Latin digits to English ones if not date or year not in date: - cit += ' | year=' + year - sfn += ' | ' + year + cit += f' | year={year}' + sfn += f' | {year}' - isbn = d['isbn'] - if isbn: - cit += ' | isbn=' + isbn + if isbn := d['isbn']: + cit += f' | isbn={isbn}' - issn = d['issn'] - if issn: - cit += ' | issn=' + issn + if issn := d['issn']: + cit += f' | issn={issn}' - pmid = d['pmid'] - if pmid: - cit += ' | pmid=' + pmid + if pmid := d['pmid']: + cit += f' | pmid={pmid}' - pmcid = d['pmcid'] - if pmcid: - cit += ' | pmc=' + pmcid + if pmcid := d['pmcid']: + cit += f' | pmc={pmcid}' - doi = d['doi'] - if doi: - cit += ' | doi=' + doi + if doi := d['doi']: + cit += f' | doi={doi}' - oclc = d['oclc'] - if oclc: - cit += ' | oclc=' + oclc + if oclc := d['oclc']: + cit += f' | oclc={oclc}' - pages = d['page'] - if pages: + if jstor := d['jstor']: + cit += f' | jstor={jstor}' + jstor_access = d['jstor-access'] + if jstor_access: + cit += f' | jstor-access=free' + + if pages := d['page']: if '–' in pages: - sfn += ' | pp=' + pages + sfn += f' | pp={pages}' else: - sfn += ' | p=' + pages + sfn += f' | p={pages}' if cite_type == 'journal': if pages: if '–' in pages: - cit += ' | pages=' + pages + cit += f' | pages={pages}' else: - cit += ' | page=' + pages + cit += f' | page={pages}' - url = d['url'] - if url: + if url := d['url']: # Don't add a DOI URL if we already have added a DOI. if not doi or not DOI_URL_MATCH(url): - cit += ' | url=' + url + cit += f' | url={url}' else: # To prevent addition of access date url = None @@ -240,58 +232,44 @@ def sfn_cit_ref(d: defaultdict) -> tuple: if not pages and cite_type != 'web': sfn += ' | p=' - archive_url = d['archive-url'] - if archive_url: + if archive_url := d['archive-url']: cit += ( - ' | archive-url=' + archive_url + - ' | archive-date=' + d['archive-date'].strftime(date_format) + - ' | dead-url=' + d['dead-url'] - ) + f' | archive-url={archive_url}' + f' | archive-date={d["archive-date"].strftime(date_format)}' + f' | url-status={d["url-status"]}') - language = d['language'] - if language: + if language := d['language']: language = TO_TWO_LETTER_CODE(language.lower(), language) if language.lower() != 'en': cit += ' | language=' + language - # Todo: Template:Citation generates anchors for Harvard by default - # references - # whereas the Cite templates by default do not (although they can be - # made to - # do so). - if authors: - cit += ' | ref=harv' - else: + if not authors: # order should match sfn_template - cit += ' | ref={{sfnref | ' +\ - (publisher or journal or website or title or 'Anon.') + cit += ' | ref={{sfnref | ' \ + f'{publisher or journal or website or title or "Anon."}' if year: - cit += ' | ' + year + cit += f' | {year}' cit += '}}' if url: - cit += ' | access-date=' + datetime_date.today().strftime(date_format) + cit += f' | access-date={datetime_date.today().strftime(date_format)}' cit += '}}' sfn += '}}' # Finally create the ref tag. name = sfn[8:-2].replace(' | ', ' ').replace("'", '') - text = re.sub( - r'( \| ref=({{.*?}}|harv))(?P \| |}})', - r'\g', - cit[2:], - ) + text = refless(cit[2:]) if ' p=' in name and ' | page=' not in text: name = name.replace(' p=', ' p. ') if pages: - text = text[:-2] + ' | page=' + pages + '}}' + text = f'{text[:-2]} | page={pages}}}}}' else: - text = text[:-2] + ' | page=}}' + text = f'{text[:-2]} | page=}}}}' elif ' pp=' in name: name = name.replace(' pp=', ' pp. ') if pages and ' | pages=' not in text: - text = text[:-2] + ' | pages=' + pages + '}}' - ref = '<ref name="' + name + '">' + text + '</ref>' + text = f'{text[:-2]} | pages={pages}}}}}' + ref = f'<ref name="{name}">{text}</ref>' return sfn, cit, ref @@ -303,38 +281,35 @@ def names2para(names, fn_parameter, ln_parameter, nofn_parameter=None): c += 1 if c == 1: if first or not nofn_parameter: - s += ' | ' + ln_parameter + '=' + last - s += ' | ' + fn_parameter + '=' + first + s += f' | {ln_parameter}={last} | {fn_parameter}={first}' else: - s += ' | ' + nofn_parameter + '=' + fullname(first, last) + s += f' | {nofn_parameter}={fullname(first, last)}' else: if first or not nofn_parameter: - s += ' | ' + ln_parameter + str(c) + '=' + last - s += ' | ' + fn_parameter + str(c) + '=' + first + s += f' | {ln_parameter}{c}={last} | {fn_parameter}{c}={first}' else: - s += ' | ' + nofn_parameter + str(c) + '=' + \ - fullname(first, last) + s += f' | {nofn_parameter}{c}={fullname(first, last)}' return s def names1para(translators, para): """Take list of names. Return the string to be appended to citation.""" - s = ' | ' + para + '=' + s = f' | {para}=' c = 0 for first, last in translators: c += 1 if c == 1: s += fullname(first, last) elif c == len(translators): - s += ', and ' + fullname(first, last) + s += f', and {fullname(first, last)}' else: - s += ', ' + fullname(first, last) + s += f', {fullname(first, last)}' return s def fullname(first: str, last: str) -> str: if first: - return first + ' ' + last + return f'{first} {last}' return last diff --git a/lib/generator_fa.py b/lib/generator_fa.py index abc023d1..be6439c9 100644 --- a/lib/generator_fa.py +++ b/lib/generator_fa.py @@ -1,6 +1,3 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- - """Codes required to create citation templates for wikifa.""" @@ -63,6 +60,7 @@ 'journal': 'ژورنال', # https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id=22368089&retmode=json&tool=my_tool&email=my_email@example.com 'Journal Article': 'ژورنال', + 'article-journal': 'ژورنال', 'component': '', 'book-chapter': 'کتاب', 'report-series': 'report', @@ -76,6 +74,7 @@ 'book-series': 'کتاب', 'edited-book': 'کتاب', 'standard-series': '', + 'jour': 'ژورنال', }.get # According to https://en.wikipedia.org/wiki/Help:Footnotes, @@ -87,9 +86,8 @@ def sfn_cit_ref(d: defaultdict) -> tuple: - """Create citation templates using the given dictionary.""" - cite_type = TYPE_TO_CITE(d['cite_type']) - if not cite_type: + """Return sfn, citation, and ref.""" + if not (cite_type := TYPE_TO_CITE(d['cite_type'])): logger.warning('Unknown citation type: %s, d: %s', cite_type, d) cite_type = '' if cite_type in ('کتاب', 'ژورنال', 'وب'): @@ -97,8 +95,7 @@ def sfn_cit_ref(d: defaultdict) -> tuple: else: return en_citations(d) - authors = d['authors'] - if authors: + if authors := d['authors']: cit += names2para(authors, 'نام', 'نام خانوادگی', 'نویسنده') sfn = '<ref>{{پک' for first, last in authors[:4]: @@ -106,22 +103,17 @@ def sfn_cit_ref(d: defaultdict) -> tuple: else: sfn = '<ref>{{پک/بن' - editors = d['editors'] - if editors: + if editors := d['editors']: cit += names2para( - editors, 'نام ویراستار', 'نام خانوادگی ویراستار', 'ویراستار' - ) + editors, 'نام ویراستار', 'نام خانوادگی ویراستار', 'ویراستار') - translators = d['translators'] - if translators: + if translators := d['translators']: cit += names1para(translators, 'ترجمه') - others = d['others'] - if others: + if others := d['others']: cit += names1para(others, 'دیگران') - year = d['year'] - if year: + if year := d['year']: sfn += ' | ' + year if cite_type == 'book': @@ -148,81 +140,67 @@ def sfn_cit_ref(d: defaultdict) -> tuple: else: website = d['website'] if website: - cit += ' | وب‌گاه=' + website + cit += ' | وبگاه=' + website - chapter = d['chapter'] - if chapter: + if chapter := d['chapter']: cit += ' | فصل=' + chapter - publisher = d['publisher'] or d['organization'] - if publisher: + if publisher := (d['publisher'] or d['organization']): cit += ' | ناشر=' + publisher - address = d['address'] or d['publisher-location'] - if address: + if address := (d['address'] or d['publisher-location']): cit += ' | مکان=' + address - edition = d['edition'] - if edition: + if edition := d['edition']: cit += ' | ویرایش=' + edition - series = d['series'] - if series: + if series := d['series']: cit += ' | سری=' + series - volume = d['volume'] - if volume: + if volume := d['volume']: cit += ' | جلد=' + volume - issue = d['issue'] or d['number'] - if issue: + if issue := (d['issue'] or d['number']): cit += ' | شماره=' + issue - ddate = d['date'] - if ddate: + if ddate := d['date']: if isinstance(ddate, str): cit += ' | تاریخ=' + ddate else: cit += ' | تاریخ=' + date.isoformat(ddate) - - if year: + elif year: cit += ' | سال=' + year - month = d['month'] - if month: - cit += ' | ماه=' + month - - isbn = d['isbn'] - if isbn: + if isbn := d['isbn']: cit += ' | شابک=' + isbn - issn = d['issn'] - if issn: + if issn := d['issn']: cit += ' | issn=' + issn - pmid = d['pmid'] - if pmid: + if pmid := d['pmid']: cit += ' | pmid=' + pmid - pmcid = d['pmcid'] - if pmcid: + if pmcid := d['pmcid']: cit += ' | pmc=' + pmcid - doi = d['doi'] - if doi: + if doi := d['doi']: cit += ' | doi=' + doi - oclc = d['oclc'] - if oclc: + if oclc := d['oclc']: cit += ' | oclc=' + oclc + if jstor := d['jstor']: + cit += f' | jstor={jstor}' + jstor_access = d['jstor-access'] + if jstor_access: + cit += f' | jstor-access=free' + pages = d['page'] if cite_type == 'ژورنال': if pages: cit += ' | صفحه=' + pages - url = d['url'] - if url: + if url := d['url']: # Don't add a DOI URL if we already have added a DOI. if not doi or not DOI_URL_MATCH(url): cit += ' | پیوند=' + url @@ -230,33 +208,29 @@ def sfn_cit_ref(d: defaultdict) -> tuple: # To prevent addition of access date url = None - archive_url = d['archive-url'] - if archive_url: + if archive_url := d['archive-url']: cit += ( - ' | پیوند بایگانی=' + archive_url + - ' | تاریخ بایگانی=' + d['archive-date'].isoformat() + - ' | پیوند مرده=' + ('آری' if d['dead-url'] == 'yes' else 'نه') - ) + f' | پیوند بایگانی={archive_url}' + f' | تاریخ بایگانی={d["archive-date"].isoformat()}' + f" | پیوند مرده={('آری' if d['url-status'] == 'yes' else 'نه')}") - language = d['language'] - if language: + if language := d['language']: language = TO_TWO_LETTER_CODE(language.lower(), language) if cite_type == 'وب': - cit += ' | کد زبان=' + language + cit += f' | کد زبان={language}' else: - cit += ' | زبان=' + language - sfn += ' | زبان=' + language + cit += f' | زبان={language}' + sfn += f' | زبان={language}' if pages: - sfn += ' | ص=' + pages + sfn += f' | ص={pages}' # Seed the random generator before adding today's date. randseed(cit) ref_name = ( randchoice(ascii_lowercase) # it should contain at least one non-digit - + ''.join(randchoice(LOWER_ALPHA_DIGITS) for _ in range(4)) - ) + + ''.join(randchoice(LOWER_ALPHA_DIGITS) for _ in range(4))) if url: - cit += ' | تاریخ بازبینی=' + date.today().isoformat() + cit += f' | تاریخ بازبینی={date.today().isoformat()}' if not pages and cite_type != 'وب': sfn += ' | ص=' @@ -266,10 +240,10 @@ def sfn_cit_ref(d: defaultdict) -> tuple: # Finally create the ref tag. ref = cit[2:] if pages and ' | صفحه=' not in ref: - ref = ref[:-2] + ' | صفحه=' + pages + '}}' + ref = f'{ref[:-2]} | صفحه={pages}}}}}' elif not url: - ref = ref[:-2] + ' | صفحه=}}' - ref = '<ref name="' + ref_name + '">' + ref + '\u200F</ref>' + ref = f'{ref[:-2]} | صفحه=}}}}' + ref = f'<ref name="{ref_name}">{ref}\u200F</ref>' return sfn, cit, ref @@ -282,39 +256,35 @@ def names2para(names, fn_parameter, ln_parameter, nofn_parameter=None): if c == 1: if first or not nofn_parameter: s += ( - ' | ' + ln_parameter + '=' + last + - ' | ' + fn_parameter + '=' + first - ) + f' | {ln_parameter}=' + last + + f' | {fn_parameter}=' + first) else: - s += ' | ' + nofn_parameter + '=' + fullname(first, last) + s += f' | {nofn_parameter}=' + fullname(first, last) else: if first or not nofn_parameter: s += ( - ' | ' + ln_parameter + str(c).translate(DIGITS_TO_FA) - + '=' + last + - ' | ' + fn_parameter + str(c).translate(DIGITS_TO_FA) - + '=' + first - ) + f' | {ln_parameter}{str(c).translate(DIGITS_TO_FA)}' + f'={last} | {fn_parameter}{str(c).translate(DIGITS_TO_FA)}' + f'={first}') else: s += ( - ' | ' + nofn_parameter + str(c).translate(DIGITS_TO_FA) - + '=' + fullname(first, last) - ) + f' | {nofn_parameter}{str(c).translate(DIGITS_TO_FA)}' + f'={fullname(first, last)}') return s def names1para(translators, para): """Take list of names. Return the string to be appended to citation.""" - s = ' | ' + para + '=' + s = f' | {para}=' c = 0 for first, last in translators: c += 1 if c == 1: s += fullname(first, last) elif c == len(translators): - s += ' و ' + fullname(first, last) + s += f' و {fullname(first, last)}' else: - s += '، ' + fullname(first, last) + s += f'، {fullname(first, last)}' return s diff --git a/lib/googlebooks.py b/lib/googlebooks.py index bc09a133..e5729c7e 100644 --- a/lib/googlebooks.py +++ b/lib/googlebooks.py @@ -1,59 +1,30 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- - -"""All things specifically related to the Google Books website.""" - - from urllib.parse import parse_qs -from urllib.parse import urlparse from langid import classify -# import bibtex [1] -from lib.commons import fetch -from lib.ris import parse as ris_parse -from lib.commons import dict_to_sfn_cit_ref +from lib.commons import request +from lib.ris import ris_parse -def googlebooks_sfn_cit_ref(url, date_format='%Y-%m-%d') -> tuple: +def url_to_dict(parsed_url, date_format='%Y-%m-%d') -> dict: """Create the response namedtuple.""" - # bibtex_result = get_bibtex(url) [1] - # dictionary = bibtex.parse(bibtex_result) [1] - dictionary = ris_parse(get_ris(url)) + parsed_query = parse_qs(parsed_url.query) + + if (id_ := parsed_query.get('id')) is not None: + volume_id = id_[0] + else: # the new URL format + volume_id = parsed_url.path.rpartition('/')[2] + + dictionary = ris_parse(request( + f'https://{parsed_url.netloc}/books/download/?id={volume_id}' + f'&output=ris', spoof=True).content.decode('utf8')) dictionary['date_format'] = date_format - pu = urlparse(url) - pq = parse_qs(pu.query) - # default domain is prefered: - dictionary['url'] = 'https://' + pu.netloc + '/books?id=' + pq['id'][0] # manually adding page number to dictionary: - if 'pg' in pq: - dictionary['page'] = pq['pg'][0][2:] - dictionary['url'] += '&pg=' + pq['pg'][0] + if (pg := parsed_query.get('pg')) is not None: + pg0 = pg[0] + dictionary['page'] = pg0[2:] + dictionary['url'] += f'&pg={pg0}' # although google does not provide a language field: if not dictionary['language']: dictionary['language'] = classify(dictionary['title'])[0] - return dict_to_sfn_cit_ref(dictionary) - - -def get_bibtex(googlebook_url) -> bytes: - """Get bibtex file content from a noormags url.""" - # getting id: - pu = urlparse(googlebook_url) - pq = parse_qs(pu.query) - bookid = pq['id'][0] - url = 'http://books.google.com/books/download/?id=' +\ - bookid + '&output=bibtex' - # Agent spoofing is needed, otherwise: HTTP Error 401: Unauthorized - return fetch(url, spoof=True, timeout=10).content - - -def get_ris(googlebook_url): - """Get ris file content from a noormags url.""" - # getting id: - pu = urlparse(googlebook_url) - pq = parse_qs(pu.query) - bookid = pq['id'][0] - url = 'http://books.google.com/books/download/?id=' +\ - bookid + '&output=ris' - # Agent spoofing is needed, otherwise: HTTP Error 401: Unauthorized - return fetch(url, spoof=True).text + return dictionary diff --git a/lib/html/en.html b/lib/html/en.html index 17c1db6a..df30c33c 100644 --- a/lib/html/en.html +++ b/lib/html/en.html @@ -1,4 +1,5 @@ + Citer @@ -17,11 +18,11 @@

Date format:

- 2017-01-01 - January 1, 2017 - Jan 1, 2017 - 1 January 2017 - 1 Jan 2017 + 2020-01-01 + January 1, 2020 + Jan 1, 2020 + 1 January 2020 + 1 Jan 2020

Shortened footnote and citation: @@ -36,13 +37,13 @@ Google Books URL, DOI, ISBN, PMID, PMCID, OCLC number, or the URL of many major news websites.

- Note that there is always a chance of error in the generated output. Please check the results before using them on Wiki. + Note that there is always a chance of error in the generated output. Please check the results before using them on wiki.

- Found a bug or have a suggestion? Contact me on my talk page or open an issue on github. - Bookmarklet + Found a bug or have a suggestion? Contact me on my talk page or open an issue on GitHub. + Bookmarklet (drag to favorites bar)

- \ No newline at end of file + diff --git a/lib/html/en.js b/lib/html/en.js index 6e92ef0c..9e2a096f 100644 --- a/lib/html/en.js +++ b/lib/html/en.js @@ -1,5 +1,5 @@ /*jslint browser: true, regexp: true, white: true */ -var months = [ +var longMonths = [ 'January', 'February', 'March', @@ -13,6 +13,9 @@ var months = [ 'November', 'December' ]; +var shortMonths = longMonths.map((s) => s.slice(0, 3)); +var monthPattern = '(' + shortMonths.join('|') + '|' + longMonths.join('|') + ')'; + function getCheckedRadio() { 'use strict'; @@ -26,54 +29,63 @@ function getCheckedRadio() { } } -function ymd(date) { +function ymd(y, m, d) { 'use strict'; - return date.toISOString() - .slice(0, 10); + return `${y}-${(m + 1).toString().padStart(2,0)}-${d.toString().padStart(2,0)}`; } -function bbdy(date) { +function bbdy(y, m, d) { 'use strict'; - return months[date.getMonth()] + ' ' + date.toISOString() - .slice(8, 10) + ', ' + date.getFullYear(); + return longMonths[m] + ' ' + d + ', ' + y; } -function bdy(date) { +function bdy(y, m, d) { 'use strict'; - return months[date.getMonth()].slice(0, 3) + ' ' + date.toISOString() - .slice(8, 10) + ', ' + date.getFullYear(); + return shortMonths[m] + ' ' + d + ', ' + y; } -function dbby(date) { +function dbby(y, m, d) { 'use strict'; - return date.toISOString() - .slice(8, 10) + ' ' + months[date.getMonth()] + ' ' + date.getFullYear(); + return d + ' ' + longMonths[m] + ' ' + y; } -function dby(date) { +function dby(y, m, d) { 'use strict'; - return date.toISOString() - .slice(8, 10) + ' ' + months[date.getMonth()].slice(0, 3) + ' ' + date.getFullYear(); + return d + ' ' + shortMonths[m] + ' ' + y; +} + +function parseDate(s) { + 'use strict'; + var m = /(\d{4})-(\d{1,2})-(\d{1,2})/.exec(s); + if (m) { + return [parseInt(m[1]), parseInt(m[2]) - 1, parseInt(m[3])]; + } + m = RegExp(monthPattern + ' ' + /(\d{1,2})/.source + ', ' + /(\d{4})/.source, 'i').exec(s); + if (m) { + return [parseInt(m[3]), shortMonths.indexOf(m[1].slice(0, 3)), parseInt(m[2])]; + } + m = RegExp(/(\d{1,2})/.source + ' ' + monthPattern + ' ' + /(\d{4})/.source, 'i').exec(s); + if (m) { + return [parseInt(m[3]), shortMonths.indexOf(m[2].slice(0, 3)), parseInt(m[1])]; + } } function changeDates(id) { 'use strict'; - var i, utcdate, text1, text2, dates, date, newdate, formatter; + var i, text1, text2, dates, date, newdate, formatter; text1 = document.getElementById('shortened').innerHTML; text2 = document.getElementById('named_ref').innerHTML; dates = text1.match(/date=.*?(?=\}\}| \| )/g); if (!dates) return; for (i = 0; i < dates.length; i = i + 1) { - date = dates[i].slice(5); - if (date.indexOf('-') !== -1) { - utcdate = date; - } else { - utcdate = date + " UTC"; - } + date = dates[i].slice(5); // omit the `date=` part formatter = window[id]; - newdate = formatter(new Date(utcdate)) - .replace(/^[0]+/g, "") - .replace(" 0", " "); + var ymd = parseDate(date); + if (!ymd) { + console.warn('parseDate(date) returned null'); + continue; + } + newdate = formatter(...ymd); text1 = text1.replace(new RegExp('((?:access)?date=)' + date + '(?=}}| \\| )'), '$1' + newdate); text2 = text2.replace(new RegExp('((?:access)?date=)' + date + '(?=}}| \\| )'), '$1' + newdate); document.getElementById('shortened').innerHTML = text1; @@ -86,7 +98,7 @@ function setCookie(cname, cvalue, exdays) { var expires, d = new Date(); d.setTime(d.getTime() + (exdays * 24 * 60 * 60 * 1000)); expires = 'expires=' + d.toGMTString(); - document.cookie = cname + '=' + cvalue + '; ' + expires; + document.cookie = cname + '=' + cvalue + '; ' + expires + '; SameSite=None; Secure'; } function getCookie(cname) { diff --git a/lib/html/en.py b/lib/html/en.py index 064130f0..a0c458c8 100644 --- a/lib/html/en.py +++ b/lib/html/en.py @@ -1,77 +1,70 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- - """HTML skeleton of predefined en responses.""" from string import Template from os import name as osname +from os.path import dirname from zlib import adler32 +from config import STATIC_PATH + +htmldir = dirname(__file__) # Predefined responses -DEFAULT_SFN_CIT_REF = ( - 'Generated citation will appear here...', '', '', -) +DEFAULT_SCR = ( + 'Generated citation will appear here...', '', '') -UNDEFINED_INPUT_SFN_CIT_REF = ( +UNDEFINED_INPUT_SCR = ( 'Undefined input.', - 'Sorry, the input was not recognized. The error was logged.', - '', -) + 'Sorry, the input was not recognized.', + '') -HTTPERROR_SFN_CIT_REF = ( +HTTPERROR_SCR = ( 'HTTP error:', 'One or more of the web resources required to ' 'create this citation are not accessible at this moment.', - '', -) + '') -OTHER_EXCEPTION_SFN_CIT_REF = ( +OTHER_EXCEPTION_SCR = ( 'An unknown error occurred.', - 'The error was logged.', '', -) + '') -CSS = open('lib/html/en.css', 'rb').read() +CSS = open(f'{htmldir}/en.css', 'rb').read() CSS_HEADERS = [ ('Content-Type', 'text/css; charset=UTF-8'), ('Content-Length', str(len(CSS))), - ('Cache-Control', 'immutable, public, max-age=31536000'), -] + ('Cache-Control', 'immutable, public, max-age=31536000')] -JS = open('lib/html/en.js', 'rb').read() +JS = open(f'{htmldir}/en.js', 'rb').read() # Invalidate cache after css change. JS_HEADERS = [ ('Content-Type', 'application/javascript; charset=UTF-8'), ('Content-Length', str(len(JS))), - ('Cache-Control', 'immutable, public, max-age=31536000'), -] + ('Cache-Control', 'immutable, public, max-age=31536000')] # None-zero-padded day directive is os dependant ('%#d' or '%-d') # See http://stackoverflow.com/questions/904928/ HTML_SUBST = Template( - open('lib/html/en.html', encoding='utf8').read().replace( + open(f'{htmldir}/en.html', encoding='utf8').read().replace( # Invalidate css cache after any change in css file. '"stylesheet" href="./static/en', - '"stylesheet" href="./static/en' + str(adler32(CSS)), + '"stylesheet" href="' + STATIC_PATH + str(adler32(CSS)), 1, ).replace( # Invalidate js cache after any change in js file. 'src="./static/en', - 'src="./static/en' + str(adler32(JS)), + 'src="' + STATIC_PATH + str(adler32(JS)), 1, - ) - .replace('{d}', '#d' if osname == 'nt' else '-d') + ).replace('{d}', '#d' if osname == 'nt' else '-d') ).substitute -def sfn_cit_ref_to_html(sfn_cit_ref: tuple, date_format: str, input_type: str): +def scr_to_html(sfn_cit_ref: tuple, date_format: str, input_type: str): """Insert sfn_cit_ref into the HTML template and return response_body.""" date_format = date_format or '%Y-%m-%d' sfn, cit, ref = sfn_cit_ref return HTML_SUBST( sfn=sfn, cit=cit, ref=ref, - ).replace(date_format + '"', date_format + '" checked', 1).replace( - '="' + input_type + '"', '="' + input_type + '" selected', 1 - ) + ).replace(f'{date_format}"', f'{date_format}" checked', 1).replace( + f'="{input_type}"', f'="{input_type}" selected', 1) diff --git a/lib/html/fa.html b/lib/html/fa.html index 4f23d7b6..b8ed5e4c 100644 --- a/lib/html/fa.html +++ b/lib/html/fa.html @@ -1,4 +1,5 @@ + یادفا @@ -25,14 +26,14 @@
$ref
- \ No newline at end of file + diff --git a/lib/html/fa.py b/lib/html/fa.py index 2e73629c..b9c40a74 100644 --- a/lib/html/fa.py +++ b/lib/html/fa.py @@ -1,52 +1,48 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- - """HTML skeleton of the predefined fa responses.""" +from os.path import dirname from string import Template from zlib import adler32 +from config import STATIC_PATH + + +htmldir = dirname(__file__) -CSS = open('lib/html/fa.css', 'rb').read() +CSS = open(htmldir + '/fa.css', 'rb').read() CSS_HEADERS = [ ('Content-Type', 'text/css; charset=UTF-8'), ('Content-Length', str(len(CSS))), - ('Cache-Control', 'max-age=31536000'), -] + ('Cache-Control', 'max-age=31536000')] HTML_SUBST = Template( - open('lib/html/fa.html', encoding='utf8').read().replace( + open(htmldir + '/fa.html', encoding='utf8').read().replace( # Invalidate css cache after any change in css file. '"stylesheet" href="./static/fa', - '"stylesheet" href="./static/fa' + str(adler32(CSS)), - ) -).substitute + '"stylesheet" href="' + STATIC_PATH + str(adler32(CSS)) + )).substitute # Predefined responses -DEFAULT_SFN_CIT_REF = ('یادکرد ساخته‌شده اینجا نمایان خواهد شد...', '', '') -HTTPERROR_SFN_CIT_REF = ( +DEFAULT_SCR = ('یادکرد ساخته‌شده اینجا نمایان خواهد شد...', '', '') +HTTPERROR_SCR = ( 'خطای اچ‌تی‌تی‌پی:', 'یک یا چند مورد از منابع اینترنتی مورد ' 'نیاز برای ساخت این یادکرد در این لحظه ' 'در دسترس نیستند و یا ورودی نامعتبر است.', - '', -) -OTHER_EXCEPTION_SFN_CIT_REF = ( + '') +OTHER_EXCEPTION_SCR = ( 'خطای ناشناخته‌ای رخ داد..', 'اطلاعات خطا در سیاهه ثبت شد.', - '', -) -UNDEFINED_INPUT_SFN_CIT_REF = ( + '') +UNDEFINED_INPUT_SCR = ( 'ورودی تجزیه‌ناپذیر', 'پوزش، ورودی قابل پردازش نبود. خطا در سیاهه ثبت شد.', - '', -) + '') -def sfn_cit_ref_to_html(sfn_cit_ref: tuple, date_format: str, input_type: str): +def scr_to_html(sfn_cit_ref: tuple, date_format: str, input_type: str): """Insert sfn_cite_ref into the HTML template and return response_body.""" sfn, cit, ref = sfn_cit_ref return HTML_SUBST(sfn=sfn, cit=cit, ref=ref).replace( - '="' + input_type + '"', '="' + input_type + '" selected', 1 - ) + '="' + input_type + '"', '="' + input_type + '" selected', 1) diff --git a/lib/isbn_oclc.py b/lib/isbn_oclc.py index 574109ae..30bc8b99 100644 --- a/lib/isbn_oclc.py +++ b/lib/isbn_oclc.py @@ -1,48 +1,18 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- -"""Define functions to process ISBNs and OCLC numbers.""" - -# from collections import defaultdict +from collections import defaultdict +from logging import getLogger from threading import Thread -from typing import Optional +from typing import Optional, Any +from json import loads from langid import classify -from regex import compile as regex_compile, DOTALL - -from lib.adinebook import url2dictionary as adinebook_url2dictionary -from lib.adinebook import isbn2url as adinebook_isbn2url -from lib.bibtex import parse as bibtex_parse -from lib.commons import dict_to_sfn_cit_ref, fetch # , Name -from lib.ris import parse as ris_parse - - -# original regex from: -# https://www.debuggex.com/r/0Npla56ipD5aeTr9 -# https://www.debuggex.com/r/2s3Wld3CVCR1wKoZ -ISBN_10OR13_SEARCH = regex_compile( - r'97[89]([ -]?+)(?=\d{1,5}\1?+\d{1,7}\1?+\d{1,6}\1?+\d)(?:\d\1*){9}\d' - r'|(?=\d{1,5}([ -]?+)\d{1,7}\1?+\d{1,6}\1?+\d)(?:\d\1*+){9}[\dX]' -).search +from isbnlib import info as isbn_info -ISBN10_SEARCH = regex_compile( - r'(?=\d{1,5}([ -]?+)\d{1,7}\1?+\d{1,6}\1?+\d)(?:\d\1*+){9}[\dX]' -).search +from config import LANG +from lib.ketabir import url_to_dict as ketabir_url_to_dict +from lib.ketabir import isbn_to_url as ketabir_isbn2url +from lib.commons import request, ISBN13_SEARCH, ISBN10_SEARCH, ReturnError, \ + FOUR_DIGIT_NUM -ISBN13_SEARCH = regex_compile( - r'97[89]([ -]?+)(?=\d{1,5}\1?+\d{1,7}\1?+\d{1,6}\1?+\d)(?:\d\1*+){9}\d' -).search - - -# original regex from: http://stackoverflow.com/a/14260708/2705757 -# ISBN_REGEX = regex_compile( -# r'(?=[-0-9 ]{17}|[-0-9X ]{13}|[0-9X]{10})(?:97[89][- ]?)' -# r'?[0-9]{1,5}[- ]?(?:[0-9]+[- ]?){2}[0-9X]' -# ) - -OTTOBIB_SEARCH = regex_compile( - ']*+>(.*?)', - DOTALL, -).search RM_DASH_SPACE = str.maketrans('', '', '- ') @@ -54,87 +24,95 @@ class IsbnError(Exception): pass -def isbn_sfn_cit_ref( - isbn_container_str: str, pure: bool = False, date_format: str = '%Y-%m-%d' -) -> tuple: - """Create the response namedtuple.""" +def isbn_to_dict( + isbn_container_str: str, + pure: bool = False, + date_format: str = '%Y-%m-%d', +) -> dict: if pure: isbn = isbn_container_str else: # search for isbn13 - m = ISBN13_SEARCH(isbn_container_str) - if m: - isbn = m.group(0) + if (m := ISBN13_SEARCH(isbn_container_str)) is not None: + isbn = m[0] else: # search for isbn10 - m = ISBN10_SEARCH(isbn_container_str) - isbn = m.group(0) + isbn = ISBN10_SEARCH(isbn_container_str)[0] - adinebook_result_list = [] - adine_book_thread = Thread( - target=adinebook_thread_target, - args=(isbn, adinebook_result_list), - ) - adine_book_thread.start() + if (iranian_isbn := isbn_info(isbn) == 'Iran') is True: + ketabir_result_list = [] + ketabir_thread = Thread( + target=ketabir_thread_target, + args=(isbn, ketabir_result_list)) + ketabir_thread.start() citoid_result_list = [] citoid_thread = Thread( target=citoid_thread_target, - args=(isbn, citoid_result_list), - ) + args=(isbn, citoid_result_list)) citoid_thread.start() - ottobib_bibtex = ottobib(isbn) - if ottobib_bibtex: - otto_dict = bibtex_parse(ottobib_bibtex) - else: - otto_dict = None - - adine_book_thread.join() - if adinebook_result_list: - adine_dict = adinebook_result_list[0] + if iranian_isbn is True: + # noinspection PyUnboundLocalVariable + ketabir_thread.join() + # noinspection PyUnboundLocalVariable + if ketabir_result_list: + # noinspection PyUnboundLocalVariable + ketabir_dict = ketabir_result_list[0] + else: + ketabir_dict = None else: - adine_dict = None - dictionary = choose_dict(adine_dict, otto_dict) + ketabir_dict = None citoid_thread.join() if citoid_result_list: - dictionary['oclc'] = citoid_result_list[0]['oclc'] + citoid_dict = citoid_result_list[0] + else: + citoid_dict = None + + dictionary = combine_dicts(ketabir_dict, citoid_dict) dictionary['date_format'] = date_format if 'language' not in dictionary: dictionary['language'] = classify(dictionary['title'])[0] - return dict_to_sfn_cit_ref(dictionary) - - -def adinebook_thread_target(isbn: str, result: list) -> None: - """Append the dictionary generated by adinebook module to the result.""" - d = adinebook_url2dictionary(adinebook_isbn2url(isbn)) - if d: - result.append(d) + return dictionary + + +def ketabir_thread_target(isbn: str, result: list) -> None: + # noinspection PyBroadException + try: + if (url := ketabir_isbn2url(isbn)) is None: + return # ketab.ir does not have any entries for this isbn + if d := ketabir_url_to_dict(url): + result.append(d) + except Exception: + logger.exception('isbn: %s', isbn) + return -def choose_dict(adine_dict, otto_dict): +def combine_dicts(ketabir: dict, citoid: dict) -> dict: """Choose which source to use. - Return adine_dict if both dicts contain the same ISBN. - Return adine_dict if adine_dict is None. + Return ketabir_dict if both dicts are available and lang is fa. + Return otto_dict if both dicts are available and lang is not fa. + Return ketabir_dict if ketabir_dict is None. Return otto_dict otherwise. - - Note: AdineBook resolver removes 3 digits from ISBNs when converting - them into URLs. This makes it vulnerable to resolving wrong ISBNs. Thus - AdineBook should be passed as adine_dict. """ - if not otto_dict and not adine_dict: + if not ketabir and not citoid: raise IsbnError('Bibliographic information not found.') - if adine_dict and otto_dict: - # both exist - if isbn2int(adine_dict['isbn']) == isbn2int(otto_dict['isbn']): - return adine_dict # both isbns are equal - return otto_dict # isbns are not equal - if adine_dict: - return adine_dict # only adinebook exists - return otto_dict # only ottobib exists + + if not ketabir: + return citoid + elif not citoid: + return ketabir + + # both ketabid and citoid are available + if LANG == 'fa': + result = ketabir + if (oclc := citoid['oclc']) is not None: + result['oclc'] = oclc + return result + return citoid def isbn2int(isbn): @@ -143,62 +121,82 @@ def isbn2int(isbn): def get_citoid_dict(isbn) -> Optional[dict]: # https://www.mediawiki.org/wiki/Citoid/API - r = fetch( + r = request( 'https://en.wikipedia.org/api/rest_v1/data/citation/mediawiki/' + isbn) if r.status_code != 200: return - return r.json()[0] - # Currently get_citoid_dict is only used to get oclc id (T160845) - # j0 = r.json()[0] - # d = defaultdict(lambda: None, j0) - # d['cite_type'] = j0['itemType'] - # d['isbn'] = d['ISBN'][0] - # if 'date' in j0: - # d['year'] = j0['date'] - # if 'author' in j0: - # d['authors'] = [ - # Name(first.rstrip('.,'), last.rstrip('.,')) - # for last, first in j0['author'] - # ] - # if 'url' in j0: - # del d['url'] - # if 'place' in j0: - # d['publisher-location'] = j0['place'] - # return d + j0 = r.json()[0] + get = j0.get -def citoid_thread_target(isbn: str, result: list) -> None: - citoid_dict = get_citoid_dict(isbn) - if citoid_dict: - result.append(citoid_dict) + d = defaultdict(lambda: None) + + d['cite_type'] = j0['itemType'] + d['isbn'] = j0['ISBN'][0] + # worldcat url is not needed since OCLC param will create it + # d['url'] = j0['url'] + d['oclc'] = j0['oclc'] + d['title'] = j0['title'] + + authors = get('author') + contributors = get('contributor') + + if authors is not None and contributors is not None: + d['authors'] = authors + contributors + elif authors is not None: + d['authors'] = authors + elif contributors is not None: + d['authors'] = contributors + + if (publisher := get('publisher')) is not None: + d['publisher'] = publisher + + if (place := get('place')) is not None: + d['publisher-location'] = place + if (date := get('date')) is not None: + d['date'] = date -def ottobib(isbn): - """Convert ISBN to bibtex using ottobib.com.""" - m = OTTOBIB_SEARCH( - fetch('http://www.ottobib.com/isbn/' + isbn + '/bibtex').text) - if m: - return m.group(1) + return d -def oclc_sfn_cit_ref(oclc: str, date_format: str = '%Y-%m-%d') -> tuple: - text = fetch( - 'https://www.worldcat.org/oclc/' + oclc + '?page=endnote' - '&client=worldcat.org-detailed_record').text - if ' None: + if citoid_dict := get_citoid_dict(isbn): + result.append(citoid_dict) + + +def oclc_dict(oclc: str, date_format: str = '%Y-%m-%d', /) -> dict: + content = request('https://www.worldcat.org/title/' + oclc).content + j = loads(content[ + (s := (f := content.find)(b' type="application/json">') + 25) + :f(b'', s) + ]) + record = j['props']['pageProps']['record'] + if record is None: # invalid OCLC number + raise ReturnError( 'Error processing OCLC number: ' + oclc, - 'Perhaps you entered an invalid OCLC number?', - '') - d = ris_parse(text) - authors = d['authors'] - if authors: - # worldcat has a '.' the end of the first name - d['authors'] = [( - fn.rstrip('.') if not fn.isupper() else fn, - ln.rstrip('.') if not ln.isupper() else ln, - ) for fn, ln in authors] - d['date_format'] = date_format + 'Make sure the OCLC identifier is valid.', + '' + ) + d: defaultdict[str, Any] = defaultdict(lambda: None) + d['cite_type'] = record['generalFormat'].lower() + d['title'] = record['title'] + d['authors'] = [ + ('', c['nonPersonName']['text']) + if 'nonPersonName' in c else + (c["firstName"]['text'], c["secondName"]['text']) + for c in record["contributors"] + ] + d['publisher'] = record['publisher'] + d['publisher-location'] = record['publicationPlace'] + if m := FOUR_DIGIT_NUM(record['publicationDate']): + d['year'] = m[0] + d['language'] = record['catalogingLanguage'] + if isbn := record['isbn13']: + d['isbn'] = isbn d['oclc'] = oclc - d['title'] = d['title'].rstrip('.') - return dict_to_sfn_cit_ref(d) + d['date_format'] = date_format + return d + + +logger = getLogger(__name__) diff --git a/lib/jstor.py b/lib/jstor.py new file mode 100644 index 00000000..9647fad8 --- /dev/null +++ b/lib/jstor.py @@ -0,0 +1,25 @@ +from threading import Thread +from urllib.parse import urlparse + +from lib.commons import request +from lib.bibtex import parse as bibtex_parse + + +def url_to_dict(url: str, date_format: str = '%Y-%m-%d') -> dict: + open_access = [] + thread = Thread(target=is_open_access, args=(url, open_access)) + thread.start() + id_ = urlparse(url).path.rpartition('/')[2] + bibtex = request('https://www.jstor.org/citation/text/' + id_).content.decode('utf8') + dictionary = bibtex_parse(bibtex) + dictionary['jstor'] = id_ + dictionary['date_format'] = date_format + thread.join() + if open_access: + dictionary['jstor-access'] = 'free' + return dictionary + + +def is_open_access(url: str, result: list): + if '"openAccess" : "True"' in request(url, spoof=True).text: + result.append(True) diff --git a/lib/ketabir.py b/lib/ketabir.py new file mode 100644 index 00000000..91d1735b --- /dev/null +++ b/lib/ketabir.py @@ -0,0 +1,97 @@ +"""All things that are specifically related to adinebook website""" + +from collections import defaultdict +from logging import getLogger +from typing import Optional, Any + +from langid import classify +from regex import compile as rc +from requests import RequestException +from bs4 import BeautifulSoup + +from lib.commons import first_last, request + + +AUTHORS_FINDALL = rc(r'(\S+?)\s*+:\s*+(.*)').findall +VOLUME_SEARCH = rc(r'\bجلد (\d+)').search + + +def url_to_dict(url: str, date_format='%Y-%m-%d', /) -> dict: + """Return the response namedtuple.""" + dictionary = _url_to_dict(url) + dictionary['date_format'] = date_format + if 'language' not in dictionary: + # Assume that language is either fa or en. + # Todo: give warning about this assumption? + dictionary['language'] = classify(dictionary['title'])[0] + return dictionary + + +def isbn_to_url(isbn: str) -> Optional[str]: + """Return the ketab.ir book-url for the given isbn.""" + r = request(f'https://msapi.ketab.ir/search/?query={isbn}&limit=1') + j = r.json() + return 'https://ketab.ir/book/' \ + + j['result']['groups']['printableBook']['items'][0]['url'] + + +def _url_to_dict(ketabir_url: str) -> Optional[dict]: + try: + # Try to see if ketabir is available, + # ottobib should continue its work in isbn.py if it is not. + r = request(ketabir_url) + except RequestException: + logger.exception(ketabir_url) + return + + soup = BeautifulSoup(r.content) + d : defaultdict[str, Any] = defaultdict(lambda: None, cite_type='book') + d['title'] = soup.select_one('.card-title').text.strip() + + table = {(tds := tr.select('td'))[0].text: tds[1] for tr in soup.select('tr')} + + # initiating name lists: + others = [] + authors = [] + editors = [] + translators = [] + # building lists: + for span in table['پدیدآور'].select('span'): + role = span.find(text=True).strip(' :\n') + name = span.select_one('a').find(text=True) + name = first_last(name, ' ، ') + if role == 'نويسنده': + authors.append(name) + elif role == 'مترجم': + translators.append(name) + elif role == 'ويراستار': + editors.append(name) + else: + others.append(('', f'{name[0]} {name[1]} ({role})')) + if authors: + d['authors'] = authors + if others: + d['others'] = others + if editors: + d['editors'] = editors + if translators: + d['translators'] = translators + + d['publisher'] = table['ناشر'].find('a').find(text=True).strip() + + if len(date := table['تاریخ نشر'].text.strip()) == 8 and date.isdecimal(): + d['month'] = date[4:6] + d['year'] = date[:4] + + d['isbn'] = table['شابک'].text + + if loc := table['محل نشر'].text.strip(): + d['publisher-location'] = loc + + if m := VOLUME_SEARCH(table['توضیحات'].text): + d['volume'] = m[1] + + return d + + +logger = getLogger(__name__) diff --git a/lib/noorlib.py b/lib/noorlib.py index 774bd316..2d44aa21 100644 --- a/lib/noorlib.py +++ b/lib/noorlib.py @@ -1,44 +1,39 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- - """Codes specifically related to Noormags website.""" -from re import compile as re_compile +from regex import compile as regex_compile -from lib.commons import dict_to_sfn_cit_ref, fetch +from lib.commons import request from lib.bibtex import parse as bibtex_parse -BIBTEX_ARTICLE_ID_SEARCH = re_compile( - r'(?<=CitationHandler\.ashx\?id=)\d+' -).search -RIS_ARTICLE_ID_SEARCH = re_compile(r'(?<=RIS&id=)\d+').search +BIBTEX_ARTICLE_ID_SEARCH = regex_compile( + r'(?<=CitationHandler\.ashx\?id=)\d+').search +RIS_ARTICLE_ID_SEARCH = regex_compile(r'(?<=RIS&id=)\d+').search -def noorlib_sfn_cit_ref(url: str, date_format: str = '%Y-%m-%d') -> tuple: - """Create the response namedtuple.""" - dictionary = bibtex_parse(get_bibtex(url)) +def url_to_dict(url: str, date_format: str = '%Y-%m-%d') -> dict: + dictionary = bibtex_parse(dict_from_bibtex(url)) dictionary['date_format'] = date_format # risr = get_ris(url)[1] # dictionary = risr.parse(ris)[1] - return dict_to_sfn_cit_ref(dictionary) + return dictionary -def get_bibtex(noorlib_url): +def dict_from_bibtex(noorlib_url): """Get bibtex file content from a noormags url. Return as string.""" - pagetext = fetch(noorlib_url).text + pagetext = request(noorlib_url).text article_id = BIBTEX_ARTICLE_ID_SEARCH(pagetext)[0] url = 'http://www.noorlib.ir/View/HttpHandler/CitationHandler.ashx?id=' +\ article_id + '&format=BibTex' - return fetch(url).text + return request(url).text -def get_ris(noorlib_url): +def dict_from_ris(noorlib_url): # This is copied from noormags module (currently not supported but may # be)[1] """Get ris file content from a noormags url. Return as string.""" - pagetext = fetch(noorlib_url).text + pagetext = request(noorlib_url).text article_id = RIS_ARTICLE_ID_SEARCH(pagetext)[0] - url = 'http://www.noormags.com/view/CitationHandler.ashx?format=RIS&id=' +\ + url = 'http://www.noormags.ir/view/CitationHandler.ashx?format=RIS&id=' +\ article_id - return fetch(url).text + return request(url).text diff --git a/lib/noormags.py b/lib/noormags.py index 2ccb0a70..c99888e2 100644 --- a/lib/noormags.py +++ b/lib/noormags.py @@ -1,21 +1,19 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- - """Codes specifically related to Noormags website.""" -from re import compile as re_compile from threading import Thread -from lib.commons import dict_to_sfn_cit_ref, fetch +from regex import compile as regex_compile + +from lib.commons import request from lib.bibtex import parse as bibtex_parse -from lib.ris import parse as ris_parse +from lib.ris import ris_parse -BIBTEX_ARTICLE_ID_SEARCH = re_compile(r'(?<=/citation/bibtex/)\d+').search -RIS_ARTICLE_ID_SEARCH = re_compile(r'(?<=/citation/ris/)\d+').search +BIBTEX_ARTICLE_ID_SEARCH = regex_compile(r'(?<=/citation/bibtex/)\d+').search +RIS_ARTICLE_ID_SEARCH = regex_compile(r'(?<=/citation/ris/)\d+').search -def noormags_sfn_cit_ref(url: str, date_format: str = '%Y-%m-%d') -> tuple: +def url_to_dict(url: str, date_format: str = '%Y-%m-%d') -> dict: """Create the response namedtuple.""" ris_collection = {} ris_thread = Thread(target=ris_fetcher_thread, args=(url, ris_collection)) @@ -24,35 +22,33 @@ def noormags_sfn_cit_ref(url: str, date_format: str = '%Y-%m-%d') -> tuple: dictionary['date_format'] = date_format # language parameter needs to be taken from RIS # other information are more accurate in bibtex - # for example: http://www.noormags.com/view/fa/articlepage/104040 + # for example: http://www.noormags.ir/view/fa/articlepage/104040 # "IS - 1" is wrong in RIS but "number = { 45 }," is correct in bibtex ris_thread.join() dictionary.update(ris_collection) - return dict_to_sfn_cit_ref(dictionary) + return dictionary def get_bibtex(noormags_url): """Get BibTex file content from a noormags_url. Return as string.""" - page_text = fetch(noormags_url).text + page_text = request(noormags_url).text article_id = BIBTEX_ARTICLE_ID_SEARCH(page_text)[0] url = 'http://www.noormags.ir/view/fa/citation/bibtex/' + article_id - return fetch(url).text + return request(url).text def get_ris(noormags_url): """Get ris file content from a noormags url. Return as string.""" - page_text = fetch(noormags_url).text + page_text = request(noormags_url).text article_id = RIS_ARTICLE_ID_SEARCH(page_text)[0] - return fetch( + return request( 'http://www.noormags.ir/view/fa/citation/ris/' + article_id).text def ris_fetcher_thread(url, ris_collection): """Fill the ris_dict. This function is called in a thread.""" ris_dict = ris_parse(get_ris(url)) - language = ris_dict.get('language') - if language: + if language := ris_dict.get('language'): ris_collection['language'] = language - authors = ris_dict.get('authors') - if authors: + if authors := ris_dict.get('authors'): ris_collection['authors'] = authors diff --git a/lib/pubmed.py b/lib/pubmed.py index 158b95e1..6203a811 100644 --- a/lib/pubmed.py +++ b/lib/pubmed.py @@ -1,25 +1,24 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- - """Codes specifically related to PubMed inputs.""" from collections import defaultdict +from typing import Any + from config import NCBI_API_KEY, NCBI_EMAIL, NCBI_TOOL from datetime import datetime -import logging -from re import compile as re_compile +from logging import getLogger from threading import Thread -from lib.commons import dict_to_sfn_cit_ref, b_TO_NUM, fetch +from regex import compile as regex_compile + +from lib.commons import b_TO_NUM, request from lib.doi import get_crossref_dict -NON_DIGITS_SUB = re_compile(r'[^\d]').sub +NON_DIGITS_SUB = regex_compile(r'[^\d]').sub NCBI_URL = ( 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?' 'api_key=' + NCBI_API_KEY + '&retmode=json&tool=' + NCBI_TOOL + '&email=' - + NCBI_EMAIL -) + + NCBI_EMAIL) PUBMED_URL = NCBI_URL + '&db=pubmed&id=' PMC_URL = NCBI_URL + '&db=pmc&id=' @@ -29,29 +28,29 @@ class NCBIError(Exception): pass -def pmid_sfn_cit_ref(pmid: str, date_format='%Y-%m-%d') -> tuple: +def pmid_dict(pmid: str, date_format='%Y-%m-%d', /) -> dict: """Return the response namedtuple.""" pmid = NON_DIGITS_SUB('', pmid) dictionary = ncbi('pmid', pmid) dictionary['date_format'] = date_format - return dict_to_sfn_cit_ref(dictionary) + return dictionary -def pmcid_sfn_cit_ref(pmcid: str, date_format='%Y-%m-%d') -> tuple: +def pmcid_dict(pmcid: str, date_format='%Y-%m-%d', /) -> dict: """Return the response namedtuple.""" pmcid = NON_DIGITS_SUB('', pmcid) dictionary = ncbi('pmcid', pmcid) dictionary['date_format'] = date_format - return dict_to_sfn_cit_ref(dictionary) + return dictionary def ncbi(type_: str, id_: str) -> defaultdict: """Return the NCBI data for the given id_.""" # According to https://www.ncbi.nlm.nih.gov/pmc/tools/get-metadata/ if type_ == 'pmid': - json_response = fetch(PUBMED_URL + id_).json() + json_response = request(PUBMED_URL + id_).json() else: # type_ == 'pmcid' - json_response = fetch(PMC_URL + id_).json() + json_response = request(PMC_URL + id_).json() if 'error' in json_response: # Example error message if rates are exceeded: # {"error":"API rate limit exceeded","count":"11"} @@ -59,18 +58,16 @@ def ncbi(type_: str, id_: str) -> defaultdict: # Return a 503 Service Unavailable raise NCBIError(json_response) result_get = json_response['result'][id_].get - d = defaultdict(lambda: None) + d : defaultdict[str, Any] = defaultdict(lambda: None) doi = None articleids = result_get('articleids', ()) for articleid in articleids: - idtype = articleid['idtype'] - if idtype == 'doi': + if (idtype := articleid['idtype']) == 'doi': doi = articleid['value'] crossref_dict = {} crossref_thread = Thread( - target=crossref_update, args=(crossref_dict, doi) - ) + target=crossref_update, args=(crossref_dict, doi)) crossref_thread.start() d['doi'] = doi elif idtype == 'pmcid': @@ -95,8 +92,7 @@ def ncbi(type_: str, id_: str) -> defaultdict: date = result_get('pubdate') or result_get('epubdate') \ or result_get('printpubdate') date_split = date.split(' ') - date_len = len(date_split) - if date_len == 3: + if (date_len := len(date_split)) == 3: d['date'] = datetime.strptime(date, '%Y %b %d') elif date_len == 2: d['year'], d['month'] = \ @@ -128,8 +124,7 @@ def ncbi(type_: str, id_: str) -> defaultdict: d['page'] = result_get('pages', '').replace('-', '–') - lang = result_get('lang') - if lang: + if (lang := result_get('lang')) is not None: d['language'] = lang[0] if doi: @@ -148,8 +143,7 @@ def crossref_update(dct: dict, doi: str): dct.update(get_crossref_dict(doi)) except Exception: logger.exception( - 'There was an error in resolving crossref DOI: ' + doi - ) + 'There was an error in resolving crossref DOI: ' + doi) -logger = logging.getLogger(__name__) +logger = getLogger(__name__) diff --git a/lib/ris.py b/lib/ris.py index 02be907b..679b15c1 100644 --- a/lib/ris.py +++ b/lib/ris.py @@ -1,12 +1,9 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- - from collections import defaultdict from regex import compile as regex_compile, MULTILINE, VERBOSE from lib.doi import DOI_SEARCH -from lib.commons import first_last, InvalidNameError +from lib.commons import first_last, InvalidNameError, ISBN_10OR13_SEARCH RIS_FULLMATCH = regex_compile( @@ -14,54 +11,63 @@ (?: # this group matches any line ^ (?> - A[U\d]\ {2}-\ (?.++) - |DA\ {2}-\ \d++/(?\d++).*+ - |EP\ {2}-\ (?.++) - |IS\ {2}-\ (?.++) - |J[FA]\ {2}-\ (?.++) - |LA\ {2}-\ (?.++) + A[U\d]\ {2}-\ (?[^\r\n]++) + |DA\ {2}-\ \d++/(?\d++)[^\r\n]*+ + |EP\ {2}-\ (?[^\r\n]++) + |IS\ {2}-\ (?[^\r\n]++) + |J[FA]\ {2}-\ (?[^\r\n]++) + |LA\ {2}-\ (?[^\r\n]++) |P(?> - B\ {2}-\ (?.++) - |Y\ {2}-\ (?\d++).*+ + B\ {2}-\ (?[^\r\n]++) + |Y\ {2}-\ (?\d++)[^\r\n]*+ ) |S(?> - N\ {2}-\ (?\S*+).*+ - |P\ {2}-\ (?.++) + N\ {2}-\ (?\S*+)[^\r\n]*+ + |P\ {2}-\ (?[^\r\n]++) ) |T(?> - [1I]\ {2}-\ (?.++) - |3\ {2}-\ (?<series>.++) - |Y\ {2}-\ (?<type>.++) + [1I]\ {2}-\ (?<title>[^\r\n]++) + |2\ {2}-\ (?<t2>[^\r\n]++) + |3\ {2}-\ (?<series>[^\r\n]++) + |Y\ {2}-\ (?<type>[^\r\n]++) ) - |UR\ {2}-\ (?<url>.++) - |VL\ {2}-\ (?<volume>.++) - |Y1\ {2}-\ (?<year>\d++).*+ + |UR\ {2}-\ (?<url>[^\r\n]++) + |VL\ {2}-\ (?<volume>[^\r\n]++) + |Y1\ {2}-\ (?<year>\d++)[^\r\n]*+ # any other line - |[^\n]*+ + |[^\r\n]*+ ) - \n + \r?\n )* ''', VERBOSE | MULTILINE, ).fullmatch -def parse(ris_text): +def ris_parse(ris_text): """Parse RIS_text data and return the result as a dictionary.""" d = defaultdict(lambda: None) match = RIS_FULLMATCH(ris_text) d.update(match.groupdict()) # cite_type: (book, journal, . . . ) - cite_type = d['type'].lower() + if (cite_type := d['type'].lower()) == 'jour': + if (t2 := d['t2']) is not None: + d['journal'] = t2 url = d['url'] if cite_type == 'elec' and url: d['cite_type'] = 'web' else: d['cite_type'] = cite_type + + if sn := d['sn']: + # determine if it is ISBN or ISSN according to the cite_type + if ISBN_10OR13_SEARCH(sn) is not None: + d['isbn'] = sn + else: + d['issn'] = sn # author: # d['authors'] should not be created unless there are some authors - authors = match.captures('author') - if authors: + if authors := match.captures('author'): # From RIS Format Specifications: # Each author must be on a separate line, preceded by this tag. Each # reference can contain unlimited author fields, and can contain up @@ -77,13 +83,11 @@ def parse(ris_text): continue d['authors'].append(author) # DOIs may be in N1 (notes) tag, search for it in any tag - m = DOI_SEARCH(ris_text) - if m: - d['doi'] = m.group() - start_page = d['start_page'] - if start_page: - end_page = d['end_page'] - if end_page: + if (m := DOI_SEARCH(ris_text)) is not None: + d['doi'] = m[0] + + if start_page := d['start_page']: + if end_page := d['end_page']: d['page'] = start_page + '–' + end_page else: d['page'] = start_page diff --git a/lib/urls.py b/lib/urls.py index 49b2fdbd..3ba1d485 100644 --- a/lib/urls.py +++ b/lib/urls.py @@ -1,12 +1,7 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- - -"""Codes used for parsing contents of an arbitrary URL.""" - - from collections import defaultdict from datetime import date as datetime_date from difflib import get_close_matches +from functools import partial from html import unescape as html_unescape from logging import getLogger from threading import Thread @@ -14,20 +9,19 @@ from urllib.parse import urlparse from langid import classify -from regex import compile as regex_compile, VERBOSE, IGNORECASE +from regex import compile as rc, VERBOSE, IGNORECASE from requests import Response as RequestsResponse from requests.exceptions import RequestException -from lib.commons import ( - find_any_date, dict_to_sfn_cit_ref, ANYDATE_PATTERN, - fetch) +from lib.commons import find_any_date, ANYDATE_PATTERN, request from lib.urls_authors import find_authors, CONTENT_ATTR +from lib.doi import get_crossref_dict -MAX_RESPONSE_LENGTH = 2000000 +MAX_RESPONSE_LENGTH = 10_000_000 # in bytes # https://stackoverflow.com/questions/3458217/how-to-use-regular-expression-to-match-the-charset-string-in-html -CHARSET = regex_compile( +CHARSET = rc( rb''' <meta(?!\s*+(?>name|value)\s*+=)[^>]*?charset\s*+=[\s"']*+([^\s"'/>]*) ''', @@ -39,7 +33,7 @@ (?>citation_title|title|Headline|og:title) (?P=q) ''' -TITLE_SEARCH = regex_compile( +TITLE_SEARCH = rc( r'<meta\s++(?:' + TITLE_META_NAME_OR_PROP + r'\s++' + CONTENT_ATTR + '|' @@ -50,10 +44,10 @@ VERBOSE | IGNORECASE, ).search -TITLE_TAG = regex_compile( +TITLE_TAG = rc( r''' <title\b[^>]*+> - (?P<result>[^<]*+[\s\S]*?) + (?P<result>[^<]++[\s\S]*?) </title\s*+> ''', VERBOSE | IGNORECASE, @@ -72,7 +66,7 @@ ''' DATE_CONTENT_ATTR =\ r'content=(?<q>["\'])[^"\'<]*?' + ANYDATE_PATTERN + r'[^"\'<]*+(?P=q)' -DATE_SEARCH = regex_compile( +DATE_SEARCH = rc( r'<meta\s+[^\n<]*?(?:' + DATE_META_NAME_OR_PROP + r'\s++[^\n<]*?' + DATE_CONTENT_ATTR + '|' @@ -88,7 +82,7 @@ JOURNAL_META_NAME_OR_PROP = r''' (?>name|property)=(?<q>["\'])citation_journal_title(?P=q) ''' -JOURNAL_TITLE_SEARCH = regex_compile( +JOURNAL_TITLE_SEARCH = rc( r'<meta\s++[^\n<]*?(?:' + CONTENT_ATTR + r'\s++[^\n<]*?' + JOURNAL_META_NAME_OR_PROP + '|' @@ -100,7 +94,7 @@ URL_META_NAME_OR_PROP = r''' (?>name|property)=(?<q>["\'])og:url(?P=q) ''' -URL_SEARCH = regex_compile( +URL_SEARCH = rc( r'<meta\s++[^\n<]*?(?:' + CONTENT_ATTR + r'\s++[^\n<]*?' + URL_META_NAME_OR_PROP + '|' @@ -112,7 +106,7 @@ ISSN_META_NAME_OR_PROP = r''' (?>name|property)=(?<q>["\'])citation_issn(?P=q) ''' -ISSN_SEARCH = regex_compile( +ISSN_SEARCH = rc( r'<meta\s++[^\n<]*?(?:' + CONTENT_ATTR + r'\s++[^\n<]*?' + ISSN_META_NAME_OR_PROP + '|' @@ -124,7 +118,7 @@ PMID_META_NAME_OR_PROP = r''' (?>name|property)=(?<q>["\'])citation_pmid(?P=q) ''' -PMID_SEARCH = regex_compile( +PMID_SEARCH = rc( r'<meta\s++[^\n<]*?(?:' + CONTENT_ATTR + r'\s++[^\n<]*?' + PMID_META_NAME_OR_PROP + '|' @@ -136,7 +130,7 @@ DOI_META_NAME_OR_PROP = r''' (?>name|property)=(?<q>["\'])citation_doi(?P=q) ''' -DOI_SEARCH = regex_compile( +DOI_SEARCH = rc( r'<meta\s++[^\n<]*?(?:' + CONTENT_ATTR + r'\s++[^\n<]*?' + DOI_META_NAME_OR_PROP + '|' @@ -149,7 +143,7 @@ VOLUME_META_NAME_OR_PROP = r''' (?>name|property)=(?<q>["\'])citation_volume(?P=q) ''' -VOLUME_SEARCH = regex_compile( +VOLUME_SEARCH = rc( r'<meta\s++[^\n<]*?(?:' + CONTENT_ATTR + r'\s++[^\n<]*?' + VOLUME_META_NAME_OR_PROP + '|' @@ -161,7 +155,7 @@ ISSUE_META_NAME_OR_PROP = r''' (?>name|property)=(?<q>["\'])citation_issue(?P=q) ''' -ISSUE_SEARCH = regex_compile( +ISSUE_SEARCH = rc( r'<meta\s++[^\n<]*?(?:' + CONTENT_ATTR + r'\s++[^\n<]*?' + ISSUE_META_NAME_OR_PROP + '|' @@ -173,7 +167,7 @@ FIRST_PAGE_NAME_OR_PROP = r''' (?>name|property)=(?<q>["\'])citation_firstpage(?P=q) ''' -FIRST_PAGE_SEARCH = regex_compile( +FIRST_PAGE_SEARCH = rc( r'<meta\s++[^\n<]*?(?:' + CONTENT_ATTR + r'\s++[^\n<]*?' + FIRST_PAGE_NAME_OR_PROP + '|' @@ -186,7 +180,7 @@ LAST_PAGE_NAME_OR_PROP = r''' (?>name|property)=(?<q>["\'])citation_lastpage(?P=q) ''' -LAST_PAGE_SEARCH = regex_compile( +LAST_PAGE_SEARCH = rc( r'<meta\s++[^\n<]*?(?:' + CONTENT_ATTR + r'\s++[^\n<]*?' + LAST_PAGE_NAME_OR_PROP + '|' @@ -199,7 +193,7 @@ SITE_NAME_NAME_OR_PROP = r''' (?>name|property)=(?<q>["\'])og:site_name(?P=q) ''' -SITE_NAME_SEARCH = regex_compile( +SITE_NAME_SEARCH = rc( r'<meta\s++[^\n<]*?(?:' + CONTENT_ATTR + r'\s++[^\n<]*?' + SITE_NAME_NAME_OR_PROP + '|' @@ -208,7 +202,8 @@ VERBOSE | IGNORECASE, ).search -TITLE_SPLIT = regex_compile(r' - | — |\|').split +TITLE_SPLIT = rc(r' - | — |\|').split +LANG_SEARCH = rc(r'\slang="([a-z]{2})[-"]').search class ContentTypeError(ValueError): @@ -232,32 +227,29 @@ class StatusCodeError(ValueError): pass -def urls_sfn_cit_ref(url: str, date_format: str = '%Y-%m-%d') -> tuple: +# inaccurate but should be faster than bs4 +# https://stackoverflow.com/questions/14694482/converting-html-to-text-with-python +to_text = partial(rc(r'<[^>]*+>').sub, '') + + +def url_to_dict(url: str, date_format: str = '%Y-%m-%d', /) -> dict: """Create the response namedtuple.""" - try: - dictionary = url2dict(url) - except (ContentTypeError, ContentLengthError) as e: - logger.exception(url) - # Todo: i18n - return 'Could not process the fetch.', e, '' + dictionary = url2dict(url) dictionary['date_format'] = date_format - return dict_to_sfn_cit_ref(dictionary) + return dictionary def find_journal(html: str) -> Optional[str]: """Return journal title as a string.""" # http://socialhistory.ihcs.ac.ir/article_319_84.html - m = JOURNAL_TITLE_SEARCH(html) - if m: - return m.group('result') + if (m := JOURNAL_TITLE_SEARCH(html)) is not None: + return m['result'] def find_url(html: str, url: str) -> str: """Return og:url or url as a string.""" - # http://www.ft.com/cms/s/836f1b0e-f07c-11e3-b112-00144feabdc0,Authorised=false.html?_i_location=http%3A%2F%2Fwww.ft.com%2Fcms%2Fs%2F0%2F836f1b0e-f07c-11e3-b112-00144feabdc0.html%3Fsiteedition%3Duk&siteedition=uk&_i_referer=http%3A%2F%2Fwww.ft.com%2Fhome%2Fuk - m = URL_SEARCH(html) - if m: - ogurl = m.group('result') + if (m := URL_SEARCH(html)) is not None: + ogurl = m['result'] if urlparse(ogurl).path: return ogurl return url @@ -269,54 +261,40 @@ def find_issn(html: str) -> Optional[str]: Normally ISSN should be in the '\d{4}\-\d{3}[\dX]' format, but this function does not check that. """ - m = ISSN_SEARCH(html) - # http://socialhistory.ihcs.ac.ir/article_319_84.html - # http://psycnet.apa.org/journals/edu/30/9/641/ - if m: - return m.group('result') + if (m := ISSN_SEARCH(html)) is not None: + return m['result'] def find_pmid(html: str) -> Optional[str]: """Return pmid as a string.""" - # http://jn.physiology.org/content/81/1/319 - m = PMID_SEARCH(html) - if m: - return m.group('result') + if (m := PMID_SEARCH(html)) is not None: + return m['result'] def find_doi(html: str) -> Optional[str]: """Return DOI as a string.""" - # http://jn.physiology.org/content/81/1/319 - m = DOI_SEARCH(html) - if m: - return m.group('result') + if (m := DOI_SEARCH(html)) is not None: + return m['result'] def find_volume(html: str) -> Optional[str]: """Return citatoin volume number as a string.""" - # http://socialhistory.ihcs.ac.ir/article_319_84.html - m = VOLUME_SEARCH(html) - if m: - return m.group('result') + if (m := VOLUME_SEARCH(html)) is not None: + return m['result'] def find_issue(html: str) -> Optional[str]: """Return citation issue number as a string.""" - # http://socialhistory.ihcs.ac.ir/article_319_84.html - m = ISSUE_SEARCH(html) - if m: - return m.group('result') + if (m := ISSUE_SEARCH(html)) is not None: + return m['result'] def find_pages(html: str) -> Optional[str]: """Return citation pages as a string.""" # http://socialhistory.ihcs.ac.ir/article_319_84.html - fp_match = FIRST_PAGE_SEARCH(html) - if fp_match: - lp_match = LAST_PAGE_SEARCH(html) - if lp_match: - return \ - fp_match.group('result') + '–' + lp_match.group('result') + if fp_match := FIRST_PAGE_SEARCH(html): + if lp_match := LAST_PAGE_SEARCH(html): + return fp_match['result'] + '–' + lp_match['result'] def find_site_name( @@ -324,7 +302,7 @@ def find_site_name( html_title: str, url: str, authors: List[Tuple[str, str]], - home_title: List[str], + home_list: List[str], thread: Thread, ) -> str: """Return (site's name as a string, where). @@ -334,39 +312,36 @@ def find_site_name( html_title: Title of the page found in the title tag of the html. url: URL of the page. authors: Authors list returned from find_authors function. - home_title: A list containing the title of the home page as a str. + home_list: A list containing the title of the home page as a str. thread: The thread that should be joined before using home_title list. Returns site's name as a string. """ - m = SITE_NAME_SEARCH(html) - if m: - return m.group('result') + if (m := SITE_NAME_SEARCH(html)) is not None: + return m['result'] # search the title - site_name = parse_title( - html_title, url, authors, home_title, thread - )[2] - if site_name: - return site_name + if html_title is not None: + if site_name := parse_title( + html_title, url, authors, home_list, thread + )[2]: + return site_name # noinspection PyBroadException try: # using home_title thread.join() - if ':' in home_title[0]: - # http://www.washingtonpost.com/wp-dyn/content/article/2005/09/02/AR2005090200822.html - site_name = home_title[0].split(':')[0].strip() - if site_name: + home_site_name, home_title = home_list + if home_site_name is not None: + return home_site_name + if (i := home_title.find(':')) != -1: + if site_name := home_title[:i].strip(): return site_name - site_name = parse_title(home_title[0], url, None)[2] - if site_name: + if site_name := parse_title(home_title, url, None)[2]: return site_name - return home_title[0] + return home_title except Exception: logger.exception(url) # return hostname hostname = urlparse(url).hostname - if hostname.startswith('www.'): - return hostname[4:] - return hostname + return hostname.removeprefix('www.') def find_title( @@ -374,17 +349,16 @@ def find_title( html_title: str, url: str, authors: List[Tuple[str, str]], - home_title: List[str], + home_list: List[str], thread: Thread, ) -> Optional[str]: """Return (title_string, where_info).""" - m = TITLE_SEARCH(html) - if m: + if (m := TITLE_SEARCH(html)) is not None: return parse_title( - html_unescape(m.group('result')), url, authors, home_title, thread, + html_unescape(m['result']), url, authors, home_list, thread, )[1] - elif html_title: - return parse_title(html_title, url, authors, home_title, thread)[1] + elif html_title is not None: + return parse_title(html_title, url, authors, home_list, thread)[1] else: return None @@ -393,7 +367,7 @@ def parse_title( title: str, url: str, authors: Optional[List[Tuple[str, str]]], - home_title_list: Optional[List[str]] = None, + home_list: Optional[List[str]] = None, thread: Thread = None, ) -> Tuple[Optional[str], str, Optional[str]]: """Return (intitle_author, pure_title, intitle_sitename). @@ -425,7 +399,7 @@ def parse_title( title_parts = TITLE_SPLIT(title.strip()) if len(title_parts) == 1: return None, title, None - hostname = urlparse(url).hostname.replace('www.', '') + hostname = urlparse(url).hostname.replace('www.', '', 1) # Searching for intitle_sitename # 1. In hostname hnset = set(hostname.split('.')) @@ -436,16 +410,15 @@ def parse_title( else: # 2. Using difflib on hostname # Cutoff = 0.3: 'BBC - Homepage' will match u'‭BBC ‮فارسی‬' - close_matches = get_close_matches( + if close_matches := get_close_matches( hostname, title_parts, n=1, cutoff=.3 - ) - if close_matches: + ): intitle_sitename = close_matches[0] else: - if thread: + if thread is not None: thread.join() - if home_title_list: - home_title = home_title_list[0] + if home_list: + home_site_name, home_title = home_list # 3. In homepage title for part in title_parts: if part in home_title: @@ -453,9 +426,9 @@ def parse_title( break else: # 4. Using difflib on home_title - close_matches = get_close_matches( - home_title, title_parts, n=1, cutoff=.3) - if close_matches: + if close_matches := get_close_matches( + home_title, title_parts, n=1, cutoff=.3 + ): intitle_sitename = close_matches[0] # Remove sitename from title_parts if intitle_sitename: @@ -482,19 +455,19 @@ def find_date(html: str, url: str) -> datetime_date: # http://ftalphaville.ft.com/2012/05/16/1002861/recap-and-tranche-primer/?Authorised=false # Example for find_any_date(html): # https://www.bbc.com/news/uk-england-25462900 - m = DATE_SEARCH(html) - return find_any_date(m) if m else find_any_date(url) or find_any_date(html) + if (m := DATE_SEARCH(html)) is not None: + return find_any_date(m) + return find_any_date(url) or find_any_date(html) -def get_home_title(url: str, home_title_list: List[str]) -> None: - """Get homepage of the url and return it's title. +def analyze_home(url: str, home_list: list) -> None: + """Append home_title and site_name to home_list. - home_title_list will be used to return the thread result. This function is invoked through a thread. + home_list is used to return the thread result. """ - # Todo: cache the result. home_url = '://'.join(urlparse(url)[:2]) - with fetch( + with request( home_url, spoof=True, stream=True ) as r: try: @@ -505,11 +478,18 @@ def get_home_title(url: str, home_title_list: List[str]) -> None: ): return content = next(r.iter_content(MAX_RESPONSE_LENGTH)) + m = CHARSET(content) - html = content.decode(m.group(1).decode() if m else r.encoding) + html = content.decode(m[1].decode() if m else r.encoding) + + if m := SITE_NAME_SEARCH(html): + home_list.append(m['result']) + else: + home_list.append(None) + m = TITLE_TAG(html) - title = html_unescape(m.group('result')) if m else None - home_title_list.append(title) + title = html_unescape(m['result']) if m else None + home_list.append(title) def check_response_headers(r: RequestsResponse) -> None: @@ -527,8 +507,7 @@ def check_response_headers(r: RequestsResponse) -> None: raise ContentLengthError( 'Content-length was too long. ' '({mb:.2f} MB)'.format(mb=bytes_length / 1000000)) - content_type = response_headers.get('content-type') - if content_type: + if content_type := response_headers.get('content-type'): if content_type.startswith('text/'): return raise ContentTypeError( @@ -539,38 +518,55 @@ def check_response_headers(r: RequestsResponse) -> None: def get_html(url: str) -> str: """Return the html string for the given url.""" - with fetch( + with request( url, stream=True, spoof=True ) as r: check_response_headers(r) - content = next(r.iter_content(MAX_RESPONSE_LENGTH)) + size = 0 + chunks = [] + a = chunks.append + for chunk in r.iter_content(MAX_RESPONSE_LENGTH): + size += len(chunk) + if size >= MAX_RESPONSE_LENGTH: + raise ValueError( + 'response was too large: ' + f'{size=} > {MAX_RESPONSE_LENGTH=}') + a(chunk) + content = b''.join(chunks) charset_match = CHARSET(content) return content.decode( - charset_match.group(1).decode() if charset_match else r.encoding) + charset_match[1].decode() if charset_match else r.encoding) def url2dict(url: str) -> Dict[str, Any]: """Get url and return the result as a dictionary.""" - d = defaultdict(lambda: None) - # Creating a thread to fetch homepage title in background - home_title_list = [] # A mutable variable used to get the thread result - home_title_thread = Thread( - target=get_home_title, args=(url, home_title_list)) - home_title_thread.start() + d: defaultdict[str, Any] = defaultdict(lambda: None) + # Creating a thread to request homepage title in background + home_thread = Thread( + target=analyze_home, args=(url, (home_list := []))) + home_thread.start() html = get_html(url) + + if doi := find_doi(html): + # noinspection PyBroadException + try: + return get_crossref_dict(doi) + except Exception: + logger.exception(f'{url=}, {doi=}') + d['doi'] = doi + d['url'] = find_url(html, url) - m = TITLE_TAG(html) - html_title = html_unescape(m.group('result')) if m else None - if html_title: - d['html_title'] = html_title + if m := TITLE_TAG(html): + if html_title := html_unescape(m['result']): + d['html_title'] = html_title + else: + html_title = None # d['html_title'] is used in waybackmechine.py. - authors = find_authors(html) - if authors: + if authors := find_authors(html): d['authors'] = authors d['issn'] = find_issn(html) d['pmid'] = find_pmid(html) - d['doi'] = find_doi(html) d['volume'] = find_volume(html) d['issue'] = find_issue(html) d['page'] = find_pages(html) @@ -580,14 +576,20 @@ def url2dict(url: str) -> Dict[str, Any]: else: d['cite_type'] = 'web' d['website'] = find_site_name( - html, html_title, url, authors, home_title_list, home_title_thread) - d['title'] = find_title( - html, html_title, url, authors, home_title_list, home_title_thread) - date = find_date(html, url) - if date: + html, html_title, url, authors, home_list, home_thread) + if (title := find_title( + html, html_title, url, authors, home_list, home_thread + )) is not None: + d['title'] = title.strip() + if date := find_date(html, url): d['date'] = date d['year'] = str(date.year) - d['language'] = classify(html)[0] + + if (lang_match := LANG_SEARCH(html)) is not None: + d['language'] = lang_match[1] + else: + d['language'] = classify(html)[0] + return d diff --git a/lib/urls_authors.py b/lib/urls_authors.py index eb7d56fe..f98da9ed 100644 --- a/lib/urls_authors.py +++ b/lib/urls_authors.py @@ -1,17 +1,9 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- - -"""This module is used for finding the authors in a soup object. - -It is in urls.py. -""" - - from typing import List, Optional, Tuple from regex import compile as regex_compile, VERBOSE, IGNORECASE, ASCII -from lib.commons import ANYDATE_SEARCH, first_last, InvalidNameError +from lib.commons import ANYDATE_SEARCH, first_last, InvalidNameError, \ + FOUR_DIGIT_NUM # Names in byline are required to be two or three parts @@ -19,7 +11,7 @@ # BYLINE_PATTERN supports up to four names in a byline # names may be separated with "and", a "comma" or "comma and" -BYLINE_PATTERN = r''' +BYLINE_PATTERN = rf''' \s*+By\s++{NAME_PATTERN}( ,\ {NAME_PATTERN}( ,\ {NAME_PATTERN}( @@ -49,7 +41,7 @@ )? )? )?\s* -'''.format_map(locals()) +''' BYLINE_PATTERN_SEARCH = regex_compile(BYLINE_PATTERN, VERBOSE | IGNORECASE) NORMALIZE_ANDS = regex_compile(r'\s++and\s++', IGNORECASE).sub @@ -85,7 +77,7 @@ )(?P=q) ''' AUTHOR_META_NAME_OR_PROP = r''' - (?<id>(?:name|property)\s*+=\s*+(?<q>["\']) + (?<id>(?:name|property)\s*+=\s*+(?<q>["\']?) (?> # http://socialhistory.ihcs.ac.ir/article_571_84.html # http://jn.physiology.org/content/81/1/319 @@ -96,13 +88,13 @@ (?P=q)) ''' META_AUTHOR_FINDITER = regex_compile( - r''' + rf''' <meta\s[^>]*?(?: {AUTHOR_META_NAME_OR_PROP}\s[^c]*+[^>]*?{CONTENT_ATTR} | {CONTENT_ATTR}\s[^>]*?{AUTHOR_META_NAME_OR_PROP} ) - '''.format_map(locals()), + ''', VERBOSE | IGNORECASE ).finditer # id=byline @@ -118,9 +110,9 @@ <(?<tag>[a-z]\w++)\s++[^>]*? (?<id> (?>class|id|rel)= - (?<q>["\']) + (?<q>["\']?) (?> - author(?>_byline|Inline|-title)? + author(?>_byline|Inline|-title|s)? |by(?> line(?>Author|line-name)? |_line(?:_date)? @@ -129,7 +121,7 @@ |story-byline ) ) - (?P=q)[^>]*+> + \b(?P=q)[^>]*+> (?<result>[^<]*+[\s\S]*?) </(?P=tag)[^>]*+> | @@ -137,13 +129,12 @@ (?<id>authorName["\']?\s*+:\s*+["\'])(?<result>[^"\'>\n]++)["\'] | # schema.org - (?<q>["'])author(?P=q)\s*+:\s*+{\s*+(?P=q)@type(?P=q)\s*+:\s*+(?P=q) + (?<q>["'])author(?P=q)\s*+:\s*+\[?{\s*+(?P=q)@type(?P=q)\s*+:\s*+(?P=q) (?<id>Person) (?P=q)\s*+,\s*+(?P=q)name(?P=q)\s*+:\s*+(?P=q)(?<result>[^"']*+)(?P=q) ) ''', - VERBOSE | IGNORECASE | ASCII, -).finditer + VERBOSE | IGNORECASE | ASCII).finditer BYLINE_HTML_PATTERN = regex_compile( @@ -160,7 +151,7 @@ # http://www.businessnewsdaily.com/6762-male-female-entrepreneurs.html?cmpid=514642_20140715_27858876 # .byline > .author BYLINE_AUTHOR = regex_compile( - r'<[a-z]++[^c]*+[^>]*?class=(?<q>["\'])author(?P=q)' + r'<[a-z]++[^c]*+[^>]*?class=(?<q>["\']?)author\b(?P=q)' r'[^>]*+>(?<result>[^<>]++)', IGNORECASE | ASCII ).finditer @@ -186,36 +177,39 @@ IGNORECASE | VERBOSE, ).search -FOUR_DIGIT_NUM = regex_compile(r'\d\d\d\d').search - def find_authors(html) -> Optional[List[Tuple[str, str]]]: """Return authors names found in html.""" names = [] match_id = None for match in META_AUTHOR_FINDITER(html): - if match_id and match_id != match.group('id'): + if match_id and match_id != match['id']: break - name = byline_to_names(match.group('result')) - if name: + if (name := byline_to_names(match['result'])) is not None: names.extend(name) - match_id = match.group('id') + match_id = match['id'] if names: return names match_id = None + results = set() for match in BYLINE_TAG_FINDITER(html): # Only match authors using the same search criteria. - if match_id and match_id != match.group('id'): + if match_id is not None and match_id != match['id']: break - if match.group('tag'): - tag_text = TAGS_SUB('', match.group('result')) + result = match['result'] + if result in results: + break # avoid duplicate results + results.add(result) + if match['tag']: + results.add(result) + tag_text = TAGS_SUB('', result) ns = byline_to_names(tag_text) if ns: - match_id = match.group('id') + match_id = match['id'] names.extend(ns) continue - for m in BYLINE_AUTHOR(match.group('result')): - author = m.group('result') + for m in BYLINE_AUTHOR(result): + author = m['result'] ns = byline_to_names(author) if ns: names.extend(ns) @@ -223,15 +217,14 @@ def find_authors(html) -> Optional[List[Tuple[str, str]]]: return names else: # not containing tags. - ns = byline_to_names(match.group('result')) - if ns: - match_id = match.group('id') + ns = byline_to_names(result) + if ns is not None: + match_id = match['id'] names.extend(ns) if names: return names - match = BYLINE_TEXT_PATTERN(TAGS_SUB('', html)) - if match: - return byline_to_names(match.group()) + if (match := BYLINE_TEXT_PATTERN(TAGS_SUB('', html))) is not None: + return byline_to_names(match[0]) return None @@ -255,16 +248,15 @@ def byline_to_names(byline) -> Optional[List[Tuple[str, str]]]: ... ) [RawName("Erika Solomon"), RawName("Borzou Daragahi")] """ - byline = byline.partition('|')[0] - if ':' in byline or ':' in byline: + byline = byline.partition('|')[0].strip(' ;\t\n') + if ':' in byline: return None - m = ANYDATE_SEARCH(byline) - if m: + if (m := ANYDATE_SEARCH(byline)) is not None: # Removing the date part byline = byline[:m.start()] if not byline: return None - if FOUR_DIGIT_NUM(byline): + if FOUR_DIGIT_NUM(byline) is not None: return None # Normalize 'and\n' (and the similar) to standard 'and ' # This should be done before cutting the byline at the first newline @@ -282,7 +274,7 @@ def byline_to_names(byline) -> Optional[List[Tuple[str, str]]]: names = [] for fullname in fullnames: fullname = fullname.partition(' in ')[0].partition(' for ')[0] - if STOPWORDS_SEARCH(fullname): + if STOPWORDS_SEARCH(fullname) or fullname.isupper(): continue try: first, last = first_last(fullname) @@ -292,7 +284,7 @@ def byline_to_names(byline) -> Optional[List[Tuple[str, str]]]: first.startswith(('The ', 'خبرگزار')) or last.islower() ): - first, last = '', first + ' ' + last + first, last = '', f'{first} {last}' names.append((first, last)) if not names: return None diff --git a/lib/waybackmachine.py b/lib/waybackmachine.py index cce874fe..ee5090c6 100644 --- a/lib/waybackmachine.py +++ b/lib/waybackmachine.py @@ -1,6 +1,3 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- - """Define related tools for web.archive.org (aka Wayback Machine).""" import logging @@ -11,9 +8,8 @@ from regex import compile as regex_compile from requests import ConnectionError as RequestsConnectionError -from lib.commons import dict_to_sfn_cit_ref from lib.urls import ( - urls_sfn_cit_ref, url2dict, get_home_title, get_html, find_authors, + url_to_dict as urls_url_to_dict, url2dict, analyze_home, get_html, find_authors, find_journal, find_site_name, find_title, ContentTypeError, ContentLengthError, StatusCodeError, TITLE_TAG ) @@ -25,14 +21,13 @@ ).fullmatch -def waybackmachine_sfn_cit_ref( +def url_to_dict( archive_url: str, date_format: str = '%Y-%m-%d' -) -> tuple: +) -> dict: """Create the response namedtuple.""" - m = URL_FULLMATCH(archive_url) - if not m: + if (m := URL_FULLMATCH(archive_url)) is None: # Could not parse the archive_url. Treat as an ordinary URL. - return urls_sfn_cit_ref(archive_url, date_format) + return urls_url_to_dict(archive_url, date_format) archive_year, archive_month, archive_day, original_url = \ m.groups() original_dict = {} @@ -40,12 +35,7 @@ def waybackmachine_sfn_cit_ref( target=original_url2dict, args=(original_url, original_dict) ) thread.start() - try: - archive_dict = url2dict(archive_url) - except (ContentTypeError, ContentLengthError) as e: - logger.exception(archive_url) - # Todo: i18n - return 'Invalid content type or length.', e, '' + archive_dict = url2dict(archive_url) archive_dict['date_format'] = date_format archive_dict['url'] = original_url archive_dict['archive-url'] = archive_url @@ -60,19 +50,19 @@ def waybackmachine_sfn_cit_ref( or original_dict['html_title'] == archive_dict['html_title'] ): archive_dict.update(original_dict) - archive_dict['dead-url'] = 'no' + archive_dict['url-status'] = 'live' else: # and original title is the same as archive title. Otherwise it # means that the content probably has changed and the original data # cannot be trusted. - archive_dict['dead-url'] = 'unfit' + archive_dict['url-status'] = 'unfit' else: - archive_dict['dead-url'] = 'yes' + archive_dict['url-status'] = 'dead' if archive_dict['website'] == 'Wayback Machine': archive_dict['website'] = ( urlparse(original_url).hostname.replace('www.', '') ) - return dict_to_sfn_cit_ref(archive_dict) + return archive_dict def original_url2dict(ogurl: str, original_dict) -> None: @@ -96,22 +86,24 @@ def original_url2dict(ogurl: str, original_dict) -> None: def original_url_dict(url: str): """Retuan dictionary only containing required data for og:url.""" d = {} - # Creating a thread to fetch homepage title in background + # Creating a thread to request homepage title in background hometitle_list = [] # A mutable variable used to get the thread result home_title_thread = Thread( - target=get_home_title, args=(url, hometitle_list) + target=analyze_home, args=(url, hometitle_list) ) home_title_thread.start() html = get_html(url) - m = TITLE_TAG(html) - html_title = m.group('result') if m else None - if html_title: - d['html_title'] = html_title - authors = find_authors(html) - if authors: + + if (m := TITLE_TAG(html)) is not None: + if html_title := m['result']: + d['html_title'] = html_title + else: + html_title = None + + if authors := find_authors(html): d['authors'] = authors - journal = find_journal(html) - if journal: + + if journal := find_journal(html): d['journal'] = journal d['cite_type'] = 'journal' else: diff --git a/requirements.txt b/requirements.txt index cedb36ac..2140ec5c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ -requests isbnlib -langid jdatetime +langid +bs4 regex -flup6 -typing ; python_version < '3.5' \ No newline at end of file +requests diff --git a/test/__init__.py b/test/__init__.py index fdeb90c9..6e9d9667 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -1,50 +1,168 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- -from atexit import register as atexit_register -from pickle import dump, load +import atexit +from contextlib import contextmanager +from hashlib import sha1 +from json import dump, loads, load +from typing import Optional +from functools import partial -from requests import Session +# noinspection PyPackageRequirements +from path import Path +from requests import Session, Response, ConnectionError as RConnectionError +# noinspection PyPackageRequirements +from environs import Env -# Do not import library parts here. The config should not be initialized + +# Do not import library parts here. commons.py should not be loaded # until LANG is set by test_fa and test_en. -FORCE_CACHE_OVERWRITE = False # Use for updating cache entries -NEW_DOWNLOAD = False +env = Env() +env.read_env() +# Use for updating cache entries +FORCE_OVERWRITE_TESTDATA = env.bool('FORCE_OVERWRITE_TESTDATA', False) +READONLY_TESTDATA = env.bool('READONLY_TESTDATA', True) +REMOVE_UNUSED_TESTDATA = env.bool('REMOVE_UNUSED_TESTDATA', False) +TESTDATA = Path(__file__).parent / 'testdata' -def fake_request(method, url, **kwargs): - assert method == 'get' - response = cache.get(url) - if FORCE_CACHE_OVERWRITE or response is None: - print('Downloading ' + url) - response = Session().request(method, url, **kwargs) - cache[url] = response - global NEW_DOWNLOAD - NEW_DOWNLOAD = True - return response +json_dump = partial( + dump, ensure_ascii=False, check_circular=False, sort_keys=True, + indent='\t') + + +class FakeResponse: + + # todo: remove mechanicalsoup stuff + __slots__ = ( + 'content', 'iter_content', 'status_code', 'headers', 'encoding', 'url', + 'soup' # required by mechanicalsoup + ) + + request = None # required by mechanicalsoup + + def __init__( + self, url: str, content: bytes, status_code: int, headers: dict, + encoding: str + ): + self.url = url + self.content = content + self.status_code = status_code + self.encoding = encoding + self.headers = headers + def json(self): + return loads(self.content) -def save_cache(cache_dict): - """Save cache as pickle.""" - if not NEW_DOWNLOAD: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): return - print('saving new cache') - with open(__file__ + '/../.tests_cache', 'w+b') as f: - dump(cache_dict, f) + + @property + def text(self): + return self.content.decode(self.encoding) -def load_cache(): - """Return cache as a dict.""" +def load_response(hsh: str) -> Optional[FakeResponse]: + filename = f'{hsh}.json' try: - with open(__file__ + '/../.tests_cache', 'r+b') as f: - return load(f) + with open(f'{TESTDATA}/{filename}', 'rb') as f: + d = load(f) except FileNotFoundError: - return {} + return None + + if REMOVE_UNUSED_TESTDATA is True: + USED_TESTDATA.add(filename) + + if 'raise' in d: + raise RConnectionError('per json data') + + filename = f'{hsh}.html' + with open(f'{TESTDATA}/{filename}', 'rb') as f: + content = f.read() + + if REMOVE_UNUSED_TESTDATA is True: + USED_TESTDATA.add(filename) + + return FakeResponse( + d['url'], content, d['status_code'], d['headers'], d['encoding']) + + +def dump_response(hsh, response: Response) -> None: + d = { + 'status_code': response.status_code, + # CaseInsensitiveDict is not JSON serializable + 'headers': {**response.headers}, + 'encoding': response.encoding, + 'url': response.url} + with open(f'{TESTDATA}/{hsh}.json', 'w') as f: + json_dump(d, f) + with open(f'{TESTDATA}/{hsh}.html', 'wb') as f: + f.write(response.content) + + +def dump_connection_error(hsh): + with open(f'{TESTDATA}/{hsh}.json', 'w') as f: + json_dump({'raise': True}, f) + + +# noinspection PyDecorator +@staticmethod +def fake_request(method, url, data=None, stream=False, **kwargs): + if data: + cache_key = url + repr(sorted(data)) + else: + cache_key = url + sha1_hex = sha1(cache_key.encode()).hexdigest() + + if FORCE_OVERWRITE_TESTDATA is True: + response = None + else: + response = load_response(sha1_hex) + + if response is None: # either FileNotFoundError or FORCE_CACHE_OVERWRITE + if READONLY_TESTDATA: + raise RuntimeError( + f'testdata file not found. ' + f'{READONLY_TESTDATA=} {FORCE_OVERWRITE_TESTDATA=}') + print('Downloading ' + url) + with real_request(): + try: + response = Session().request( + method, url, data=data, **kwargs) + except RConnectionError: + dump_connection_error(sha1_hex) + dump_response(sha1_hex, response) + + if stream is True: + def iter_content(*_): + # this closure over response will simulate a bound method + return iter((response.content,)) + response.iter_content = iter_content + + return response + + +@contextmanager +def real_request(): + Session.request = original_request + yield + Session.request = fake_request + + +original_request = Session.request +Session.request = fake_request + +if REMOVE_UNUSED_TESTDATA is True: + all_testdata_files = {f.name for f in TESTDATA.files()} + USED_TESTDATA = {*()} -Session.request = staticmethod(fake_request) + def rm_unused_files(): + unused_files = (all_testdata_files - USED_TESTDATA) + for f in unused_files: + (TESTDATA / f).remove() + print(f'removed {len(all_testdata_files - USED_TESTDATA)} unused testdata files') -cache = load_cache() -print('len(cache) ==', len(cache)) -atexit_register(save_cache, cache) + atexit.register(rm_unused_files) diff --git a/test/adinebook_test.py b/test/adinebook_test.py deleted file mode 100644 index 47b54e07..00000000 --- a/test/adinebook_test.py +++ /dev/null @@ -1,166 +0,0 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- - -"""Test adinebook.py module.""" - - -from unittest import TestCase, main - -from lib.adinebook import adinehbook_sfn_cit_ref - - -class AdineBookTest(TestCase): - - def test_ab1(self): - """authors = 1, translators = 2, otheo = 1, isbn13""" - self.assertEqual( - '* {{cite book ' - '| last=لانسکی ' - '| first=ویکی ' - '| others= کی وایت (تصویرگر), فیروزه دالکی (مترجم)' - ', and مژگان امیرفروغی (مترجم) ' - '| title=101 راه برای اینکه پدر بهتری باشید ' - '| publisher=پیک ادبیات ' - '| year=1386 ' - '| isbn=978-964-8165-81-4 ' - '| language=fa ' - '| ref=harv}}', - adinehbook_sfn_cit_ref( - 'http://www.adinebook.com/gp/product/9648165814/' - 'ref=sr_1_1000_42/905-6618179-9188955' - )[1] - ) - - def test_ab2(self): - """authors = 3, translators = 2, otheo = 0, isbn13""" - i = ( - 'http://www.adinebook.com/gp/product/9642823352/' - 'ref=sr_1_1000_41/905-6618179-9188955' - ) - o = adinehbook_sfn_cit_ref(i) - e = ( - '* {{cite book ' - '| last=کرسول ' - '| first=جان ' - '| last2=کلارک ' - '| first2=ویکی پلانو ' - '| others=محسن نیازی (مترجم), and عباس زارعی (مترجم) ' - '| title=روش های تحقیق تلفیقی ' - '| publisher=ثامن الحجج ' - '| year=1387 ' - '| isbn=978-964-2823-35-2 ' - '| language=fa ' - '| ref=harv' - ) - self.assertIn(e, o[1]) - - def test_ab3(self): - """authors = 2, translators = 0, otheo = 4, isbn13""" - i = 'http://www.adinebook.com/gp/product/6005883435' - o = adinehbook_sfn_cit_ref(i) - e = ( - '* {{cite book ' - '| last=فخررحیمی ' - '| first=علیرضا ' - '| last2=فخررحیمی ' - '| first2=الهام ' - '| others= آرش نادرپور (مقدمه), امیر جابری (مقدمه)' - ', وحید شهبازیان (مقدمه), and رضا مقدم (مقدمه) ' - '| title=آموزش گام به گام پیکربندی مسیریابهای میکروتیک' - ': آمادگی آزمون MTCNA ' - '| publisher=نشرگستر ' - '| year=1391 ' - '| isbn=978-600-5883-43-5 ' - '| language=fa ' - '| ref=harv' - ) - self.assertIn(e, o[1]) - - def test_ab4(self): - """authors = 3, translators = 0, otheo = 0, isbn13""" - i = ( - 'http://www.adinebook.com/gp/product/9649563342/' - 'ref=ftr_1/905-6618179-9188955' - ) - o = adinehbook_sfn_cit_ref(i) - e = ( - '* {{cite book ' - '| last=کریمی ' - '| first=نجمه ' - '| last2=یزدخواستی ' - '| first2=فروغ ' - '| last3=مختاری ' - '| first3=صفورا ' - '| title=11 سپتامبر ... آرماگدون ' - '| publisher=حدیث راه عشق ' - '| year=1386 ' - '| isbn=978-964-95633-4-3 ' - '| language=fa ' - '| ref=harv' - ) - self.assertIn(e, o[1]) - - def test_ab5(self): - """Year is interesting here.""" - i = 'http://www.adinebook.com/gp/product/9642656349/' - o = adinehbook_sfn_cit_ref(i) - e = ( - '* {{cite book ' - '| last=نژاد ' - '| first=یوسف علی یوسف ' - '| title=فراهنجاری در مثنوی سرایی ' - '| publisher=اردیبهشت ' - '| year=1388 ' - '| isbn=978-964-2656-34-9 ' - '| language=fa ' - '| ref=harv' - ) - self.assertIn(e, o[1]) - - def test_ab6(self): - """Month and year detection.""" - i = ( - 'http://www.adinebook.com/gp/product/9645300363/' - 'ref=pd_sim_b_title_4/905-6618179-9188955' - ) - o = adinehbook_sfn_cit_ref(i) - e = ( - '* {{cite book ' - '| last=مونس ' - '| first=حسین ' - '| others=حمیدرضا شیخی (مترجم) ' - '| title=تاریخ و تمدن مغرب - جلد اول ' - '| publisher=سازمان مطالعه و تدوین کتب علوم انسانی دانشگاهها' - ' (سمت) ' - '| year=1392 ' - '| isbn=978-964-530-036-2 ' - '| language=fa ' - '| ref=harv' - ) - self.assertIn(e, o[1]) - - def test_ab7(self): - """1 Editor.""" - self.assertIn( - '* {{cite book ' - '| last=دیماتیو ' - '| first=ام.رابین ' - '| editor-last=جباری ' - '| editor-first=کریم ' - '| others= کیانوش هاشمیان (زیرنظر), and محمد کاویانی (مترجم) ' - '| title=روانشناسی سلامت به ضمیمه نگرشی بر منابع اسلامی - جلد اول ' - '| publisher=سازمان مطالعه و تدوین کتب علوم انسانی دانشگاهها' - ' (سمت) ' - '| year=1392 ' - '| isbn=978-964-459-398-7 ' - '| language=fa ' - '| ref=harv', - adinehbook_sfn_cit_ref( - 'http://www.adinebook.com/gp/product/9644593987/' - 'ref=pd_pos_b_title_4/905-6618179-9188955' - )[1], - ) - - -if __name__ == '__main__': - main() diff --git a/test/doi_test.py b/test/doi_test.py index ad1134ff..c234be57 100644 --- a/test/doi_test.py +++ b/test/doi_test.py @@ -1,120 +1,169 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- - -"""Test doi.py module.""" - - -from unittest import main, TestCase - -from lib.doi import doi_sfn_cit_ref - - -class DoiTest(TestCase): - - def test_doi1(self): - self.assertEqual( - "* {{cite journal | last=Atkins | first=Joshua H. | " - "last2=Gershell | first2=Leland J. | title=" - "Selective anticancer drugs | journal=Nature Reviews Drug " - "Discovery | publisher=Springer Nature | volume=1 | issue=7 | " - "year=2002 | issn=1474-1776 | doi=10.1038/nrd842 | pages=491–492 " - "| ref=harv}}", - doi_sfn_cit_ref('https://doi.org/10.1038%2Fnrd842')[1], - ) - - def test_doi2(self): - """Title of this DOI could not be detected in an older version.""" - self.assertEqual( - '* {{cite journal | title=Books of Critical Interest ' - '| journal=Critical Inquiry ' - '| publisher=University of Chicago Press | volume=40 ' - '| issue=3 | year=2014 | issn=0093-1896 | doi=10.1086/677379 ' - '| pages=272–281 ' - '| ref={{sfnref | University of Chicago Press | 2014}}' - '}}', - doi_sfn_cit_ref( - 'http://www.jstor.org/stable/info/10.1086/677379' - )[1], - ) - - def test_doi3(self): - """No author. URL contains %2F.""" - self.assertEqual( - '* {{cite journal | last=Spitzer | first=H. F. ' - '| title=Studies in retention. ' - '| journal=Journal of Educational Psychology ' - '| publisher=American Psychological Association (APA) ' - '| volume=30 | issue=9 | year=1939 | issn=0022-0663 ' - '| doi=10.1037/h0063404 | pages=641–656 ' - '| ref=harv}}', - doi_sfn_cit_ref('https://doi.org/10.1037%2Fh0063404')[1], - ) - - def test_doi4(self): - """publisher=Informa {UK""" - self.assertEqual( - '* {{cite journal | last=Davis | first=Margaret I. | last2=Jason ' - '| first2=Leonard A. | last3=Ferrari | first3=Joseph R. ' - '| last4=Olson | first4=Bradley D. | last5=Alvarez ' - '| first5=Josefina ' - '| title=A Collaborative Action Approach to Researching Substance' - ' Abuse Recovery ' - '| journal=The American Journal of Drug and Alcohol Abuse ' - '| publisher=Informa UK Limited ' - '| volume=31 | issue=4 | year=2005 | issn=0095-2990 ' - '| doi=10.1081/ada-200068110 | pages=537–553 ' - '| ref=harv}}', - doi_sfn_cit_ref('10.1081%2Fada-200068110')[1], - ) - - def test_incollection(self): - """Test the `incollection` type.""" - self.assertEqual( - '* {{cite book ' - '| last=Meyer ' - '| first=Albert R. ' - '| title=Lecture Notes in Mathematics ' - '| chapter=Weak monadic second order theory of succesor is not' - ' elementary-recursive ' - '| publisher=Springer Berlin Heidelberg ' - '| publication-place=Berlin, Heidelberg ' - '| year=1975 ' - '| isbn=978-3-540-07155-6 ' - '| issn=0075-8434 ' - '| doi=10.1007/bfb0064872 ' - '| ref=harv' - '}}', - doi_sfn_cit_ref('DOI 10.1007/BFb0064872')[1] - ) - - def test_doi_isbn_no_year(self): - """Test when issue date is empty.""" - self.assertEqual( - '* {{cite thesis | last=Ambati | first=V.R. ' - '| title=Forecasting water waves and currents :' - ' a space-time approach ' - '| publisher=University Library/University of Twente ' - '| isbn=978-90-365-2632-6 | doi=10.3990/1.9789036526326 ' - '| ref=harv}}', - doi_sfn_cit_ref('10.3990/1.9789036526326')[1] - ) - - def test_conference_location(self): - """Test citing a conference with location.""" - self.assertEqual( - "* {{cite conference " - "| title=Proceedings of the international workshop on System-level" - " interconnect prediction - SLIP'06 " - "| publisher=ACM Press " - "| publication-place=New York, New York, USA " - "| year=2006 " - "| isbn=1-59593-255-0 " - "| doi=10.1145/1117278 " - "| ref={{sfnref | ACM Press | 2006}}" - "}}", - doi_sfn_cit_ref('10.1145/1117278')[1] - ) - - -if __name__ == '__main__': - main() +from lib.doi import doi_to_dict +from lib.commons import dict_to_sfn_cit_ref + + +doi_scr = lambda doi: dict_to_sfn_cit_ref(doi_to_dict(doi)) + + +def test_doi1(): + assert ( + "* {{cite journal | last=Atkins | first=Joshua H. | " + "last2=Gershell | first2=Leland J. | title=" + "Selective anticancer drugs | journal=Nature Reviews Drug " + "Discovery | publisher=Springer Science and Business Media LLC " + "| volume=1 | issue=7 " + "| year=2002 | issn=1474-1776 | doi=10.1038/nrd842 " + "| pages=491–492}}" + ) == doi_scr('https://doi.org/10.1038%2Fnrd842')[1] + + +def test_doi2(): + """Title of this DOI could not be detected in an older version.""" + assert ( + '* {{cite journal | title=Books of Critical Interest ' + '| journal=Critical Inquiry ' + '| publisher=University of Chicago Press | volume=40 ' + '| issue=3 | year=2014 | issn=0093-1896 | doi=10.1086/677379 ' + '| pages=272–281 ' + '| ref={{sfnref | University of Chicago Press | 2014}}' + '}}') == doi_scr( + 'http://www.jstor.org/stable/info/10.1086/677379' + )[1] + + +def test_doi3(): + """No author. URL contains %2F.""" + assert ( + '* {{cite journal | last=Spitzer | first=H. F. ' + '| title=Studies in retention. ' + '| journal=Journal of Educational Psychology ' + '| publisher=American Psychological Association (APA) ' + '| volume=30 | issue=9 | year=1939 | issn=0022-0663 ' + '| doi=10.1037/h0063404 | pages=641–656}}' + ) == doi_scr('https://doi.org/10.1037%2Fh0063404')[1] + + +def test_doi4(): + """publisher=Informa {UK""" + assert ( + '* {{cite journal | last=Davis | first=Margaret I. | last2=Jason ' + '| first2=Leonard A. | last3=Ferrari | first3=Joseph R. ' + '| last4=Olson | first4=Bradley D. | last5=Alvarez ' + '| first5=Josefina ' + '| title=A Collaborative Action Approach to Researching Substance' + ' Abuse Recovery ' + '| journal=The American Journal of Drug and Alcohol Abuse ' + '| publisher=Informa UK Limited ' + '| volume=31 | issue=4 | year=2005 | issn=0095-2990 ' + '| doi=10.1081/ada-200068110 | pages=537–553}}' + ) == doi_scr('10.1081%2Fada-200068110')[1] + + +def test_incollection(): + """Test the `incollection` type.""" + assert ( + '* {{cite book ' + '| last=Meyer ' + '| first=Albert R. ' + '| title=Lecture Notes in Mathematics ' + '| chapter=Weak monadic second order theory of succesor is not' + ' elementary-recursive ' + '| publisher=Springer Berlin Heidelberg ' + '| publication-place=Berlin, Heidelberg ' + '| year=1975 ' + '| isbn=978-3-540-07155-6 ' + '| issn=0075-8434 ' + '| doi=10.1007/bfb0064872}}' + ) == doi_scr('DOI 10.1007/BFb0064872')[1] + + +def test_doi_isbn_no_year(): + """Test when issue date is empty.""" + assert ( + '* {{cite thesis | last=Ambati | first=V.R. ' + '| title=Forecasting water waves and currents :' + ' a space-time approach ' + '| publisher=University Library/University of Twente ' + '| isbn=978-90-365-2632-6 | doi=10.3990/1.9789036526326}}' + ) == doi_scr('10.3990/1.9789036526326')[1] + + +def test_conference_location(): + """Test citing a conference with location.""" + assert ( + "* {{cite conference " + "| title=Proceedings of the international workshop on System-level" + " interconnect prediction - SLIP'06 " + "| publisher=ACM Press " + "| publication-place=New York, New York, USA " + "| year=2006 " + "| isbn=1-59593-255-0 " + "| doi=10.1145/1117278 " + "| ref={{sfnref | ACM Press | 2006}}" + "}}" + ) == doi_scr('10.1145/1117278')[1] + + +def test_non_numeric_volume(): + assert ( + '* {{cite journal | last=Niemeyer | first=Jurgen | last2=Hinken ' + '| first2=Johann H. | last3=Kautz | first3=Richard L. ' + '| title=Near-Zero Bias Arrays of Josephson Tunnel Junctions ' + 'Providing Standard Voltages up to 1 V | journal=IEEE ' + 'Transactions on Instrumentation and Measurement ' + '| publisher=Institute of Electrical and Electronics Engineers ' + '(IEEE) | volume=IM-34 | issue=2 | year=1985 | issn=0018-9456 ' + '| doi=10.1109/tim.1985.4315297 | pages=185–187}}' + ) == doi_scr('10.1109/TIM.1985.4315297')[1] + + +def test_bad_author_name(): + assert ( + '* {{cite journal | last=Giusti | first=D. | last2=Lubicz ' + '| first2=V. | last3=Martinelli | first3=G. | last4=Sanfilippo ' + '| first4=F. | last5=Simula | first5=S. ' + '| title=Strange and charm HVP contributions to the muon (g − 2) ' + 'including QED corrections with twisted-mass fermions ' + '| journal=Journal of High Energy Physics ' + '| publisher=Springer Science and Business Media LLC ' + '| volume=2017 | issue=10 ' + '| year=2017 | issn=1029-8479 | doi=10.1007/jhep10(2017)157}}' + ) == doi_scr('10.1007/JHEP10(2017)157')[1] + + +def test_contains_brackets(): # 33 + assert ( # note `[zhu]` in doi, it should not be escaped + '* {{cite journal | last=Zhu | first=Liping | last2=Lin | first2=Xiao ' + '| last3=Li | first3=Yuanfang | last4=Li | first4=Bingyuan ' + '| last5=Xie | first5=Manping ' + '| title=Ostracoda Assemblages in Core Sediments and Their ' + 'Environmental Significance in a Small Lake in Northwest Tibet, China ' + '| journal=Arctic, Antarctic, and Alpine Research ' + '| publisher=Informa UK Limited | volume=39 | issue=4 | year=2007 ' + '| issn=1523-0430 | doi=10.1657/1523-0430(07-512)[zhu]2.0.co;2 ' + '| pages=658–662}}') == doi_scr('10.1657/1523-0430(07-512)[ZHU]2.0.CO;2')[1] + + +def test_non_crossref_doi(): # 35 + assert ( + '* {{cite journal | last=Hein | first=Andreas M. | last2=Baxter ' + '| first2=Stephen ' + '| title=Artificial Intelligence for Interstellar Travel ' + '| journal=arXiv | publisher=arXiv | doi=10.48550/ARXIV.1811.06526 ' + '| url=https://arxiv.org/abs/1811.06526 ' + '| access-date=' + ) == doi_scr('10.48550/arXiv.1811.06526')[1][:-12] + + +def test_doi_with_full_date(): # 36 + assert ( + '* {{cite journal | last=Webber | first=W. R. | last2=McDonald ' + '| first2=F. B. | last3=Lockwood | first3=J. A. | last4=Heikkila ' + '| first4=B. ' + '| title=The effect of the July 14, 2000 “Bastille Day” solar flare ' + 'event on >70 MeV galactic cosmic rays observed at V1 and V2 in ' + 'the distant heliosphere | journal=Geophysical Research Letters ' + '| publisher=American Geophysical Union (AGU) | volume=29 ' + '| issue=10 | date=2002-05-15 | issn=0094-8276 ' + '| doi=10.1029/2002gl014729 | pages=15–1–15–3}}' + ) == doi_scr('10.1029/2002GL014729')[1] diff --git a/test/googlebooks_test.py b/test/googlebooks_test.py index 96266a5c..907cce04 100644 --- a/test/googlebooks_test.py +++ b/test/googlebooks_test.py @@ -1,182 +1,154 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- +from urllib.parse import urlparse -"""Test googlebooks.py module.""" +from pytest import mark +from lib.commons import dict_to_sfn_cit_ref +from lib.googlebooks import url_to_dict -from unittest import main, TestCase -from lib.googlebooks import googlebooks_sfn_cit_ref +def _googlebooks_scr(url): + return dict_to_sfn_cit_ref(url_to_dict(url)) -class GooglebooksTest(TestCase): +def googlebooks_scr(url): + return _googlebooks_scr(urlparse(url)) - def test_gb1(self): - i = ( - 'http://books.google.com/books?' - 'id=pzmt3pcBuGYC&pg=PR11&lpg=PP1&dq=digital+library' - ) - o = googlebooks_sfn_cit_ref(i) - e = ( - '* {{cite book ' - '| last=Arms ' - '| first=W.Y. ' - '| title=Digital Libraries ' - '| publisher=MIT Press ' - '| series=Digital libraries and electronic publishing ' - '| year=2001 ' - '| isbn=978-0-262-26134-0 ' - '| url=https://books.google.com/books?id=pzmt3pcBuGYC&pg=PR11 ' - '| ref=harv ' - '| access-date=' - ) - self.assertIn(e, o[1]) - - def test_gb2(self): - """a book with more than 4 authors (10 authors)""" - i = ( + +def test_gb1(): + assert ( + '* {{cite book ' + '| last=Arms ' + '| first=W.Y. ' + '| title=Digital Libraries ' + '| publisher=MIT Press ' + '| series=Digital Libraries and Electronic Publishing ' + '| year=2001 ' + '| isbn=978-0-262-26134-0 ' + '| url=https://books.google.com/books?id=pzmt3pcBuGYC&pg=PR11 ' + '| access-date=') in googlebooks_scr( 'http://books.google.com/books?' - 'id=U46IzqYLZvAC&pg=PT57#v=onepage&q&f=false' - ) - o = googlebooks_sfn_cit_ref(i) - e1 = ( - '{{sfn ' - '| Anderson ' - '| DeBolt ' - '| Featherstone ' - '| Gunther ' - '| 2010 ' - '| p=57}}' - ) - e2 = ( - '* {{cite book ' - '| last=Anderson ' - '| first=E. ' - '| last2=DeBolt ' - '| first2=V. ' - '| last3=Featherstone ' - '| first3=D. ' - '| last4=Gunther ' - '| first4=L. ' - '| last5=Jacobs ' - '| first5=D.R. ' - '| last6=Mills ' - '| first6=C. ' - '| last7=Schmitt ' - '| first7=C. ' - '| last8=Sims ' - '| first8=G. ' - '| last9=Walter ' - '| first9=A. ' - '| last10=Jensen-Inman ' - '| first10=L. ' - '| title=InterACT with Web Standards: ' - 'A holistic approach to web design ' - '| publisher=Pearson Education ' - '| series=Voices That Matter ' - '| year=2010 ' - '| isbn=978-0-13-270490-8 ' - '| url=https://books.google.com/books?id=U46IzqYLZvAC&pg=PT57 ' - '| ref=harv ' - '| access-date=' - ) - self.assertIn(e1, o[0]) - self.assertIn(e2, o[1]) - - def test_gb3(self): - """Non-ascii characters in title (Some of them where removed later)""" - i = ( - 'http://books.google.com/books?id=icMEAAAAQBAJ&pg=PA588&dq=%22a+' - 'Delimiter+is%22&hl=en&sa=X&ei=oNKSUrKeDovItAbO_4CoBA&ved=' - '0CC4Q6AEwAA#v=onepage&q=%22a%20Delimiter%20is%22&f=false' - ) - o = googlebooks_sfn_cit_ref(i) - e1 = '{{sfn | Farrell | 2009 | p=588}}' - e2 = ( - '* {{cite book ' - '| last=Farrell ' - '| first=J. ' - '| title=Microsoft Visual C# 2008 Comprehensive: ' - 'An Introduction to Object-Oriented Programming ' - '| publisher=Cengage Learning ' - '| year=2009 ' - '| isbn=978-1-111-78619-9 ' - '| url=https://books.google.com/books?id=icMEAAAAQBAJ&pg=PA588 ' - '| ref=harv ' - '| access-date=' - ) - self.assertIn(e1, o[0]) - self.assertIn(e2, o[1]) - - def test_gb4(self): - """Non-ascii characters in author's name.""" - i = ( - 'https://books.google.com/books?id=' - 'i8nZjjo_9ikC&pg=PA229&dq=%22legal+translation+is%22&hl=en&sa=' - 'X&ei=hEuYUr_mOsnKswb49oDQCA&ved=0CC4Q6AEwAA#v=onepage&q=' - '%22legal%20translation%20is%22&f=false' - ) - o = googlebooks_sfn_cit_ref(i) - e1 = '{{sfn | Šarčević | 1997 | p=229}}' - e2 = ( - '* {{cite book ' - '| last=Šarčević ' - '| first=S. ' - '| title=New Approach to Legal Translation ' - '| publisher=Springer Netherlands ' - '| year=1997 ' - '| isbn=978-90-411-0401-4 ' - '| url=https://books.google.com/books?id=i8nZjjo_9ikC&pg=PA229 ' - '| ref=harv ' - '| access-date=' - ) - self.assertIn(e1, o[0]) - self.assertIn(e2, o[1]) - - def test_gb5(self): - """ref checking""" - i = ( - 'https://encrypted.google.com/books?id=6upvonUt0O8C&pg=PA378&' - 'dq=density+of+granite&hl=en&sa=X&ei=YBHIU-qCBIyX0QXusoDgAg&ved=' - '0CEIQ6AEwBjgK#v=onepage&q=density%20of%20granite&f=false' - ) - o = googlebooks_sfn_cit_ref(i) - ctnt = ( - '* {{cite book ' - '| last=Serway ' - '| first=R.A. ' - '| last2=Jewett ' - '| first2=J.W. ' - '| title=Physics for Scientists and Engineers, Volume 1, ' - 'Chapters 1-22 | publisher=Cengage Learning ' - '| series=Physics for Scientists and Engineers ' - '| year=2009 ' - '| isbn=978-1-4390-4838-2 ' - '| url=https://encrypted.google.com/books?id=6upvonUt0O8C&pg=PA378' - ' ' - '| ref=harv ' - '| access-date=' - ) - reft = ( - '<ref name="Serway Jewett 2009 p. 378">' - '{{cite book ' - '| last=Serway ' - '| first=R.A. ' - '| last2=Jewett ' - '| first2=J.W. ' - '| title=Physics for Scientists and Engineers, Volume 1, ' - 'Chapters 1-22 | publisher=Cengage Learning ' - '| series=Physics for Scientists and Engineers ' - '| year=2009 ' - '| isbn=978-1-4390-4838-2 ' - '| url=https://encrypted.google.com/books?id=6upvonUt0O8C&pg=PA378' - ' ' - '| access-date=' - ) - self.assertIn(ctnt, o[1]) - self.assertIn(reft, o[2]) - self.assertIn(' | page=378}}</ref>', o[2]) - - -if __name__ == '__main__': - main() + 'id=pzmt3pcBuGYC&pg=PR11&lpg=PP1&dq=digital+library')[1] + + +def test_gb2(): + """a book with more than 4 authors (10 authors)""" + o = googlebooks_scr( + 'http://books.google.com/books?' + 'id=U46IzqYLZvAC&pg=PT57#v=onepage&q&f=false') + assert ( + '{{sfn ' + '| Anderson ' + '| DeBolt ' + '| Featherstone ' + '| Gunther ' + '| 2010 ' + '| p=57}}') in o[0] + assert ( + '* {{cite book ' + '| last=Anderson ' + '| first=E. ' + '| last2=DeBolt ' + '| first2=V. ' + '| last3=Featherstone ' + '| first3=D. ' + '| last4=Gunther ' + '| first4=L. ' + '| last5=Jacobs ' + '| first5=D.R. ' + '| last6=Mills ' + '| first6=C. ' + '| last7=Schmitt ' + '| first7=C. ' + '| last8=Sims ' + '| first8=G. ' + '| last9=Walter ' + '| first9=A. ' + '| last10=Jensen-Inman ' + '| first10=L. ' + '| title=InterACT with Web Standards: ' + 'A holistic approach to web design ' + '| publisher=Pearson Education ' + '| series=Voices That Matter ' + '| year=2010 ' + '| isbn=978-0-13-270490-8 ' + '| url=https://books.google.com/books?id=U46IzqYLZvAC&pg=PT57 ' + '| access-date=') in o[1] + + +def test_gb3(): + """Non-ascii characters in title (Some of them where removed later)""" + o = googlebooks_scr( + 'http://books.google.com/books?id=icMEAAAAQBAJ&pg=PA588&dq=%22a+' + 'Delimiter+is%22&hl=en&sa=X&ei=oNKSUrKeDovItAbO_4CoBA&ved=' + '0CC4Q6AEwAA#v=onepage&q=%22a%20Delimiter%20is%22&f=false') + assert '{{sfn | Farrell | 2009 | p=588}}' in o[0] + assert ( + '* {{cite book ' + '| last=Farrell ' + '| first=J. ' + '| title=Microsoft Visual C# 2008 Comprehensive: ' + 'An Introduction to Object-Oriented Programming ' + '| publisher=Cengage Learning ' + '| year=2009 ' + '| isbn=978-1-111-78619-9 ' + '| url=https://books.google.com/books?id=icMEAAAAQBAJ&pg=PA588 ' + '| access-date=') in o[1] + + +@mark.xfail +def test_gb4(): + """Non-ascii characters in author's name.""" + o = googlebooks_scr( + 'https://books.google.com/books?id=' + 'i8nZjjo_9ikC&pg=PA229&dq=%22legal+translation+is%22&hl=en&sa=' + 'X&ei=hEuYUr_mOsnKswb49oDQCA&ved=0CC4Q6AEwAA#v=onepage&q=' + '%22legal%20translation%20is%22&f=false') + assert '{{sfn | Šarčević | 1997 | p=229}}' in o[0] + assert ( + '* {{cite book ' + '| last=Šarčević ' + '| first=S. ' + '| title=New Approach to Legal Translation ' + '| publisher=Springer Netherlands ' + '| year=1997 ' + '| isbn=978-90-411-0401-4 ' + '| url=https://books.google.com/books?id=i8nZjjo_9ikC&pg=PA229 ' + '| access-date=') in o[1] + + +def test_gb5(): + """ref checking""" + o = googlebooks_scr( + 'https://encrypted.google.com/books?id=6upvonUt0O8C&pg=PA378&' + 'dq=density+of+granite&hl=en&sa=X&ei=YBHIU-qCBIyX0QXusoDgAg&ved=' + '0CEIQ6AEwBjgK#v=onepage&q=density%20of%20granite&f=false') + assert ( + '* {{cite book ' + '| last=Serway ' + '| first=R.A. ' + '| last2=Jewett ' + '| first2=J.W. ' + '| title=Physics for Scientists and Engineers, Volume 1, ' + 'Chapters 1-22 | publisher=Cengage Learning ' + '| series=Physics for Scientists and Engineers ' + '| year=2009 ' + '| isbn=978-1-4390-4838-2 ' + '| url=https://books.google.com/books?id=6upvonUt0O8C&pg=PA378' + ' ' + '| access-date=') in o[1] + assert ( + '<ref name="Serway Jewett 2009 p. 378">' + '{{cite book ' + '| last=Serway ' + '| first=R.A. ' + '| last2=Jewett ' + '| first2=J.W. ' + '| title=Physics for Scientists and Engineers, Volume 1, ' + 'Chapters 1-22 | publisher=Cengage Learning ' + '| series=Physics for Scientists and Engineers ' + '| year=2009 ' + '| isbn=978-1-4390-4838-2 ' + '| url=https://books.google.com/books?id=6upvonUt0O8C&pg=PA378' + ' ' + '| access-date=') in o[2] + assert ' | page=378}}</ref>' in o[2] diff --git a/test/isbn_oclc_test.py b/test/isbn_oclc_test.py index 894a8720..66567381 100644 --- a/test/isbn_oclc_test.py +++ b/test/isbn_oclc_test.py @@ -1,106 +1,99 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- - -"""Test isbn.py module.""" - - -from unittest import TestCase, main - -from lib.isbn_oclc import isbn_sfn_cit_ref, oclc_sfn_cit_ref - - -class IsbnTest(TestCase): - - def test_is1(self): - """not found in adinebook""" - self.assertIn(( - '* {{cite book ' - '| last=Adkins ' - '| first=Roy ' - '| title=The war for all the oceans : ' - 'from Nelson at the Nile to Napoleon at Waterloo ' - '| publisher=Abacus ' - '| publication-place=London ' - '| year=2007 ' - '| isbn=978-0-349-11916-8 ' - '| oclc=137313052 ' - '| ref=harv}}' - ), isbn_sfn_cit_ref('9780349119168', pure=True)[1]) - - def test_is2(self): - """not found in ottobib""" - self.assertEqual( - '* {{cite book | last=منصور | first=جهانگیر ' - '| others=بدیل بن علی خاقانی (شاعر), and بدیع الزمان فروزانفر' - ' (مقدمه) | title=دیوان خاقانی شروانی | publisher=نگاه ' - '| year=1389 | isbn=978-964-6736-71-9 | language=fa | ref=harv}}', - isbn_sfn_cit_ref('978-964-6736-71-9', pure=True)[1] +from pytest import raises + +from lib.isbn_oclc import isbn_to_dict, oclc_dict +from lib.commons import ISBN_10OR13_SEARCH, dict_to_sfn_cit_ref, ReturnError + + +def isbn_scr(*args): + return dict_to_sfn_cit_ref(isbn_to_dict(*args)) + + +def oclc_scr(*args): + return dict_to_sfn_cit_ref(oclc_dict(*args)) + + +def test_is1(): + # not in ketabir + assert ( + '* {{cite book ' + '| last=Adkins ' + '| first=Roy ' + '| last2=Adkins ' + '| first2=Lesley ' + '| title=The war for all the oceans : ' + 'from Nelson at the Nile to Napoleon at Waterloo ' + '| publisher=Abacus ' + '| publication-place=London ' + '| date=2007 ' + '| isbn=978-0-349-11916-8 ' + '| oclc=137313052}}' + ) == isbn_scr('9780349119168', True)[1] + + +def test_is3(): + # on both ketabid and citoid + assert ( + '* {{cite book | last=Sipihrī | first=Suhrāb. ' + '| title=Rāz-i gul-i surkh ' + '| publisher=Muʼassasah-ʼi Intishārāt-i Nigāh ' + '| publication-place=Tihrān | date=1379 [2000 or 2001] ' + '| isbn=964-6736-34-3 ' + '| oclc=53446327}}' + ) == isbn_scr('964-6736-34-3 ')[1] + + +def test_is4(): + """unpure isbn10 not found in ottobib""" + assert ( + '* {{cite book | last=حافظ | first=شمس‌الدین‌محمد ' + '| others=رضا نظرزاده (به‌اهتمام) ' + '| title=دیوان کامل حافظ همراه با فالنامه | publisher=دیوان ' + '| publication-place=قم - قم | year=1385 | isbn=978-964-92962-6-5 ' + '| language=fa}}' + ) == isbn_scr('choghondar 964-92962-6-3 شلغم')[1] + + +def test_oclc1(): + assert oclc_scr('875039842')[1] == ( + '* {{cite book | last=Lewis | first=James Bryant | last2=Sesay | first2=Amadu | title=Korea and globalization : politics, economics and culture | publisher=RoutledgeCurzon | publication-place=Richmond | year=2002 | isbn=978-0-7007-1512-1 | oclc=875039842}}' + ) + + +def test_elec_type_with_url(): + assert oclc_scr('809771201')[1] == ( + "* {{cite book | last=Rahman | first=Mizanur | title=MediaWiki Administrators' Tutorial Guide | publisher=Packt Pub. | publication-place=Birmingham | year=2007 | isbn=978-1-84719-045-1 | oclc=809771201}}" + ) + + +def test_fullname_in_ris(): + assert oclc_scr('24680975')[1] == ( + '* {{cite book ' + '| author=Universidade Federal do Rio de Janeiro ' + '| title=Universidade do Brasil, 1948-1966 ' + '| year=1966 ' + '| oclc=24680975 ' + '| language=pt}}' + ) + + +def test_hyphened_isbn_match(): # 30 + assert ISBN_10OR13_SEARCH('2-253-00422-7') + + +def test_citoid_only(): # 31 + assert ( + '* {{cite book | last=Ramseier | first=Walter ' + '| title=Münchenstein - Heimatkunde | publication-place=[Liestal] ' + '| isbn=978-3-85673-522-7 | oclc=613273377 ' + '| language=de}}' + ) == isbn_scr('3-85673-522-4')[1] + + +def test_invalid_oclc(): + with raises(ReturnError) as e: + oclc_dict('99999999999999') + assert e.args == ( + 'Error processing OCLC number: 99999999999999', + 'Make sure the OCLC identifier is valid.', + '' ) - - def test_is3(self): - """exists in both""" - self.assertEqual(( - '* {{cite book | last=معصومی | first=سحر | title=راز گل سرخ: نقد ' - 'و گزیده شعرهای سهراب سپهری | publisher=نگاه | year=1386 | ' - 'isbn=964-6736-34-3 | oclc=53446327 | language=fa | ref=harv}}' - ), isbn_sfn_cit_ref('964-6736-34-3 ')[1]) - - def test_is4(self): - """unpure isbn10 not found in ottobib""" - self.assertEqual(( - '* {{cite book | last=حافظ | first=شمس الدین محمد | ' - 'last2=نظرزاده | first2=رضا | title=دیوان کامل حافظ همراه با ' - 'فالنامه | publisher=دیوان | year=1385 | isbn=964-92962-6-3 | ' - 'language=fa | ref=harv}}' - ), isbn_sfn_cit_ref('choghondar 964-92962-6-3 شلغم')[1]) - - -class OCLCTest(TestCase): - - def test_oclc1(self): - self.maxDiff = None - self.assertEqual(( - '* {{cite book ' - '| last=Lewis ' - '| first=James Bryant ' - '| last2=Sesay ' - '| first2=Amadu ' - '| title=Korea and globalization :' - ' politics, economics and culture ' - '| publisher=RoutledgeCurzon ' - '| year=2002 ' - '| isbn=0-7007-1512-6 ' - '| oclc=875039842 ' - '| ref=harv}}' - ), oclc_sfn_cit_ref('875039842')[1]) - - def test_elec_type_with_url(self): - self.assertIn(( - "* {{cite web " - "| last=Rahman " - "| first=Mizanur " - "| title=MediaWiki Administrators' Tutorial Guide " - "| publisher=Packt Pub. " - "| year=2007 " - "| isbn=978-1-84719-045-1 " - "| oclc=809771201 " - "| url=http://public.eblib.com/choice/publicfullrecord.aspx?p=" - "995605 " - "| ref=harv " - "| access-date=" - ), oclc_sfn_cit_ref('809771201')[1]) - - def test_fullname_in_ris(self): - self.assertEqual(( - '* {{cite book ' - '| author=Universidade Federal do Rio de Janeiro ' - '| title=Universidade do Brasil, 1948-1966 ' - '| year=1966 ' - '| oclc=24680975 ' - '| language=pt ' - '| ref=harv}}' - ), oclc_sfn_cit_ref('24680975')[1]) - - -if __name__ == '__main__': - main() diff --git a/test/jstor_test.py b/test/jstor_test.py new file mode 100644 index 00000000..6814089c --- /dev/null +++ b/test/jstor_test.py @@ -0,0 +1,38 @@ +from lib.jstor import url_to_dict +from lib.commons import dict_to_sfn_cit_ref + + +jstor_scr = lambda *args: dict_to_sfn_cit_ref(url_to_dict(*args)) + + +def test_1(): + s, c, r = jstor_scr('https://www.jstor.org/stable/30078788') + assert s == '{{sfn | Lloyd | 1831 | pp=171–177}}' + assert c[:c.index('| access-date=')] == ( + '* {{cite journal | last=Lloyd | first=Humphrey ' + '| title=On a New Case of Interference of the Rays of Light ' + '| journal=The Transactions of the Royal Irish Academy ' + '| publisher=Royal Irish Academy | volume=17 | year=1831 ' + '| issn=07908113 | jstor=30078788 | pages=171–177 ' + '| url=http://www.jstor.org/stable/30078788 ') + + +def test_2(): + s, c, r = jstor_scr('https://www.jstor.org/stable/resrep26363.7?Search=yes&resultItemClick=true&searchText=google&searchUri=%2Faction%2FdoBasicSearch%3FQuery%3Dgoogle%26acc%3Doff%26wc%3Don%26fc%3Doff%26group%3Dnone%26refreqid%3Dsearch%253A2e627536469ca8786b576957a9797d56&ab_segments=0%2Fbasic_search_gsv2%2Fcontrol&refreqid=fastly-default%3Af90c911269c590baf37330b9d16ae1cd&seq=1#metadata_info_tab_contents') + assert c[:c.index('| access-date=')] == ( + '* {{cite techreport | last=Singh | first=Spandana | last2=Blase ' + '| first2=Margerite ' + '| title=Protecting the Vote: How Internet Platforms Are Addressing Election and Voter Suppression-Related Misinformation and Disinformation ' + '| year=2020 | jstor=resrep26363.7 | jstor-access=free ' + '| url=http://www.jstor.org/stable/resrep26363.7 ') + + +def test_encoding(): # 25 + s, c, r = jstor_scr('https://www.jstor.org/stable/40991855') + assert c[:c.index('| access-date=')] == ( + '* {{cite journal | last=Monteiro ' + '| first=Carlos Augusto de Figueiredo ' + '| title=“Calamidades Meteorológicas no Brasil Meridional, em Agôsto de 1965” ' + '| journal=Revista Geográfica | publisher=Pan American Institute of Geography and History ' + '| volume=35 | issue=63 | year=1965 | issn=00310581 | jstor=40991855 ' + '| pages=173–178 | url=http://www.jstor.org/stable/40991855 ') diff --git a/test/ketabir_test.py b/test/ketabir_test.py new file mode 100644 index 00000000..48d8fa4f --- /dev/null +++ b/test/ketabir_test.py @@ -0,0 +1,95 @@ +from lib.ketabir import url_to_dict, isbn_to_url +from lib.commons import dict_to_sfn_cit_ref + + +ketabir_scr = lambda *args: dict_to_sfn_cit_ref(url_to_dict(*args)) + + +def test_ab1(): + """authors = 1, translators = 2, otheo = 1, isbn13""" + assert ( + '* {{cite book | last=لانسکی | first=ویکی ' + '| others=کی وایت (تصويرگر), فیروزه دالکی (مترجم), ' + 'and مژگان امیرفروغی (مترجم) ' + '| title=101 راه برای اینکه پدر بهتری باشید ' + '| publisher=پیک ادبیات | publication-place=تهران - تهران ' + '| year=1386 | isbn=978-964-8165-81-4 | language=fa}}' + ) == ketabir_scr('https://ketab.ir/book/27b3444f-1175-4db0-8411-b1719a5d7ed1')[1] + + +def test_ab2(): + """authors = 3, translators = 2, otheo = 0, isbn13""" + assert ( + '* {{cite book | last=کرسول | first=جان | last2=پلانو‌کلارک ' + '| first2=ویکی ' + '| others=محسن نیازی (مترجم), and عباس زارعی (مترجم) ' + '| title=روش\u200cهای تحقیق تلفیقی ' + '| publisher=ثامن الحجج | publication-place=تهران - تهران ' + '| volume=1 | year=1387 | isbn=978-964-2823-35-2 | language=fa}}' + ) == ketabir_scr('https://ketab.ir/book/667c900a-69bd-4a1a-a651-1870d2f63a68')[1] + + +def test_ab3(): + """authors = 2, translators = 0, otheo = 4, isbn13""" + assert ( + '* {{cite book | last=فخررحیمی | first=علیرضا | last2=فخررحیمی ' + '| first2=الهام ' + '| others=آرش نادرپور ' + '(مقدمه), وحید شهبازیان (مقدمه), رضا مقدم (مقدمه), and' + ' امیر جابری (مقدمه) ' + '| title=آموزش گام به گام پیکربندی مسیریابهای میکروتیک:' + ' آمادگی آزمون MTCNA | publisher=نشرگستر ' + '| publication-place=تهران - تهران | year=1391 ' + '| isbn=978-600-5883-43-5 | language=fa}}' + ) == ketabir_scr('https://ketab.ir/book/f37fad8e-8f0b-4cd9-8875-f5de0e0d86ef')[1] + + +def test_ab4(): + """authors = 3, translators = 0, otheo = 0, isbn13""" + assert ( + '* {{cite book | last=کریمی | first=نجمه | last2=یزدخواستی ' + '| first2=فروغ | last3=مختاری | first3=صفورا ' + '| title=11 سپتامبر ... آرماگدون | publisher=حدیث راه عشق ' + '| publication-place=اصفهان - اصفهان | year=1386 ' + '| isbn=978-964-95633-4-3 | language=fa}}' + ) == ketabir_scr('https://ketab.ir/book/13a52229-5e3f-479e-8954-092b65e85923')[1] + + +def test_ab5(): + """Year is interesting here.""" + assert ( + '* {{cite book | last=یوسف‌نژاد | first=یوسف‌علی ' + '| title=فراهنجاری ' + 'در مثنوی‌سرایی: بررسی قالب غزل - مثنوی در ادب فارسی ' + '| publisher=هنر رسانه اردیبهشت ' + '| publication-place=تهران - تهران | year=1388 ' + '| isbn=978-964-2656-34-9 | language=fa}}' + ) == ketabir_scr('https://ketab.ir/book/a5958832-5c43-460d-bf42-65acb6077e52')[1] + + +def test_ab6(): + """Month and year detection.""" + assert ketabir_scr( + 'https://ketab.ir/book/cb1989dc-ba09-4df6-aaee-fcdbd25ad322' + )[1] == ( + '* {{cite book | last=مونس | first=حسین | others=حمیدرضا شیخی (مترجم) | ' + 'title=تاریخ و تمدن مغرب | publisher=سمت | publication-place=مشهد - خراسان ' + 'رضوی | volume=1 | year=1390 | isbn=978-964-530-036-2 | language=fa}}' + ) + + +def test_ab7(): + """1 Editor.""" + assert ( + '* {{cite book | last=دیماتیو | first=ام.رابین | editor-last=جباری | ' + 'editor-first=کریم | others=کیانوش هاشمیان (زيرنظر), and محمد کاویانی (مترجم) ' + '| title=روانشناسی سلامت به ضمیمه نگرشی بر منابع اسلامی | publisher=سمت | ' + 'publication-place=تهران - تهران | volume=1 | year=1379 | ' + 'isbn=978-964-459-398-7 | language=fa}}' + ) == ketabir_scr('https://ketab.ir/book/4cc231f9-35c2-4b60-a714-a0a11135e932')[1] + + +def test_isbn2url(): + assert isbn_to_url( + '978-964-459-398-7' + ) == 'https://ketab.ir/book/4cc231f9-35c2-4b60-a714-a0a11135e932' diff --git a/test/noorlib_test.py b/test/noorlib_test.py index 25ad3dea..b8c0095f 100644 --- a/test/noorlib_test.py +++ b/test/noorlib_test.py @@ -1,63 +1,51 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- - -"""Test noorlib.py module.""" - - -from unittest import main, TestCase - -from lib.noorlib import noorlib_sfn_cit_ref - - -class NoorlibTest(TestCase): - - def test_nl1(self): - i = 'http://www.noorlib.ir/View/fa/Book/BookView/Image/6120' - o = noorlib_sfn_cit_ref(i) - e = ( - '* {{cite book ' - '| last=رشید یاسمی ' - '| first=غلامرضا ' - '| last2=کریستن سن ' - '| first2=آرتور امانویل ' - '| title=ایران در زمان ساسانیان: تاریخ ایران ساسانی تا' - ' حمله عرب و وضع دولت و ملت در زمان ساسانیان ' - '| publisher=دنیای کتاب ' - '| publication-place=تهران - ایران ' - '| series=ایران در زمان ساسانیان: تاریخ ایران ساسانی تا' - ' حمله عرب و وضع دولت و ملت در زمان ساسانیان ' - '| volume=1 ' - '| year=1368 ' - '| url=http://www.noorlib.ir/View/fa/Book/BookView/Image/6120 ' - '| language=فارسی ' - '| ref=harv ' - '| access-date=' - ) - self.assertIn(e, o[1]) - - def test_nl2(self): - """The year parameter is not present.""" - i = 'http://www.noorlib.ir/View/fa/Book/BookView/Image/18454' - o = noorlib_sfn_cit_ref(i) - self.assertIn('{{sfn | کورانی | p=}}', o[0]) - self.assertIn( - '* {{cite book ' - '| last=کورانی ' - '| first=علی ' - '| title=المعجم الموضوعی لاحادیث ' - 'الامام المهدی عجل الله تعالی فرجه الشریف ' - '| publisher=دار المرتضی ' - '| publication-place=بيروت ' - '| series=المعجم الموضوعي لإحادیث' - ' الإمام المهدي (عجل الله فرجه الشریف) ' - '| volume=1 ' - '| url=http://www.noorlib.ir/View/fa/Book/BookView/Image/18454 ' - '| language=عربی ' - '| ref=harv ' - '| access-date=', - o[1], - ) - - -if __name__ == '__main__': - main() +from lib.noorlib import url_to_dict +from lib.commons import dict_to_sfn_cit_ref + + +noorlib_scr = lambda *args: dict_to_sfn_cit_ref(url_to_dict(*args)) + + +def test_nl1(): + i = 'http://www.noorlib.ir/View/fa/Book/BookView/Image/6120' + o = noorlib_scr(i) + e = ( + '* {{cite book ' + '| last=رشید یاسمی ' + '| first=غلامرضا ' + '| last2=کریستن سن ' + '| first2=آرتور امانویل ' + '| title=ایران در زمان ساسانیان: تاریخ ایران ساسانی تا' + ' حمله عرب و وضع دولت و ملت در زمان ساسانیان ' + '| publisher=دنیای کتاب ' + '| publication-place=تهران - ایران ' + '| series=ایران در زمان ساسانیان: تاریخ ایران ساسانی تا' + ' حمله عرب و وضع دولت و ملت در زمان ساسانیان ' + '| volume=1 ' + '| year=1368 ' + '| url=http://www.noorlib.ir/View/fa/Book/BookView/Image/6120 ' + '| language=فارسی ' + '| access-date=' + ) + assert e in o[1] + + +def test_nl2(): + """The year parameter is not present.""" + i = 'http://www.noorlib.ir/View/fa/Book/BookView/Image/18454' + o = noorlib_scr(i) + assert '{{sfn | کورانی | p=}}' in o[0] + assert ( + '* {{cite book ' + '| last=کورانی ' + '| first=علی ' + '| title=المعجم الموضوعی لاحادیث ' + 'الامام المهدی عجل الله تعالی فرجه الشریف ' + '| publisher=دار المرتضی ' + '| publication-place=بيروت ' + '| series=المعجم الموضوعي لإحادیث' + ' الإمام المهدي (عجل الله فرجه الشریف) ' + '| volume=1 ' + '| url=http://www.noorlib.ir/View/fa/Book/BookView/Image/18454 ' + '| language=عربی ' + '| access-date=' + ) in o[1] diff --git a/test/noormags_test.py b/test/noormags_test.py index a07d3839..7a302739 100644 --- a/test/noormags_test.py +++ b/test/noormags_test.py @@ -1,83 +1,70 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- +from lib.noormags import url_to_dict +from lib.commons import dict_to_sfn_cit_ref -"""Test noormags.py module.""" +noormags_scr = lambda *args: dict_to_sfn_cit_ref(url_to_dict(*args)) -from unittest import main, TestCase -from lib.noormags import noormags_sfn_cit_ref +def test_nm1(): + """The second author does not have a last name. (Bibtex file error)""" + i = 'http://www.noormags.ir/view/fa/articlepage/105489/' \ + '%d8%aa%d8%ad%d9%84%db%8c%d9%84-%d9%85%d9%86%d8%a7%d9%81%d8%b9-' \ + '%d8%a8%d9%87%d8%b1%d9%87-%d9%88%d8%b1%db%8c-' \ + '%d9%86%d8%a7%d8%b4%db%8c-%d8%a7%d8%b2-' \ + '%d8%a7%d8%b5%d9%84%d8%a7%d8%ad%d8%a7%d8%aa-' \ + '%d8%b5%d9%86%d8%b9%d8%aa-%d8%a8%d8%b1%d9%82-' \ + '%d8%a7%d8%b3%d8%aa%d8%b1%d8%a7%d9%84%db%8c%d8%a7--' \ + '%da%86%d8%a7%d8%b1%da%86%d9%88%d8%a8-%d9%87%d8%a7%db%8c-' \ + '%d8%b1%d9%88%d8%b4-%d8%b4%d9%86%d8%a7%d8%ae%d8%aa%db%8c?q=' \ + '%D8%A8%D8%B1%D9%82&score=21.639421&rownumber=1' + o = noormags_scr(i) + e = ( + '* {{cite journal ' + '| last=فتح\u200cالله\u200cزاده\u200cاقدم ' + '| first=\u200cرضا ' + '| title=تحلیل منافع بهره وری ناشی' + ' از اصلاحات صنعت برق استرالیا: چارچوب های روش شناختی ' + '| journal=مطالعات اقتصاد انرژی ' + '| issue=3 ' + '| year=1383 ' + '| pages=55–55 ' + '| url=http://www.noormags.ir/view/fa/articlepage/105489 ' + '| language=fa ' + '| access-date=' + ) + assert e in o[1] -class NoormagsTest(TestCase): - - def test_nm2(self): - """The second author does not have a last name. (Bibtex file error)""" - i = 'http://www.noormags.ir/view/fa/articlepage/105489/' \ - '%d8%aa%d8%ad%d9%84%db%8c%d9%84-%d9%85%d9%86%d8%a7%d9%81%d8%b9-' \ - '%d8%a8%d9%87%d8%b1%d9%87-%d9%88%d8%b1%db%8c-' \ - '%d9%86%d8%a7%d8%b4%db%8c-%d8%a7%d8%b2-' \ - '%d8%a7%d8%b5%d9%84%d8%a7%d8%ad%d8%a7%d8%aa-' \ - '%d8%b5%d9%86%d8%b9%d8%aa-%d8%a8%d8%b1%d9%82-' \ - '%d8%a7%d8%b3%d8%aa%d8%b1%d8%a7%d9%84%db%8c%d8%a7--' \ - '%da%86%d8%a7%d8%b1%da%86%d9%88%d8%a8-%d9%87%d8%a7%db%8c-' \ - '%d8%b1%d9%88%d8%b4-%d8%b4%d9%86%d8%a7%d8%ae%d8%aa%db%8c?q=' \ - '%D8%A8%D8%B1%D9%82&score=21.639421&rownumber=1' - o = noormags_sfn_cit_ref(i) - e = ( - '* {{cite journal ' - '| last=فتح\u200cالله\u200cزاده\u200cاقدم ' - '| first=\u200cرضا ' - '| title=تحلیل منافع بهره وری ناشی' - ' از اصلاحات صنعت برق استرالیا: چارچوب های روش شناختی ' - '| journal=مطالعات اقتصاد انرژی ' - '| issue=3 ' - '| year=1383 ' - '| pages=55–55 ' - '| url=http://www.noormags.ir/view/fa/articlepage/105489 ' - '| language=fa ' - '| ref=harv ' - '| access-date=' - ) - self.assertIn(e, o[1]) - - def test_nm3(self): - """Reftag check.""" - o = noormags_sfn_cit_ref( - 'http://www.noormags.ir/view/fa/articlepage/' - '692447?sta=%D8%AF%D8%B9%D8%A7%DB%8C%20%D8%A7%D8%A8%D9%88%D8%AD%' - 'D9%85%D8%B2%D9%87%20%D8%AB%D9%85%D8%A7%D9%84%DB%8C' - ) - self.assertIn( - '{{sfn | سلیمانی\u200cمیمند | 1389 | pp=103–124}}', o[0] - ) - self.assertIn( - '* {{cite journal ' - '| last=سلیمانی\u200cمیمند ' - '| first=\u200cمریم ' - '| title=بررسی فضایل قرآنی در دعای ابوحمزه ثمالی ' - '| journal=بینات (موسسه معارف اسلامی امام رضا علیه السلام) ' - '| issue=68 ' - '| year=1389 ' - '| pages=103–124 ' - '| url=http://www.noormags.ir/view/fa/articlepage/692447 ' - '| language=fa ' - '| ref=harv ' - '| access-date=', o[1]) - self.assertIn( - '<ref name="سلیمانی\u200cمیمند 1389 pp. 103–124">' - '{{cite journal ' - '| last=سلیمانی\u200cمیمند ' - '| first=\u200cمریم ' - '| title=بررسی فضایل قرآنی در دعای ابوحمزه ثمالی ' - '| journal=بینات (موسسه معارف اسلامی امام رضا علیه السلام) ' - '| issue=68 ' - '| year=1389 ' - '| pages=103–124 ' - '| url=http://www.noormags.ir/view/fa/articlepage/692447 ' - '| language=fa ' - '| access-date=', o[2]) - - -if __name__ == '__main__': - main() +def test_nm2(): + """Reftag check.""" + o = noormags_scr( + 'http://www.noormags.ir/view/fa/articlepage/' + '692447?sta=%D8%AF%D8%B9%D8%A7%DB%8C%20%D8%A7%D8%A8%D9%88%D8%AD%' + 'D9%85%D8%B2%D9%87%20%D8%AB%D9%85%D8%A7%D9%84%DB%8C' + ) + assert '{{sfn | سلیمانی\u200cمیمند | 1389 | pp=103–124}}' in o[0] + assert ( + '* {{cite journal ' + '| last=سلیمانی\u200cمیمند ' + '| first=\u200cمریم ' + '| title=بررسی فضایل قرآنی در دعای ابوحمزه ثمالی ' + '| journal=بینات (موسسه معارف اسلامی امام رضا علیه السلام) ' + '| issue=68 ' + '| year=1389 ' + '| pages=103–124 ' + '| url=http://www.noormags.ir/view/fa/articlepage/692447 ' + '| language=fa ' + '| access-date=') in o[1] + assert ( + '<ref name="سلیمانی\u200cمیمند 1389 pp. 103–124">' + '{{cite journal ' + '| last=سلیمانی\u200cمیمند ' + '| first=\u200cمریم ' + '| title=بررسی فضایل قرآنی در دعای ابوحمزه ثمالی ' + '| journal=بینات (موسسه معارف اسلامی امام رضا علیه السلام) ' + '| issue=68 ' + '| year=1389 ' + '| pages=103–124 ' + '| url=http://www.noormags.ir/view/fa/articlepage/692447 ' + '| language=fa ' + '| access-date=') in o[2] diff --git a/test/pubmed_test.py b/test/pubmed_test.py index dab4d1dc..4a8f79d8 100644 --- a/test/pubmed_test.py +++ b/test/pubmed_test.py @@ -1,70 +1,55 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- - -"""Test noormags.py module.""" - - -from unittest import main, TestCase from unittest.mock import patch, Mock from lib import pubmed -from lib.pubmed import pmid_sfn_cit_ref, pmcid_sfn_cit_ref - - -class PMCID(TestCase): - - """Test pmcid_sfn_cit_ref.""" - - def test_doi_update(self): - """Updated using doi.""" - self.assertIn( - '* {{cite journal ' - '| last=Sweetser ' - '| first=Seth ' - '| title=Evaluating the Patient With Diarrhea:' - ' A Case-Based Approach ' - '| journal=Mayo Clinic Proceedings ' - '| publisher=Elsevier BV ' - '| volume=87 ' - '| issue=6 ' - '| year=2012 ' - '| issn=0025-6196 ' - '| pmid=22677080 ' - '| pmc=3538472 ' - '| doi=10.1016/j.mayocp.2012.02.015 ' - '| pages=596–602 ' - '| ref=harv}}', - pmcid_sfn_cit_ref('3538472')[1], - ) - - def test_spanish_no_doi(self): - """Test retrieval without doi.""" - self.assertIn( - '* {{cite journal | last=Mendozo Hernández | first=P ' - '| title=&#91;Clinical diagnosis and therapy.' - ' Intravenous and oral rehydration&#93;. ' - '| journal=Boletin de la Oficina Sanitaria Panamericana.' - ' Pan American Sanitary Bureau | volume=78 | issue=4 | year=1975 ' - '| issn=0030-0632 | pmid=123455 | pages=307–17 | language=es ' - '| ref=harv}}', - pmid_sfn_cit_ref('123455')[1], - ) - - @patch.object(pubmed, 'crossref_update', Mock(return_value=None)) - def test_has_doi_but_no_crossref(self): - """Test while doi exists but crossref_update is disabled.""" - self.assertIn( - '* {{cite journal | last=Bannen | first=RM | last2=Suresh ' - '| first2=V | last3=Phillips | first3=GN Jr | last4=Wright ' - '| first4=SJ | last5=Mitchell | first5=JC ' - '| title=Optimal design of thermally stable proteins ' - '| journal=Bioinformatics | volume=24 | issue=20 ' - '| date=22 August 2008 | pmid=18723523 | pmc=2562006 ' - '| doi=10.1093/bioinformatics/btn450 | pages=2339–2343 ' - '| ref=harv}}', - pmcid_sfn_cit_ref('2562006', '%d %B %Y')[1], - ) - - -if __name__ == '__main__': - main() +from lib.pubmed import pmid_dict, pmcid_dict +from lib.commons import dict_to_sfn_cit_ref + + +pmid_scr = lambda *args: dict_to_sfn_cit_ref(pmid_dict(*args)) +pmcid_scr = lambda *args: dict_to_sfn_cit_ref(pmcid_dict(*args)) + + +def test_doi_update(): + """Updated using doi.""" + assert ( + '* {{cite journal ' + '| last=Sweetser ' + '| first=Seth ' + '| title=Evaluating the Patient With Diarrhea:' + ' A Case-Based Approach ' + '| journal=Mayo Clinic Proceedings ' + '| publisher=Elsevier BV ' + '| volume=87 ' + '| issue=6 ' + '| year=2012 ' + '| issn=0025-6196 ' + '| pmid=22677080 ' + '| pmc=3538472 ' + '| doi=10.1016/j.mayocp.2012.02.015 ' + '| pages=596–602}}') in pmcid_scr('3538472')[1] + + +def test_spanish_no_doi(): + """Test retrieval without doi.""" + assert ( + '* {{cite journal | last=Mendozo Hernández | first=P ' + '| title=[Clinical diagnosis and therapy.' + ' Intravenous and oral rehydration]. ' + '| journal=Boletin de la Oficina Sanitaria Panamericana.' + ' Pan American Sanitary Bureau | volume=78 | issue=4 | year=1975 ' + '| issn=0030-0632 | pmid=123455 | pages=307–17 | language=es}}' + ) in pmid_scr('123455')[1] + + +@patch.object(pubmed, 'crossref_update', Mock(return_value=None)) +def test_has_doi_but_no_crossref(): + """Test while doi exists but crossref_update is disabled.""" + assert ( + '* {{cite journal | last=Bannen | first=RM | last2=Suresh ' + '| first2=V | last3=Phillips | first3=GN Jr | last4=Wright ' + '| first4=SJ | last5=Mitchell | first5=JC ' + '| title=Optimal design of thermally stable proteins ' + '| journal=Bioinformatics | volume=24 | issue=20 ' + '| date=22 August 2008 | pmid=18723523 | pmc=2562006 ' + '| doi=10.1093/bioinformatics/btn450 | pages=2339–2343}}' + ) in pmcid_scr('2562006', '%d %B %Y')[1] diff --git a/test/test_app.py b/test/test_app.py new file mode 100644 index 00000000..6c12317c --- /dev/null +++ b/test/test_app.py @@ -0,0 +1,79 @@ +from urllib.parse import urlparse +from unittest.mock import patch + +# noinspection PyPackageRequirements +from pytest import raises +from requests import JSONDecodeError + +from app import ( + input_to_dict, TLDLESS_NETLOC_RESOLVER, google_books_dict, + noorlib_url_to_dict, noormags_url_to_dict, google_encrypted_dict +) + + +def fake_resolver(*_): + raise NotImplementedError + + +def assert_and_patch_resolver(url, resolver): + d = TLDLESS_NETLOC_RESOLVER.__self__ + netloc = urlparse('http://' + url).netloc + if netloc[:4] == 'www.': + netloc = netloc[4:] + tldless_netloc = netloc.rpartition('.')[0] + assert d[tldless_netloc] == resolver + return patch.dict(d, {tldless_netloc: fake_resolver}) + + +def assert_scr(url, resolver): + with assert_and_patch_resolver(url, resolver), raises(NotImplementedError): + input_to_dict(url, '%Y-%m-%d') + + +def assert_google_books_scr(url, resolver=google_books_dict): + assert_scr(url, resolver) + + +def test_google_books_netloc(): + ag = assert_google_books_scr + # note that top level domains are ignored + ag('encrypted.google.com/books?id=6upvonUt0O8C', google_encrypted_dict) + ag('books.google.com/books?id=pzmt3pcBuGYC') + ag('books.google.de/books?id=pzmt3pcBuGYC') + ag('books.google.com.ar/books?id=pzmt3pcBuGYC') + ag('books.google.co.il/books?id=pzmt3pcBuGYC') + with patch('app.google_books_dict') as mock: + input_to_dict('www.google.com/books?id=bwfoCAAAQBAJ', None) + input_to_dict('www.google.com/books/edition/_/bwfoCAAAQBAJ', None) + assert mock.call_count == 2 + + +def assert_noormags_scr(url): + assert_scr(url, noormags_url_to_dict) + + +def test_noormags(): + an = assert_noormags_scr + an('www.noormags.ir/view/fa/articlepage/105489/') + an('www.noormags.net/view/fa/articlepage/105489/') + an('noormags.ir/view/fa/articlepage/105489/') + + +def assert_noorlib_scr(url): + assert_scr(url, noorlib_url_to_dict) + + +def test_noorlib(): + an = assert_noorlib_scr + an('www.noorlib.ir/View/fa/Book/BookView/Image/6120') + an('www.noorlib.net/View/fa/Book/BookView/Image/6120') + an('noorlib.ir/View/fa/Book/BookView/Image/6120') + + +@patch('app.urls_url_to_dict') +@patch('app.doi_to_dict', side_effect=JSONDecodeError('msg', 'doc', 1)) +def test_doi_url_fallback_to_url(doi_scr, urls_scr): + user_input = 'https://dl.acm.org/doi/10.5555/3157382.3157535' + assert input_to_dict(user_input, '%B %#d, %Y') is urls_scr.return_value + doi_scr.assert_called_once_with('10.5555/3157382.3157535', True, '%B %#d, %Y') + urls_scr.assert_called_once_with(user_input, '%B %#d, %Y') diff --git a/test/test_en.py b/test/test_en.py deleted file mode 100644 index dd903ebd..00000000 --- a/test/test_en.py +++ /dev/null @@ -1,17 +0,0 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- - -"""Discover all tests (except the ones inside test_fa.py) and run them.""" - - -from unittest import defaultTestLoader -from unittest.runner import TextTestRunner - -import config - - -if __name__ == '__main__': - config.LANG = 'en' - tests = defaultTestLoader.discover('.', '*_test.py') - runner = TextTestRunner() - runner.run(tests) diff --git a/test/test_fa.py b/test/test_fa.py index b51c41e6..ff02d4c6 100644 --- a/test/test_fa.py +++ b/test/test_fa.py @@ -1,360 +1,312 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- - -from unittest import TestCase, main - -import config; config.LANG = 'fa' - -from lib.adinebook import adinehbook_sfn_cit_ref -from lib.doi import doi_sfn_cit_ref -from lib.googlebooks import googlebooks_sfn_cit_ref -from lib.isbn_oclc import isbn_sfn_cit_ref -from lib.noormags import noormags_sfn_cit_ref -from lib.noorlib import noorlib_sfn_cit_ref -from lib.pubmed import pmid_sfn_cit_ref - - -class AdinebookTest(TestCase): - - def test_ab1(self): - """authors = 1, translators = 2, otheo = 1, isbn13""" - self.assertIn( - '* {{یادکرد کتاب | نام خانوادگی=لانسکی |' - ' نام=ویکی | ترجمه=فیروزه دالکی و مژگان امیرفروغی |' - ' دیگران= کی وایت (تصویرگر) |' - ' عنوان=101 راه برای اینکه پدر بهتری باشید |' - ' ناشر=پیک ادبیات | سال=1386 |' - ' ماه=شهریور | شابک=978-964-8165-81-4 | زبان=fa}}', - adinehbook_sfn_cit_ref( - 'http://www.adinebook.com/gp/product/9648165814/ref' - '=sr_1_1000_42/905-6618179-9188955' - )[1], - ) - - def test_ab2(self): - """authors = 3, translators = 2, otheo = 0, isbn13""" - self.assertIn( - '* {{یادکرد کتاب |' - ' نام خانوادگی=کرسول | نام=جان | نام خانوادگی۲=کلارک |' - ' نام۲=ویکی پلانو | ترجمه=محسن نیازی و عباس زارعی |' - ' عنوان=روش های تحقیق تلفیقی | ناشر=ثامن الحجج | سال=1387 |' - ' ماه=خرداد | شابک=978-964-2823-35-2 | زبان=fa}}', - adinehbook_sfn_cit_ref( - 'http://www.adinebook.com/gp/product/9642823352/' - 'ref=sr_1_1000_41/905-6618179-9188955' - )[1] - ) - - def test_ab3(self): - """authors = 2, translators = 0, otheo = 4, isbn13""" - self.assertIn( - '* {{یادکرد کتاب | نام خانوادگی=فخررحیمی |' - ' نام=علیرضا | نام خانوادگی۲=فخررحیمی |' - ' نام۲=الهام |' - ' دیگران= آرش نادرپور (مقدمه)، امیر جابری (مقدمه)، ' - ' وحید شهبازیان (مقدمه) و رضا مقدم (مقدمه) |' - ' عنوان=آموزش گام به گام پیکربندی مسیریابهای میکروتیک:' - ' آمادگی آزمون MTCNA |' - ' ناشر=نشرگستر | سال=1391 |' - ' ماه=خرداد | شابک=978-600-5883-43-5 | زبان=fa}}', - adinehbook_sfn_cit_ref( - 'http://www.adinebook.com/gp/product/6005883435' - )[1] - ) - - def test_ab4(self): - """authors = 3, translators = 0, otheo = 0, isbn13""" - self.assertIn( - '* {{یادکرد کتاب | نام خانوادگی=کریمی | نام=نجمه |' - ' نام خانوادگی۲=یزدخواستی |' - ' نام۲=فروغ | نام خانوادگی۳=مختاری |' - ' نام۳=صفورا | عنوان=11 سپتامبر ... آرماگدون |' - ' ناشر=حدیث راه عشق | سال=1386 | ماه=شهریور |' - ' شابک=978-964-95633-4-3 | زبان=fa}}', - adinehbook_sfn_cit_ref( - 'http://www.adinebook.com/gp/product/9649563342/ref=ftr_1/' - '905-6618179-9188955' - )[1] - ) - - def test_ab5(self): - """Year is interesting here.""" - self.assertIn( - '* {{یادکرد کتاب | نام خانوادگی=نژاد | نام=یوسف علی یوسف |' - ' عنوان=فراهنجاری در مثنوی سرایی | ناشر=اردیبهشت | سال=1388 |' - ' شابک=978-964-2656-34-9 | زبان=fa}}', - adinehbook_sfn_cit_ref( - 'http://www.adinebook.com/gp/product/9642656349/' - )[1] - ) - - def test_ab6(self): - """Month and year detection.""" - self.assertIn( - '* {{یادکرد کتاب |' - ' نام خانوادگی=مونس |' - ' نام=حسین | ترجمه=حمیدرضا شیخی |' - ' عنوان=تاریخ و تمدن مغرب - جلد اول |' - ' ناشر=سازمان مطالعه و تدوین کتب علوم انسانی دانشگاهها (سمت) |' - ' سال=1392 | ماه=شهریور |' - ' شابک=978-964-530-036-2 |' - ' زبان=fa}}', - adinehbook_sfn_cit_ref( - 'http://www.adinebook.com/gp/product/9645300363/' - 'ref=pd_sim_b_title_4/905-6618179-9188955' - )[1], - ) - - def test_ab7(self): - """1 Editor.""" - self.assertIn( - '* {{یادکرد کتاب | نام خانوادگی=دیماتیو |' - ' نام=ام.رابین | نام خانوادگی ویراستار=جباری |' - ' نام ویراستار=کریم |' - ' ترجمه=محمد کاویانی | دیگران= کیانوش هاشمیان (زیرنظر) |' - ' عنوان=روانشناسی سلامت به ضمیمه نگرشی بر منابع اسلامی ' - '- جلد اول |' - ' ناشر=سازمان مطالعه و تدوین کتب علوم انسانی دانشگاهها (سمت) |' - ' سال=1392 | ماه=بهمن |' - ' شابک=978-964-459-398-7 |' - ' زبان=fa}}', - adinehbook_sfn_cit_ref( - 'http://www.adinebook.com/gp/product/9644593987/' - 'ref=pd_pos_b_title_4/905-6618179-9188955' - )[1] - ) - - -class GooglebookTest(TestCase): - - def test_gb1(self): - self.assertIn( - '* {{یادکرد کتاب | نام خانوادگی=Arms |' - ' نام=W.Y. | عنوان=Digital Libraries | ناشر=MIT Press |' - ' سری=Digital libraries and electronic publishing |' - ' سال=2001 | شابک=978-0-262-26134-0 |' - ' پیوند=https://books.google.com/books?id=pzmt3pcBuGYC&pg=PR11 |' - ' زبان=en | تاریخ بازبینی=', - googlebooks_sfn_cit_ref( - 'http://books.google.com/books?' - 'id=pzmt3pcBuGYC&pg=PR11&lpg=PP1&dq=digital+library' - )[1], - ) - - def test_gb2(self): - """a book with more than 4 authors (10 authors)""" - o = googlebooks_sfn_cit_ref( - 'http://books.google.com/books?id=' - 'U46IzqYLZvAC&pg=PT57#v=onepage&q&f=false') - self.assertIn( - '<ref>' - '{{پک | Anderson | DeBolt | Featherstone | Gunther | 2010' - ' | ک=InterACT with Web Standards: A' - ' holistic approach to web design | زبان=en | ص=57}}' - '\u200f</ref>', - o[0], - ) - self.assertIn( - '* {{یادکرد کتاب |' - ' نام خانوادگی=Anderson |' - ' نام=E. |' - ' نام خانوادگی۲=DeBolt | نام۲=V. |' - ' نام خانوادگی۳=Featherstone |' - ' نام۳=D. | نام خانوادگی۴=Gunther |' - ' نام۴=L. |' - ' نام خانوادگی۵=Jacobs | نام۵=D.R. | نام خانوادگی۶=Mills |' - ' نام۶=C. |' - ' نام خانوادگی۷=Schmitt | نام۷=C. | نام خانوادگی۸=Sims |' - ' نام۸=G. |' - ' نام خانوادگی۹=Walter | نام۹=A. |' - ' نام خانوادگی۱۰=Jensen-Inman |' - ' نام۱۰=L. |' - ' عنوان=InterACT with Web Standards:' - ' A holistic approach to web design |' - ' ناشر=Pearson Education |' - ' سری=Voices That Matter | سال=2010 |' - ' شابک=978-0-13-270490-8 |' - ' پیوند=https://books.google.com/books?id=U46IzqYLZvAC&pg=PT57 |' - ' زبان=en |' - ' تاریخ بازبینی=', - o[1], - ) - - def test_gb3(self): - """Non-ascii characters in title""" - o = googlebooks_sfn_cit_ref( - 'http://books.google.com/books?' - 'id=icMEAAAAQBAJ&pg=PA588&dq=%22a+Delimiter+is%22&hl=' - 'en&sa=X&ei=oNKSUrKeDovItAbO_4CoBA&ved=0CC4Q6AEwAA#v=' - 'onepage&q=%22a%20Delimiter%20is%22&f=false' - ) - self.assertIn( - '<ref>' - '{{پک | Farrell | 2009 ' - '| ک=Microsoft Visual C# 2008 Comprehensive: ' - 'An Introduction to Object-Oriented Programming |' - ' زبان=en | ص=588}}' - '\u200f</ref>', - o[0], - ) - self.assertIn( - '* {{یادکرد کتاب | نام خانوادگی=Farrell |' - ' نام=J. | عنوان=Microsoft Visual C# 2008 Comprehensive: ' - 'An Introduction to Object-Oriented Programming |' - ' ناشر=Cengage Learning | سال=2009 | شابک=978-1-111-78619-9 |' - ' پیوند=https://books.google.com/books?id=icMEAAAAQBAJ&pg=PA588 |' - ' زبان=en | تاریخ بازبینی=', - o[1], - ) - - def test_gb4(self): - """Non-ascii characters in author's name.""" - o = googlebooks_sfn_cit_ref( - 'http://books.google.com/books?id=' - 'i8nZjjo_9ikC&pg=PA229&dq=%22legal+translation+is%22&hl=en&sa=' - 'X&ei=hEuYUr_mOsnKswb49oDQCA&ved=0CC4Q6AEwAA#v=onepage&q=' - '%22legal%20translation%20is%22&f=false' - ) - self.assertIn( - '<ref>{{پک | Šarčević | 1997 ' - '| ک=New Approach to Legal Translation |' - ' زبان=en | ص=229}}' - '\u200f</ref>', - o[0], - ) - self.assertIn( - '* {{یادکرد کتاب | نام خانوادگی=Šarčević |' - ' نام=S. |' - ' عنوان=New Approach to Legal Translation |' - ' ناشر=Springer Netherlands |' - ' سال=1997 |' - ' شابک=978-90-411-0401-4 |' - ' پیوند=https://books.google.com/books?id=i8nZjjo_9ikC&pg=PA229 |' - ' زبان=en |' - ' تاریخ بازبینی=', - o[1], - ) - - -class NoormagsTest(TestCase): - - def test_nm2(self): - self.assertIn( - '* {{یادکرد ژورنال |' - ' عنوان=زندگی نامه علمی دکتر کاووس حسن لی |' - ' ژورنال=شعر | شماره=62 | سال=1387 | صفحه=17–19 |' - ' پیوند=https://www.noormags.ir/view/fa/articlepage/454096 |' - ' زبان=fa | تاریخ بازبینی=', - noormags_sfn_cit_ref( - 'http://www.noormags.com/view/fa/ArticlePage/454096' - )[1], - ) - - -class NoorlibTest(TestCase): - - def test_nl1(self): - i = 'http://www.noorlib.ir/View/fa/Book/BookView/Image/3232' - o = noorlib_sfn_cit_ref(i) - e = ( - '* {{یادکرد کتاب ' - '| نام خانوادگی=ابن اثیر ' - '| نام=علی بن محمد ' - '| عنوان=الكامل في التاريخ ' - '| ناشر=دار صادر ' - '| مکان=بیروت - لبنان ' - '| سری=الكامل في التاريخ ' - '| جلد=13 ' - '| پیوند=https://www.noorlib.ir/View/fa/Book/BookView/Image/3232 ' - '| زبان=عربی ' - '| تاریخ بازبینی=' - ) - self.assertIn(e, o[1]) - - -class DoiTest(TestCase): - - def test_di1(self): - self.maxDiff = None - # Note: Language detection is wrong, it should be en - self.assertIn( - "* {{یادکرد ژورنال | نام خانوادگی=Atkins |" - " نام=Joshua H. | نام خانوادگی۲=Gershell | نام۲=Leland J. |" - " عنوان=Selective anticancer drugs |" - " ژورنال=Nature Reviews Drug Discovery |" - " ناشر=Springer Nature | جلد=1 | شماره=7 |" - " سال=2002 | ماه=7 | issn=1474-1776 | doi=10.1038/nrd842 |" - " صفحه=491–492 |" - " زبان=da}}", - doi_sfn_cit_ref('http://dx.doi.org/10.1038/nrd842')[1], - ) - - -class IsbnTest(TestCase): - - def test_is1(self): - """not found in adinebook""" - self.assertIn( - '* {{یادکرد کتاب | نام خانوادگی=Adkins | نام=Roy |' - ' عنوان=The war for all the oceans : ' - 'from Nelson at the Nile to Napoleon at Waterloo |' - ' ناشر=Abacus | مکان=London | سال=2007 |' - ' شابک=978-0-349-11916-8 ' - '| oclc=137313052 ' - '| زبان=en}}', - isbn_sfn_cit_ref('9780349119168', pure=True)[1], - ) - - def test_is2(self): - """not found in ottobib""" - self.assertEqual( - '* {{یادکرد کتاب | نام خانوادگی=منصور |' - ' نام=جهانگیر | دیگران=بدیل بن علی خاقانی (شاعر)' - ' و بدیع الزمان فروزانفر (مقدمه) |' - ' عنوان=دیوان خاقانی شروانی | ناشر=نگاه |' - ' سال=1389 | ماه=مرداد | شابک=978-964-6736-71-9 | زبان=fa}}', - isbn_sfn_cit_ref('978-964-6736-71-9', pure=True)[1] - ) - - def test_is3(self): - """exists in both""" - self.assertEqual( - '* {{یادکرد کتاب | نام خانوادگی=معصومی | نام=سحر |' - ' عنوان=راز گل سرخ: نقد و گزیده شعرهای سهراب سپهری |' - ' ناشر=نگاه | سال=1386 | ماه=بهمن |' - ' شابک=964-6736-34-3 ' - '| oclc=53446327 ' - '| زبان=fa}}', - isbn_sfn_cit_ref('964-6736-34-3 ')[1], - ) - - def test_is4(self): - """unpure isbn10""" - self.assertEqual( - '* {{یادکرد کتاب | نام خانوادگی=حافظ | نام=شمس الدین محمد |' - ' نام خانوادگی۲=نظرزاده | نام۲=رضا |' - ' عنوان=دیوان کامل حافظ همراه با فالنامه |' - ' ناشر=دیوان | سال=1385 |' - ' ماه=آذر | شابک=964-92962-6-3 | زبان=fa}}', - isbn_sfn_cit_ref('choghondar 964-92962-6-3 شلغم')[1] - ) - - def test_2letter_langcode(self): - """Test that 3letter language code is converted to a 2-letter one.""" - # Todo: The fawiki template mixes persian and chinese characters... - self.assertIn( - '* {{یادکرد ژورنال | نام خانوادگی=Huang | نام=Y ' - '| نام خانوادگی۲=Lu | نام۲=J | نام خانوادگی۳=Shen ' - '| نام۳=Y | نام خانوادگی۴=Lu | نام۴=J ' - '| عنوان=&#91;The protective effects of total flavonoids from ' - 'Lycium Barbarum L. on lipid peroxidation of liver mitochondria ' - 'and red blood cell in rats&#93;. ' - '| ژورنال=Wei sheng yan jiu = Journal of hygiene research ' - '| جلد=28 | شماره=2 | تاریخ=1999-03-30 | issn=1000-8020 ' - '| pmid=11938998 | صفحه=115–6 | زبان=zh}}', - pmid_sfn_cit_ref('11938998')[1], - ) - - -if __name__ == '__main__': - main() +from datetime import date +from unittest.mock import patch + +from lib.generator_fa import sfn_cit_ref +from lib.commons import find_any_date + +from test.googlebooks_test import googlebooks_scr +from test.noormags_test import noormags_scr +from test.noorlib_test import noorlib_scr +from test.ketabir_test import ketabir_scr +from test.doi_test import doi_scr +from test.isbn_oclc_test import isbn_scr +from test.pubmed_test import pmid_scr +from test.urls_test import urls_scr + + +sfn_cit_ref_patcher = patch('lib.commons.sfn_cit_ref', sfn_cit_ref) +doi_patcher = patch('lib.doi.LANG', 'fa') +isbn_oclc_patcher = patch('lib.isbn_oclc.LANG', 'fa') + + +def setup_module(): + sfn_cit_ref_patcher.start() + doi_patcher.start() + isbn_oclc_patcher.start() + + +def test_ketabir1(): + """authors = 1, translators = 2, otheo = 1, isbn13""" + assert ( + '* {{یادکرد کتاب | نام خانوادگی=لانسکی |' + ' نام=ویکی | ترجمه=فیروزه دالکی و مژگان امیرفروغی |' + ' دیگران=کی وایت (تصويرگر) |' + ' عنوان=101 راه برای اینکه پدر بهتری باشید |' + ' ناشر=پیک ادبیات | مکان=تهران - تهران |' + ' سال=1386 | شابک=978-964-8165-81-4 | زبان=fa}}' + ) == ketabir_scr('https://ketab.ir/book/27b3444f-1175-4db0-8411-b1719a5d7ed1')[1] + + +def test_ketabir2(): + """authors = 3, translators = 2, otheo = 0, isbn13""" + assert ( + '* {{یادکرد کتاب | نام خانوادگی=کرسول | نام=جان | نام ' + 'خانوادگی۲=پلانو\u200cکلارک | نام۲=ویکی | ترجمه=محسن نیازی و عباس زارعی | ' + 'عنوان=روش\u200cهای تحقیق تلفیقی | ناشر=ثامن الحجج | مکان=تهران - تهران | ' + 'جلد=1 | سال=1387 | شابک=978-964-2823-35-2 | زبان=fa}}' + ) == ketabir_scr('https://ketab.ir/book/667c900a-69bd-4a1a-a651-1870d2f63a68')[1] + + +def test_ketabir3(): + """authors = 2, translators = 0, otheo = 4, isbn13""" + assert ( + '* {{یادکرد کتاب | نام خانوادگی=فخررحیمی |' + ' نام=علیرضا | نام خانوادگی۲=فخررحیمی |' + ' نام۲=الهام | دیگران=آرش نادرپور' + ' (مقدمه)، وحید شهبازیان (مقدمه)، رضا مقدم (مقدمه) ' + 'و امیر جابری (مقدمه) | عنوان=آموزش گام' + ' به گام پیکربندی مسیریابهای میکروتیک: آمادگی آزمون MTCNA ' + '| ناشر=نشرگستر | مکان=تهران - تهران |' + ' سال=1391 | شابک=978-600-5883-43-5 | زبان=fa}}' + ) == ketabir_scr('https://ketab.ir/book/f37fad8e-8f0b-4cd9-8875-f5de0e0d86ef')[1] + + +def test_ketabir4(): + """authors = 3, translators = 0, otheo = 0, isbn13""" + assert ( + '* {{یادکرد کتاب | نام خانوادگی=کریمی |' + ' نام=نجمه | نام خانوادگی۲=یزدخواستی |' + ' نام۲=فروغ | نام خانوادگی۳=مختاری |' + ' نام۳=صفورا | عنوان=11 سپتامبر ... آرماگدون |' + ' ناشر=حدیث راه عشق | مکان=اصفهان - اصفهان |' + ' سال=1386 | شابک=978-964-95633-4-3 | زبان=fa}}' + ) == ketabir_scr('https://ketab.ir/book/13a52229-5e3f-479e-8954-092b65e85923')[1] + + +def test_ketabir5(): + """Year is interesting here.""" + assert ( + '* {{یادکرد کتاب | نام خانوادگی=یوسف\u200cنژاد | نام=یوسف\u200cعلی | ' + 'عنوان=فراهنجاری در مثنوی\u200cسرایی: بررسی قالب غزل - مثنوی در ادب فارسی | ' + 'ناشر=هنر رسانه اردیبهشت | مکان=تهران - تهران | سال=1388 | ' + 'شابک=978-964-2656-34-9 | زبان=fa}}' + ) == ketabir_scr('https://ketab.ir/book/a5958832-5c43-460d-bf42-65acb6077e52')[1] + + +def test_ketabir6(): + """Month and year detection.""" + assert ( + '* {{یادکرد کتاب | نام خانوادگی=مونس | نام=حسین | ترجمه=حمیدرضا شیخی | ' + 'عنوان=تاریخ و تمدن مغرب | ناشر=سمت | مکان=مشهد - خراسان رضوی | جلد=1 | ' + 'سال=1390 | شابک=978-964-530-036-2 | زبان=fa}}' + ) == ketabir_scr('https://ketab.ir/book/cb1989dc-ba09-4df6-aaee-fcdbd25ad322')[1] + + +def test_ketabir7(): + """1 Editor.""" + assert ( + '* {{یادکرد کتاب | نام خانوادگی=دیماتیو | نام=ام.رابین | نام خانوادگی ' + 'ویراستار=جباری | نام ویراستار=کریم | ترجمه=محمد کاویانی | دیگران=کیانوش ' + 'هاشمیان (زيرنظر) | عنوان=روانشناسی سلامت به ضمیمه نگرشی بر منابع اسلامی | ' + 'ناشر=سمت | مکان=تهران - تهران | جلد=1 | سال=1379 | شابک=978-964-459-398-7 | ' + 'زبان=fa}}' + ) == ketabir_scr('https://ketab.ir/book/4cc231f9-35c2-4b60-a714-a0a11135e932')[1] + + +def test_google_books_ending_page(): + assert googlebooks_scr( + 'https://www.google.com/books/edition/So_You_Want_to_Sing_World_Music/OlCwDwAAQBAJ?hl=en&gbpv=1&dq=Darya+Dadvar&pg=PA293&printsec=frontcover' + )[2][-25:] == '| صفحه=293}}‏</ref>' + + +def test_google_books_1(): + assert ( + '* {{یادکرد کتاب | نام خانوادگی=Arms |' + ' نام=W.Y. | عنوان=Digital Libraries | ناشر=MIT Press |' + ' سری=Digital Libraries and Electronic Publishing |' + ' سال=2001 | شابک=978-0-262-26134-0 |' + ' پیوند=https://books.google.com/books?id=pzmt3pcBuGYC&pg=PR11 |' + ' زبان=en | تاریخ بازبینی=') in googlebooks_scr( + 'http://books.google.com/books?' + 'id=pzmt3pcBuGYC&pg=PR11&lpg=PP1&dq=digital+library')[1] + + +def test_google_books2(): + """a book with more than 4 authors (10 authors)""" + o = googlebooks_scr( + 'http://books.google.com/books?id=' + 'U46IzqYLZvAC&pg=PT57#v=onepage&q&f=false') + assert ( + '<ref>' + '{{پک | Anderson | DeBolt | Featherstone | Gunther | 2010' + ' | ک=InterACT with Web Standards: A' + ' holistic approach to web design | زبان=en | ص=57}}' + '\u200f</ref>') in o[0] + assert ( + '* {{یادکرد کتاب |' + ' نام خانوادگی=Anderson |' + ' نام=E. |' + ' نام خانوادگی۲=DeBolt | نام۲=V. |' + ' نام خانوادگی۳=Featherstone |' + ' نام۳=D. | نام خانوادگی۴=Gunther |' + ' نام۴=L. |' + ' نام خانوادگی۵=Jacobs | نام۵=D.R. | نام خانوادگی۶=Mills |' + ' نام۶=C. |' + ' نام خانوادگی۷=Schmitt | نام۷=C. | نام خانوادگی۸=Sims |' + ' نام۸=G. |' + ' نام خانوادگی۹=Walter | نام۹=A. |' + ' نام خانوادگی۱۰=Jensen-Inman |' + ' نام۱۰=L. |' + ' عنوان=InterACT with Web Standards:' + ' A holistic approach to web design |' + ' ناشر=Pearson Education |' + ' سری=Voices That Matter | سال=2010 |' + ' شابک=978-0-13-270490-8 |' + ' پیوند=https://books.google.com/books?id=U46IzqYLZvAC&pg=PT57 |' + ' زبان=en |' + ' تاریخ بازبینی=') in o[1] + + +def test_google_books3(): + """Non-ascii characters in title""" + o = googlebooks_scr( + 'http://books.google.com/books?' + 'id=icMEAAAAQBAJ&pg=PA588&dq=%22a+Delimiter+is%22&hl=' + 'en&sa=X&ei=oNKSUrKeDovItAbO_4CoBA&ved=0CC4Q6AEwAA#v=' + 'onepage&q=%22a%20Delimiter%20is%22&f=false' + ) + assert ( + '<ref>' + '{{پک | Farrell | 2009 ' + '| ک=Microsoft Visual C# 2008 Comprehensive: ' + 'An Introduction to Object-Oriented Programming |' + ' زبان=en | ص=588}}' + '\u200f</ref>') in o[0] + assert ( + '* {{یادکرد کتاب | نام خانوادگی=Farrell |' + ' نام=J. | عنوان=Microsoft Visual C# 2008 Comprehensive: ' + 'An Introduction to Object-Oriented Programming |' + ' ناشر=Cengage Learning | سال=2009 | شابک=978-1-111-78619-9 |' + ' پیوند=https://books.google.com/books?id=icMEAAAAQBAJ&pg=PA588 |' + ' زبان=en | تاریخ بازبینی=') in o[1] + + +def test_google_books4(): + """Non-ascii characters in author's name.""" + o = googlebooks_scr( + 'http://books.google.com/books?id=i8nZjjo_9ikC&pg=PA229&dq=%22legal+translation+is%22&hl=en&sa=X&ei=hEuYUr_mOsnKswb49oDQCA&ved=0CC4Q6AEwAA#v=onepage&q=%22legal%20translation%20is%22&f=false') + assert ( + '<ref>{{پک | Sarcevic | \x8aar?evi? | 1997 | ک=New Approach to Legal Translation | زبان=en | ص=229}}\u200f</ref>' + ) == o[0] + assert ( + '* {{یادکرد کتاب | نام خانوادگی=Sarcevic | نام=S. | نام خانوادگی۲=\x8aar?evi? | نام۲=S. | عنوان=New Approach to Legal Translation | ناشر=Springer Netherlands | سال=1997 | شابک=978-90-411-0401-4 | پیوند=https://books.google.com/books?id=i8nZjjo_9ikC&pg=PA229 | زبان=en | تاریخ بازبینی=' + ) in o[1] + + +def test_noormags1(): + assert ( + '* {{یادکرد ژورنال |' + ' عنوان=زندگی نامه علمی دکتر کاووس حسن لی |' + ' ژورنال=شعر | شماره=62 | سال=1387 | صفحه=17–19 |' + ' پیوند=https://www.noormags.ir/view/fa/articlepage/454096 |' + ' زبان=fa | تاریخ بازبینی=' + ) in noormags_scr('http://www.noormags.com/view/fa/ArticlePage/454096')[1] + + +def test_noorlib1(): + i = 'http://www.noorlib.ir/View/fa/Book/BookView/Image/3232' + o = noorlib_scr(i) + e = ( + '* {{یادکرد کتاب ' + '| نام خانوادگی=ابن اثیر ' + '| نام=علی بن محمد ' + '| عنوان=الكامل في التاريخ ' + '| ناشر=دار صادر ' + '| مکان=بیروت - لبنان ' + '| سری=الكامل في التاريخ ' + '| جلد=13 ' + '| پیوند=https://www.noorlib.ir/View/fa/Book/BookView/Image/3232 ' + '| زبان=عربی ' + '| تاریخ بازبینی=' + ) + assert e in o[1] + + +def test_doi1(): + # Note: Language detection is wrong, it should be en + assert ( + "* {{یادکرد ژورنال | نام خانوادگی=Atkins |" + " نام=Joshua H. | نام خانوادگی۲=Gershell | نام۲=Leland J. |" + " عنوان=Selective anticancer drugs |" + " ژورنال=Nature Reviews Drug Discovery |" + " ناشر=Springer Science and Business Media LLC " + "| جلد=1 | شماره=7 |" + " سال=2002 | issn=1474-1776 | doi=10.1038/nrd842 |" + " صفحه=491–492 |" + " زبان=da}}" + ) in doi_scr('http://dx.doi.org/10.1038/nrd842')[1] + + +def test_isbn_exists_on_ottobib_not_ketabir(): + assert ( + '* {{یادکرد کتاب | نام خانوادگی=Adkins ' + '| نام=Roy ' + '| نام خانوادگی۲=Adkins ' + '| نام۲=Lesley ' + '| عنوان=The war for all the oceans : from Nelson at the Nile to ' + 'Napoleon at Waterloo ' + '| ناشر=Abacus ' + '| مکان=London ' + '| تاریخ=2007 ' + '| شابک=978-0-349-11916-8 ' + '| oclc=137313052 ' + '| زبان=en}}' + ) in isbn_scr('9780349119168', True)[1] + + +def test_isbn_exists_on_ketabir_not_ottobib(): + assert ( + '* {{یادکرد کتاب | دیگران=بدیل\u200cبن\u200cعلی خاقانی (شاعر)، جهانگیر منصور ' + '(به\u200cاهتمام) و محمدحسن فروزانفر (مقدمه) | عنوان=دیوان خاقانی شروانی | ' + 'ناشر=نگاه | مکان=تهران - تهران | سال=1396 | شابک=978-964-6736-71-9 | ' + 'oclc=1176150182 | زبان=fa}}' + ) == isbn_scr('978-964-6736-71-9', True)[1] + + +def test_isbn_exists_on_ketabir_and_ottobib(): + assert ( + '* {{یادکرد کتاب | دیگران=سحر معصومی (به\u200cاهتمام) | عنوان=راز گل سرخ: نقد ' + 'و گزیده شعرهای سهراب سپهری | ناشر=نگاه | مکان=تهران - تهران | سال=1386 | ' + 'شابک=978-964-6736-34-4 | oclc=53446327 | زبان=fa}}' + ) == isbn_scr('964-6736-34-3 ')[1] + + +def test_isbn_unpure_input(): + assert ( + '* {{یادکرد کتاب | نام خانوادگی=حافظ | نام=شمس\u200cالدین\u200cمحمد | ' + 'دیگران=رضا نظرزاده (به\u200cاهتمام) | عنوان=دیوان کامل حافظ همراه با فالنامه ' + '| ناشر=دیوان | مکان=قم - قم | سال=1385 | شابک=978-964-92962-6-5 | زبان=fa}}' + ) == isbn_scr('choghondar 964-92962-6-3 شلغم')[1] + + +def test_2letter_langcode(): + """Test that 3letter language code is converted to a 2-letter one.""" + # Todo: The fawiki template mixes Persian and Chinese characters... + assert ( + '* {{یادکرد ژورنال | نام خانوادگی=Huang | نام=Y ' + '| نام خانوادگی۲=Lu | نام۲=J | نام خانوادگی۳=Shen ' + '| نام۳=Y | نام خانوادگی۴=Lu | نام۴=J ' + '| عنوان=[The protective effects of total flavonoids from ' + 'Lycium Barbarum L. on lipid peroxidation of liver mitochondria ' + 'and red blood cell in rats]. ' + '| ژورنال=Wei sheng yan jiu = Journal of hygiene research ' + '| جلد=28 | شماره=2 | تاریخ=1999-03-30 | issn=1000-8020 ' + '| pmid=11938998 | صفحه=115–6 | زبان=zh}}' + ) in pmid_scr('11938998')[1] + + +def test_either_year_or_date(): + assert urls_scr( + 'https://www.shora-gc.ir/fa/news/1815/%D8%A7%D8%B5%D9%84-%D9%87%D9%81%D8%AA%D8%A7%D8%AF-%D9%88-%D8%B3%D9%88%D9%85' + )[1][:-12] == '* {{یادکرد وب | عنوان=اصل هفتاد و سوم | وبگاه=پایگاه اطلاع رسانی شورای نگهبان - shora-gc.ir | تاریخ=2020-06-07 | پیوند=http://www.shora-gc.ir/fa/news/1815 | کد زبان=fa | تاریخ بازبینی=' + + +def test_arabic_ya(): + assert find_any_date('تاریخ انتشار : جمعه ۳ ارديبهشت ۱۳۸۹ ساعت ۱۶:۴۸') == \ + date(2010, 4, 23) + + +def teardown_module(): + sfn_cit_ref_patcher.stop() + doi_patcher.stop() + isbn_oclc_patcher.stop() diff --git a/test/testdata/04e637517bb684a776a7980127d922cd9d8b17f2.html b/test/testdata/04e637517bb684a776a7980127d922cd9d8b17f2.html new file mode 100644 index 00000000..ed79c72f --- /dev/null +++ b/test/testdata/04e637517bb684a776a7980127d922cd9d8b17f2.html @@ -0,0 +1,13 @@ +@article{10.2307/40991855, + ISSN = {00310581}, + URL = {http://www.jstor.org/stable/40991855}, + author = {Carlos Augusto de Figueiredo Monteiro}, + journal = {Revista Geográfica}, + number = {63}, + pages = {173--178}, + publisher = {Pan American Institute of Geography and History}, + title = {“Calamidades Meteorológicas no Brasil Meridional, em Agôsto de 1965”}, + volume = {35}, + year = {1965} +} + diff --git a/test/testdata/04e637517bb684a776a7980127d922cd9d8b17f2.json b/test/testdata/04e637517bb684a776a7980127d922cd9d8b17f2.json new file mode 100644 index 00000000..18851fc1 --- /dev/null +++ b/test/testdata/04e637517bb684a776a7980127d922cd9d8b17f2.json @@ -0,0 +1,24 @@ +{ + "encoding": "ISO-8859-1", + "headers": { + "Accept-Ranges": "bytes", + "Connection": "keep-alive", + "Content-Disposition": "attachment;filename=10.2307_40991855.txt;", + "Content-Encoding": "gzip", + "Content-Type": "text/plain", + "Date": "Fri, 18 Jun 2021 07:00:02 GMT", + "Server": "Apache/2.4.29 (Ubuntu)", + "Set-Cookie": "ReferringRequestId=citation-export:65fcfcb30d0706992bb44df127c97f9f; Path=/; SameSite=Lax; Secure", + "Vary": "Cookie,Accept-Encoding,Fastly-SSL,Origin,X-Requested-Host", + "Via": "1.1 varnish", + "X-Cache": "MISS", + "X-Cache-Hits": "0", + "X-Frame-Options": "SAMEORIGIN", + "X-JSTOR-Restarts": "0", + "X-Served-By": "cache-fra19158-FRA", + "X-Timer": "S1623999602.174007,VS0,VE426", + "transfer-encoding": "chunked" + }, + "status_code": 200, + "url": "https://www.jstor.org/citation/text/40991855" +} \ No newline at end of file diff --git a/test/testdata/05e9df126f32fdce8f400ae36d7b093e8b7d9b84.html b/test/testdata/05e9df126f32fdce8f400ae36d7b093e8b7d9b84.html new file mode 100644 index 00000000..79b99478 --- /dev/null +++ b/test/testdata/05e9df126f32fdce8f400ae36d7b093e8b7d9b84.html @@ -0,0 +1,1681 @@ + <!DOCTYPE html> <html class=" b-pw-1280" lang="en" > <head> <!-- Barlesque 3.21.13 --> <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> <meta name="description" content="Breaking news, sport, TV, radio and a whole lot more. The BBC informs, educates and entertains - wherever you are, whatever your age." /> <meta name="keywords" content="BBC, bbc.co.uk, bbc.com, Search, British Broadcasting Corporation, BBC iPlayer, BBCi" /> <title>BBC - Homepage + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + + + +

BBC Homepage

+ +
+ +
+

 

+

+ News +

+

+ Sport +

+
+
+

+ Featured video +

+
+ + + + +

+ More from around the BBC +

+

+ BBC in other languages +

+
+ + + + + + + + + + + + + + + + + +
+ + + + \ No newline at end of file diff --git a/test/testdata/05e9df126f32fdce8f400ae36d7b093e8b7d9b84.json b/test/testdata/05e9df126f32fdce8f400ae36d7b093e8b7d9b84.json new file mode 100644 index 00000000..7f07cc73 --- /dev/null +++ b/test/testdata/05e9df126f32fdce8f400ae36d7b093e8b7d9b84.json @@ -0,0 +1,31 @@ +{ + "encoding": "ISO-8859-1", + "headers": { + "Accept-Ranges": "bytes", + "Age": "36", + "Cache-Control": "private, max-age=60", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Language": "en", + "Content-Length": "31699", + "Content-Type": "text/html", + "Date": "Tue, 23 May 2017 17:52:52 GMT", + "Etag": "\"72e5155fe4e0cf98222c9cfd488699a2\"", + "Expires": "Tue, 23 May 2017 17:51:14 GMT", + "Server": "Apache", + "Set-Cookie": "BBC-UID=530e2c160699677f2d6571d826ba633feae1fcc3f47688e91b93af93511331ef0Mozilla%2F5.0%20%28Windows%20NT%2010.0%3B%20Win64%3B%20x64%3B%20rv%3A50.0%29%20Gecko%2F20100101%20Firefox%2F50.0; expires=Sat, 22 May 2021 17:52:52 GMT; path=/; domain=.bbc.com", + "Vary": "Accept-Encoding", + "Via": "1.1 varnish", + "X-Cache": "HIT", + "X-Cache-Action": "MISS", + "X-Cache-Age": "0", + "X-Cache-Hits": "5", + "X-Fastly-Cache-Status": "HIT", + "X-LB-NoCache": "true", + "X-PAL-Host": "pal101.back.live.telhc.local:80", + "X-Served-By": "cache-iad2643-IAD", + "X-Timer": "S1495561972.111540,VS0,VE0" + }, + "status_code": 200, + "url": "http://www.bbc.com/" +} \ No newline at end of file diff --git a/test/testdata/0771c926605cbc127ca8b149d350e56ccd4c2833.html b/test/testdata/0771c926605cbc127ca8b149d350e56ccd4c2833.html new file mode 100644 index 00000000..5820bf95 --- /dev/null +++ b/test/testdata/0771c926605cbc127ca8b149d350e56ccd4c2833.html @@ -0,0 +1,101 @@ + + + + Malcolm Abbott's domestic violence past shows 'urgent action' required to support First Nations - ABC News + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Malcolm Abbott's domestic violence past shows 'urgent action' required to support First Nations

By Samantha Jonscher
Posted , updated 
 A steel fence with flowers attached
Flowers still mark where R Rubuntja died in January last year.(ABC Alice Springs: Samantha Jonscher)
Share this article
abc.net.au/news/malcolm-abbott-domestic-violence-prevention-fails/101059440

"Malcolm is half-killing me," R Rubuntja told her daughter on the phone.

WARNING: Aboriginal and Torres Strait Islander readers are advised that this article contains an image of a person who has died.

"I feel scared," a witness heard her say.

"I can't tell you where I am because Malcolm is going to hit me in the car," Ms Rubuntja told her daughter in another phone call.

An hour later, her partner, Malcolm Abbott, drove two laps of the Alice Springs Hospital car park before steering toward her.

He accelerated into Ms Rubuntja, trapping her in the car's undercarriage, before driving over her again and dragging her body across the bitumen of the Alice Springs Hospital car park. 

Ms Rubuntja, 46, died metres from the front door of the emergency department. 

Abbott, now 50, was sentenced to 25 years in prison over her death in January last year but, as Ms Rubuntja's friends and family found out in court during his sentencing, this was not his first offence.

Woman sitting with green background
R Rubuntja worked to prevent domestic violence against women and children in Alice Springs.(Supplied: Tangentyere Stories of Hope and Healing documentary)

A history of violence 

Ms Rubuntja was a founding member of the Tangentyere Women's Family Safety Group, where she worked with Australian National University researcher Chay Brown to address family violence in Alice Springs' town camps. 

This included his sentencing in 1997 to 10 years in prison for killing his then-wife and stabbing another person. 

He was later sentenced in 2009 to another five years in prison for stabbing his then-partner.

In 2014, he hit another former partner with a wheel brace and was sentenced to 15 months in jail. 

In 2018, he stabbed and punched his partner at the time in two separate incidents, which together sent him to jail for another year.

The Supreme Court building in low light, with palm trees visible in front.
Ms Rubuntja's friends and family want to know how Abbott's history went under the radar.(ABC Alice Springs: Samantha Jonscher)

'It's enraging'

Dr Brown and other advocates working in First Nations family and domestic violence said Ms Rubuntja's story showed that the system was not working. 

"We're shattered; it's enraging," Dr Brown said. 

"I would like to know how a man who had previously killed another woman was able to continue his violent offending until he was released and he murdered my friend."

Ms Rubuntja has been remembered as a leader, a mother and grandmother, but her friends and family said they had no idea about Abbott's history, and she had not engaged with any support services. 

Space to play or pause, M to mute, left and right arrows to seek, up and down arrows for volume.
Play Video. Duration: 33 seconds
More than 100 people paid tribute to R Rubuntja in Alice Springs.

No-one 'joining the dots'

Dr Brown said a lot of research had been undertaken into what level of risk a woman experiencing domestic violence was facing at any given time. 

She said Abbott's prior offending would have put Ms Rubuntja on somebody's radar if a risk assessment had been done, but that did not appear to have happened at any point in their relationship. 

"There were multiple opportunities where we could have intervened," Dr Brown said. 

Dr Brown said information-sharing between local organisations and Northern Territory police that "privileges women and children's safety above any man's right to confidentiality" would have made it clear Ms Rubuntja could be in danger. 

This does not currently exist in the NT but versions of it do exist in other jurisdictions.

Chay Brown
ANU researcher Chay Brown says the community is enraged over Ms Rubuntja's death. (ABC Alice Springs: Samantha Jonscher)

Resource scarcity

Women's Safety Services of Central Australia chief executive officer Larissa Ellis said women experiencing domestic and family violence "stared down death every day."

"It could be the first instance; it could be the 10th; it could be the 20th; it could be after 10 years of violence," she said.  

Being the only women's shelter in Central Australia, Ms Ellis said hundreds of women engaged her service every year.

She said that while assessing risk was "hard", it was also doable if you had enough information about a woman. 

Even if that information was collected and shared, however, she said women could still fall through the cracks because frontline services like hers were seriously overburdened. 

A woman in a purple shirt and cardigan leans against a door frame
Larissa Ellis runs Central Australia's only women's shelter.(ABC Alice Springs: Samantha Jonscher)

The system 'failed R'

Dr Brown said the community in general, including police and the justice system, did not understand how domestic violence worked. 

"We need to stop looking at domestic violence incidents as one-off incidents of violence," she said.

"It occurs within a long-term, ongoing pattern of power and control of violence and abuse."

Dr Brown believed this misunderstanding, and an absence of well-supported training for police and the wider public, contributed to Ms Rubuntja's death. 

"How many other women is it failing every day?"

Indigenous 'femicide'

Hannah McGlade is a Nyungar human rights lawyer who recently travelled to New York to address the United Nations (UN) on the plight of First Nations women in Australia. 

Since 2016, Dr McGlade has campaigned at the UN and at home for a standalone national action plan to end domestic violence for Indigenous Australians. 

Indigenous leaders have also called for a separate plan to address the specific complexities of domestic violence in First Nations communities.

Hannah McGlade wears a blue blazer while standing in front of a bookcase
Human rights lawyer Hannah McGlade says any plan needs to be fully funded.(ABC News: Rhiannon Shine)

Last year, federal Minister for Women's Safety Anne Ruston announced the government would develop two separate five-year plans, but Dr McGlade said advocates were yet to be given "more details about the process".

The former Labor government in 2011 released the first national action plan designed to coordinate state and federal policies, with the goal of making a "significant and sustained reduction" in family, domestic and sexual violence.

It will lapse in June 2022 and the government is currently drafting a new 10-year plan.

Labor has made creating a separate First Nations plan a federal election promise.

Dr McGlade said it was heartening to see commitments, but she believed it was vital that it translated into action. 

"They need to commit to investing and resourcing the plan properly," she said.

Posted , updated 
+ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/testdata/0771c926605cbc127ca8b149d350e56ccd4c2833.json b/test/testdata/0771c926605cbc127ca8b149d350e56ccd4c2833.json new file mode 100644 index 00000000..469ab279 --- /dev/null +++ b/test/testdata/0771c926605cbc127ca8b149d350e56ccd4c2833.json @@ -0,0 +1,32 @@ +{ + "encoding": "utf-8", + "headers": { + "Access-Control-Allow-Origin": "http://nucwed.aus.aunty.abc.net.au", + "Application": "news-web", + "Branch": "master-news-web", + "Build": "166", + "Cache-Control": "public, max-age=56", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "51430", + "Content-Security-Policy": "upgrade-insecure-requests;", + "Content-Type": "text/html; charset=utf-8", + "Date": "Sat, 04 Jun 2022 07:55:37 GMT", + "ETag": "W/\"46610-rCMBo4Yccns5iZ8XASjZ7RMmbSs-gzip\"", + "Environment": "production", + "Expires": "Sat, 04 Jun 2022 07:56:33 GMT", + "Product": "presentation-layer", + "Referrer-Policy": "no-referrer-when-downgrade", + "Server": "Apache", + "Set-Cookie": "ABCGuestID=23.58.222.6.78891654329337994; expires=Mon, 31-Dec-2038 23:59:59 GMT; path=/; domain=.abc.net.au, ABC_LD=int; path=/; domain=.abc.net.au, ABC_FF=desktop; expires=Sat, 04-Jun-2022 09:55:37 GMT; path=/", + "Vary": "Accept-Encoding, Origin, Cookie, User-Agent", + "X-Content-Type-Options": "nosniff", + "X-DNS-Prefetch-Control": "off", + "X-Download-Options": "noopen", + "X-Frame-Options": "SAMEORIGIN", + "X-XSS-Protection": "0", + "transaction-id": "YpsP-YYMBehzLsKWoWHYWgAAAIo" + }, + "status_code": 200, + "url": "https://www.abc.net.au/news/2022-05-15/malcolm-abbott-domestic-violence-prevention-fails/101059440" +} \ No newline at end of file diff --git a/test/testdata/0774d79affcfd8424a3d12a0aeac9d029669515b.html b/test/testdata/0774d79affcfd8424a3d12a0aeac9d029669515b.html new file mode 100644 index 00000000..f97c3111 --- /dev/null +++ b/test/testdata/0774d79affcfd8424a3d12a0aeac9d029669515b.html @@ -0,0 +1 @@ +{"indexed":{"date-parts":[[2022,4,5]],"date-time":"2022-04-05T10:33:43Z","timestamp":1649154823461},"reference-count":29,"publisher":"Informa UK Limited","issue":"4","content-domain":{"domain":[],"crossmark-restriction":false},"published-print":{"date-parts":[[2005,1]]},"DOI":"10.1081\/ada-200068110","type":"journal-article","created":{"date-parts":[[2005,11,15]],"date-time":"2005-11-15T20:01:40Z","timestamp":1132084900000},"page":"537-553","source":"Crossref","is-referenced-by-count":7,"title":"A Collaborative Action Approach to Researching Substance Abuse Recovery","prefix":"10.1081","volume":"31","author":[{"given":"Margaret I.","family":"Davis","sequence":"first","affiliation":[]},{"given":"Leonard A.","family":"Jason","sequence":"additional","affiliation":[]},{"given":"Joseph R.","family":"Ferrari","sequence":"additional","affiliation":[]},{"given":"Bradley D.","family":"Olson","sequence":"additional","affiliation":[]},{"given":"Josefina","family":"Alvarez","sequence":"additional","affiliation":[]}],"member":"301","published-online":{"date-parts":[[2009,7,7]]},"reference":[{"key":"CIT0001","volume-title":"Research on Alcoholics Anonymous: Opportunities and Alternatives","author":"McCrady B S","year":"1993"},{"key":"CIT0002","volume-title":"Memo on Participation of A.A. Members in Research and Other Non-A.A. Surveys","author":"Alcoholics Anonymous","year":"1991"},{"key":"CIT0003","volume-title":"Psychological Sense of Community","author":"Ferrari J R","year":"2003"},{"key":"CIT0004","doi-asserted-by":"crossref","first-page":"1","DOI":"10.2190\/TMNP-M3CC-BUPN-9EE6","volume":"31","author":"Jason L A","year":"2001","journal-title":"J Drug Educ"},{"key":"CIT0005","author":"Davis M I","journal-title":"J Community Psychol"},{"key":"CIT0006","volume-title":"Oxford House. On 60 Minutes","author":"St Pierre S","year":"1991"},{"key":"CIT0007","volume-title":"Oxford House Manual","author":"Oxford House, Inc.","year":"1988"},{"key":"CIT0008","doi-asserted-by":"publisher","DOI":"10.1300\/J020v13n03_08"},{"key":"CIT0009","doi-asserted-by":"publisher","DOI":"10.1081\/ADA-100101862"},{"key":"CIT0010","doi-asserted-by":"crossref","first-page":"332","DOI":"10.1007\/BF02832666","volume":"24","author":"Jason L A","year":"1997","journal-title":"J Mental Health Adm"},{"key":"CIT0011","doi-asserted-by":"crossref","first-page":"803","DOI":"10.1023\/A:1022241712065","volume":"26","author":"Bishop P D","year":"1998","journal-title":"Am J Community Psychol"},{"key":"CIT0012","doi-asserted-by":"crossref","first-page":"25","DOI":"10.1016\/0740-5472(94)90061-2","volume":"11","author":"Condelli W S","year":"1994","journal-title":"J Subst Abuse Treat"},{"key":"CIT0013","doi-asserted-by":"crossref","first-page":"823","DOI":"10.3109\/10826088509047755","volume":"20","author":"DeLeon G","year":"1985","journal-title":"Int J Addict"},{"key":"CIT0014","doi-asserted-by":"crossref","first-page":"296","DOI":"10.1037\/0022-006X.63.2.296","volume":"63","author":"Longabaugh R","year":"1995","journal-title":"J Consult Clin Psychol"},{"key":"CIT0015","doi-asserted-by":"crossref","first-page":"482","DOI":"10.1097\/00005053-199608000-00005","volume":"184","author":"McCusker J","year":"1996","journal-title":"J Nerv Ment Dis"},{"key":"CIT0016","doi-asserted-by":"crossref","first-page":"31","DOI":"10.1111\/j.1360-0443.1994.tb00846.x","volume":"89","author":"Moos RH","year":"1994","journal-title":"Addiction"},{"key":"CIT0017","doi-asserted-by":"crossref","first-page":"23","volume-title":"Researching Community Psychology: Issues of Theory and Methods","author":"Kingry-Westergaard C","year":"1990","DOI":"10.1037\/10073-002"},{"key":"CIT0018","first-page":"1","author":"McSherry M","year":"1995","journal-title":"The Daily Southern"},{"key":"CIT0019","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1007\/BF00942250","volume":"19","author":"Jason L A","year":"1991","journal-title":"Am J Community Psychol"},{"key":"CIT0020","author":"Gelernter C Q","year":"1994","journal-title":"Seattle Times"},{"key":"CIT0021","author":"Kinsler L","year":"1999","journal-title":"Fayetteville Observer"},{"key":"CIT0022","author":"Levin M","year":"2001","journal-title":"Austin Rev"},{"key":"CIT0023","first-page":"21","volume":"35","author":"Olson B D","year":"2001","journal-title":"Community Psychol"},{"issue":"5","key":"CIT0024","doi-asserted-by":"crossref","first-page":"757","DOI":"10.1007\/BF00938043","volume":"19","author":"Chesler M","year":"1991","journal-title":"Am J Community Psychol"},{"key":"CIT0025","doi-asserted-by":"publisher","DOI":"10.1300\/J020v15n01_05"},{"key":"CIT0026","doi-asserted-by":"crossref","first-page":"217","DOI":"10.1016\/S0962-1849(05)80096-4","volume":"3","author":"Humphreys K","year":"1994","journal-title":"Appl Prev Psychol"},{"key":"CIT0027","doi-asserted-by":"crossref","first-page":"77","DOI":"10.1016\/S0899-3289(97)90007-9","volume":"9","author":"Ferrari J R","year":"1997","journal-title":"J Subst Abuse"},{"key":"CIT0028","author":"Jason L A","journal-title":"J Prev Interv Community"},{"key":"CIT0029","volume-title":"Reflections on Gender and Society","author":"Keller E","year":"1985"}],"container-title":"The American Journal of Drug and Alcohol Abuse","original-title":[],"language":"en","link":[{"URL":"http:\/\/www.tandfonline.com\/doi\/pdf\/10.1081\/ADA-200068110","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2017,6,17]],"date-time":"2017-06-17T00:56:09Z","timestamp":1497660969000},"score":1,"resource":{"primary":{"URL":"http:\/\/www.tandfonline.com\/doi\/full\/10.1081\/ADA-200068110"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2005,1]]},"references-count":29,"journal-issue":{"issue":"4","published-online":{"date-parts":[[2009,7,7]]},"published-print":{"date-parts":[[2005,1]]}},"alternative-id":["10.1081\/ADA-200068110"],"URL":"http:\/\/dx.doi.org\/10.1081\/ada-200068110","relation":{},"ISSN":["0095-2990","1097-9891"],"subject":["Psychiatry and Mental health","Clinical Psychology","Medicine (miscellaneous)"],"container-title-short":"The American Journal of Drug and Alcohol Abuse","published":{"date-parts":[[2005,1]]}} \ No newline at end of file diff --git a/test/testdata/0774d79affcfd8424a3d12a0aeac9d029669515b.json b/test/testdata/0774d79affcfd8424a3d12a0aeac9d029669515b.json new file mode 100644 index 00000000..9d9b2734 --- /dev/null +++ b/test/testdata/0774d79affcfd8424a3d12a0aeac9d029669515b.json @@ -0,0 +1,24 @@ +{ + "encoding": null, + "headers": { + "access-control-allow-headers": "X-Requested-With, Accept, Accept-Encoding, Accept-Charset, Accept-Language, Accept-Ranges, Cache-Control", + "access-control-allow-origin": "*", + "access-control-expose-headers": "Link", + "connection": "close", + "content-encoding": "gzip", + "content-length": "2052", + "content-type": "application/vnd.citationstyles.csl+json", + "date": "Thu, 09 Jun 2022 10:37:21 GMT", + "link": "; rel=\"canonical\", ; version=\"vor\"; rel=\"item\"", + "permissions-policy": "interest-cohort=()", + "server": "Jetty(9.4.40.v20210413)", + "vary": "Accept, Accept-Encoding", + "x-api-pool": "public", + "x-rate-limit-interval": "1s", + "x-rate-limit-limit": "50", + "x-ratelimit-interval": "1s", + "x-ratelimit-limit": "50" + }, + "status_code": 200, + "url": "https://api.crossref.org/v1/works/10.1081%2Fada-200068110/transform" +} \ No newline at end of file diff --git a/test/testdata/086a572426d4bae1f773fe1d86791bcc0c98f4d2.html b/test/testdata/086a572426d4bae1f773fe1d86791bcc0c98f4d2.html new file mode 100644 index 00000000..2bb727c9 --- /dev/null +++ b/test/testdata/086a572426d4bae1f773fe1d86791bcc0c98f4d2.html @@ -0,0 +1,11 @@ + +@article{noormags454096, +title = { زندگی نامه علمی دکتر کاووس حسن لی }, +journal = { شعر }, +number = { 62 }, +year = { 1387 }, +author = { +}, +pages = { 17 -- 19 }, +url = { https://www.noormags.ir/view/fa/articlepage/454096 } +} \ No newline at end of file diff --git a/test/testdata/086a572426d4bae1f773fe1d86791bcc0c98f4d2.json b/test/testdata/086a572426d4bae1f773fe1d86791bcc0c98f4d2.json new file mode 100644 index 00000000..9ced49de --- /dev/null +++ b/test/testdata/086a572426d4bae1f773fe1d86791bcc0c98f4d2.json @@ -0,0 +1,16 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "private", + "Content-Disposition": "attachment; filename=\"noormags-454096.bib\"", + "Content-Length": "264", + "Content-Type": "application/x-bibtex; charset=UTF-8", + "Date": "Fri, 13 Apr 2018 07:58:08 GMT", + "Set-Cookie": "CRCIS_SessionId=ivcgwmi4c3rweogdni3iev3h; path=/; secure; HttpOnly", + "Strict-Transport-Security": "max-age=3153600", + "X-Download-Options": "noopen", + "X-Frame-Options": "SAMEORIGIN" + }, + "status_code": 200, + "url": "https://www.noormags.ir/view/fa/citation/bibtex/454096" +} \ No newline at end of file diff --git a/test/testdata/0a22a875aeb033c9a30e9430d5e236cbccd675e9.html b/test/testdata/0a22a875aeb033c9a30e9430d5e236cbccd675e9.html new file mode 100644 index 00000000..ff8de777 --- /dev/null +++ b/test/testdata/0a22a875aeb033c9a30e9430d5e236cbccd675e9.html @@ -0,0 +1,4391 @@ + + + + + +News, sport and opinion from the Guardian's US edition | The Guardian + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+ + +
+
+
+
+
+
+ +
+
+
+ UK concert explosion +
+Tuesday + 23 May 2017 +
+
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+
+headlines +
+
+
+
+ +
+
+ +
+
+
+
+
+
+
+
+spotlight +
+
+ +
+
+ +
+ +
+
+
+
+
+
+
+
+ +
+ +
+
+ +
+
+
+
+
+
+
+
+
+
+
+
+securedrop +
+
+
+
+
    + +
+
+
+
+
+
+
+
+ +
+
+
+ +
+
+ +
+
+
+
+
+
+
+
+
+
+ +
+
+
+ +
+
+
+
+ +
+
+
+
+
+
+ +
+
+ +
+
+
+
+
+
+
+
+documentaries +
+
+
+
+
    + +
+
+
+
+
+
+
+
+
+
+
+ +
+ +
+ +
+
+
+
+
+
+

videos

+
+
+
+ + + + +
+
+ + + + +
+ +
+
+
+
+
+
+
+
+
+
+
+explore +
+
+
+
+ +
+ +
+ +
+
+
+
+
+ + +
+
+ +
+
+
+
+
+
+
+
+
+
+
+
+in pictures +
+
+
+
+ +
+
+
+
+
+
+
+
+people +
+
+
+
+ +
+
+
+
+
+
+
+
+
+ +
+ + +
+
+
+
+
+
+ + + + + + + + + + + diff --git a/test/testdata/0a22a875aeb033c9a30e9430d5e236cbccd675e9.json b/test/testdata/0a22a875aeb033c9a30e9430d5e236cbccd675e9.json new file mode 100644 index 00000000..cb247fdc --- /dev/null +++ b/test/testdata/0a22a875aeb033c9a30e9430d5e236cbccd675e9.json @@ -0,0 +1,34 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "57", + "Cache-Control": "max-age=60, stale-while-revalidate=6, stale-if-error=864000, private", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "101611", + "Content-Security-Policy": "default-src https:; script-src https: 'unsafe-inline' 'unsafe-eval'; style-src https: 'unsafe-inline'; img-src https: data: blob:; media-src https: data: blob:; font-src https: data:; connect-src https: wss:; report-uri https://beacon.gu-web.net/csp-report", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:55:00 GMT", + "ETag": "W/\"hash-4099982973909319275\"", + "Expires": "Tue, 23 May 2017 17:55:03 GMT", + "Fastly-Debug-Digest": "6a9e3c65c9557db5ebc3ea496f7c1048bec0fc4f7d07a10fd90df37207b4a5be", + "Link": "; rel=preload; as=style; nopush,; rel=preload; as=script; nopush,; rel=preload; as=script; nopush,; rel=preload; as=script; nopush", + "Set-Cookie": "GU_mvt_id=876485; expires=Mon, 21 Aug 2017 17:55:00 GMT; path=/; domain=.theguardian.com, GU_geo_continent=NA; path=/;", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "Vary": "Accept-Encoding,User-Agent", + "Via": "1.1 varnish, 1.1 varnish", + "X-Cache": "HIT, HIT", + "X-Cache-Hits": "1, 1", + "X-Content-Type-Options": "nosniff", + "X-Frame-Options": "SAMEORIGIN", + "X-GU-Edition": "us", + "X-GU-Platform": "next-gen-router", + "X-Gu-Backend-App": "facia", + "X-Served-By": "cache-lcy1144-LCY, cache-iad2633-IAD", + "X-Timer": "S1495562101.888031,VS0,VE1", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://www.theguardian.com/us" +} \ No newline at end of file diff --git a/test/testdata/0a40086c52364c9b7ef8e42ddd1a75afe66cc254.html b/test/testdata/0a40086c52364c9b7ef8e42ddd1a75afe66cc254.html new file mode 100644 index 00000000..79b99478 --- /dev/null +++ b/test/testdata/0a40086c52364c9b7ef8e42ddd1a75afe66cc254.html @@ -0,0 +1,1681 @@ + BBC - Homepage + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + + + +

BBC Homepage

+ +
+ +
+

 

+

+ News +

+

+ Sport +

+
+
+

+ Featured video +

+
+ + + + +

+ More from around the BBC +

+

+ BBC in other languages +

+
+ + + + + + + + + + + + + + + + + +
+ + + + \ No newline at end of file diff --git a/test/testdata/0a40086c52364c9b7ef8e42ddd1a75afe66cc254.json b/test/testdata/0a40086c52364c9b7ef8e42ddd1a75afe66cc254.json new file mode 100644 index 00000000..cb461233 --- /dev/null +++ b/test/testdata/0a40086c52364c9b7ef8e42ddd1a75afe66cc254.json @@ -0,0 +1,31 @@ +{ + "encoding": "ISO-8859-1", + "headers": { + "Accept-Ranges": "bytes", + "Age": "39", + "Cache-Control": "private, max-age=60", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Language": "en", + "Content-Length": "31699", + "Content-Type": "text/html", + "Date": "Tue, 23 May 2017 17:52:54 GMT", + "Etag": "\"72e5155fe4e0cf98222c9cfd488699a2\"", + "Expires": "Tue, 23 May 2017 17:51:14 GMT", + "Server": "Apache", + "Set-Cookie": "BBC-UID=1203ee5061b7dfb247c89144eb9ab6854d64a6323f475508dc266cb61a2aa00a0Mozilla%2F5.0%20%28Windows%20NT%2010.0%3B%20Win64%3B%20x64%3B%20rv%3A50.0%29%20Gecko%2F20100101%20Firefox%2F50.0; expires=Sat, 22 May 2021 17:52:54 GMT; path=/; domain=.bbc.com", + "Vary": "Accept-Encoding", + "Via": "1.1 varnish", + "X-Cache": "HIT", + "X-Cache-Action": "MISS", + "X-Cache-Age": "0", + "X-Cache-Hits": "4", + "X-Fastly-Cache-Status": "HIT", + "X-LB-NoCache": "true", + "X-PAL-Host": "pal101.back.live.telhc.local:80", + "X-Served-By": "cache-iad2630-IAD", + "X-Timer": "S1495561975.644700,VS0,VE0" + }, + "status_code": 200, + "url": "http://www.bbc.com/" +} \ No newline at end of file diff --git a/test/testdata/0cdefa99cd2511d6db79a4d6150cd78e0c97efca.html b/test/testdata/0cdefa99cd2511d6db79a4d6150cd78e0c97efca.html new file mode 100644 index 00000000..76ad2340 --- /dev/null +++ b/test/testdata/0cdefa99cd2511d6db79a4d6150cd78e0c97efca.html @@ -0,0 +1,1461 @@ + + + + + + Sea otter return boosts ailing seagrass in California - BBC News + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+
+
+ + + + + + +
+ +
+ + + Science & Environment + + + + +
+ +
+ + Science & Environment + + + + +
+ +
+

Sea otter return boosts ailing seagrass in California

+ + + + + +
+
+ + + sea otter ecology + + + + + +
+ Image caption + + A sea otter enjoys a crab in California, and helps seagrass in the process + +
+ +

The return of sea otters to an estuary on the central Californian coast has significantly improved the health of seagrass, new research has found.

Seagrass was deemed to be heading for extinction in this region before the otters returned.

But scientists found that the animals triggered a chain reaction of events that boosted the water-dwelling plants.

The research is published in the journal, PNAS.

The urbanisation of California has led to a huge increase in nutrient pollution in coastal waters, from increasing use of nitrogen-rich fertilizers.

This is said to be the reason for the dieback of seagrass, which has also been declining worldwide.

This research suggests that the hunting to near-extinction of sea otters in the late 19th and early 20th Century may have exacerbated the problem, and conversely that their reintroduction is helping revive ailing seagrass populations, even in the face of hugely nutrient-rich water.

Links in the chain

The researchers assessed seagrass levels over the past 50 years in the Elkhorn Slough in Monterey Bay, and mapped their increases and declines.

They looked at a variety of changes that may have affected the grass, but the only factor that really matched the changes in seagrass was sea otter numbers.

They theorised that sea otters were eating the crabs which prey upon small invertebrates in the water.

These invertebrates eat a type of algae which blooms when there are more nutrients in the soil. It grows on the leaves of the seagrass, shading them from sunlight and causing them to die back.

This is quite a complex cascade of effects, so the researchers tested out their theory by comparing similar estuaries with and without sea otters, and by doing experiments in the lab, and in the field.

These experiments, which included putting cages that sea otters either could or couldn't access, down on the seagrass, confirmed their hypothesis.

+ + + +
+ + +
+ +
+ Image caption + + Sea otters have been responsible for improving the health of the seagrass in these estuaries. + +
+ +

Brent Hughes, lead author of the study, said: "This estuary is part of one of the most polluted systems in the entire world, but you can still get this healthy thriving habitat, and it's all because of the sea otters.

"So it's almost like these sea otters are fighting the effects of poor water quality."

Hughes described seagrass as "the canary in the coalmine" in terms of predicting levels of nutrient pollution in the water.

Foundation species

It also acts as a nursery habitat for many species of fish and it uses CO2 from sea water and the atmosphere, thus potentially helping with climate change.

Not only that, but it acts as protection to the stability of the shoreline.

Hughes said: "It's what we call a foundation species, like kelp forest, salt marsh or coral reef. The major problem from a global perspective is that seagrass is declining worldwide. And one of the major drivers of this decline has been nutrient inputs from anthropogenic sources, via agriculture or urban runoff."

These findings are of particular interest at the moment, as a ban on sea otters moving along the coast to southern California was lifted last year. The ban was in place as there was a fear the sea otters would impinge on fisheries in the area.

Hughes told BBC news: "That's important because there's a lot of these kind of degraded estuaries in southern California because of all the urban runoff from places like Los Angeles and San Diego.

"Coastal managers will now have a better sense of what's going to happen when sea otters move in to their systems.

"There's a huge potential benefit to sea otters returning to these estuaries, and in to these seagrass beds that might be threatened."

+
+
+ + + +
+

More on this story

+ +
+

Related Internet links

+

The BBC is not responsible for the content of external Internet sites

+
+ + + + + + +
+ +
+ + + + + +
+ + + + +
+ +
+ + + +
+ + + + + + + + + + + + + + + + + diff --git a/test/testdata/0cdefa99cd2511d6db79a4d6150cd78e0c97efca.json b/test/testdata/0cdefa99cd2511d6db79a4d6150cd78e0c97efca.json new file mode 100644 index 00000000..e16e4c30 --- /dev/null +++ b/test/testdata/0cdefa99cd2511d6db79a4d6150cd78e0c97efca.json @@ -0,0 +1,31 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "0", + "Cache-Control": "private, max-age=60, stale-while-revalidate", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Language": "en", + "Content-Length": "35305", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:52:55 GMT", + "Server": "Apache", + "Set-Cookie": "BBC-UID=b3e26cacd49dc69d8a4803eec65d4774141d3ec19e9ff63953af45a8b4da28b40Mozilla%2F5.0%20%28Windows%20NT%2010.0%3B%20Win64%3B%20x64%3B%20rv%3A50.0%29%20Gecko%2F20100101%20Firefox%2F50.0; expires=Sat, 22 May 2021 17:52:55 GMT; path=/; domain=.bbc.com", + "Vary": "Accept-Encoding", + "Via": "1.1 varnish", + "X-Cache": "MISS", + "X-Cache-Action": "MISS", + "X-Cache-Age": "0", + "X-Cache-Hits": "0", + "X-Fastly-Cache-Status": "MISS-CLUSTER", + "X-LB-NoCache": "true", + "X-News-Cache-Id": "40708", + "X-News-Data-Centre": "telhc", + "X-PAL-Host": "pal181.back.live.telhc.local:80", + "X-Served-By": "cache-iad2136-IAD", + "X-Timer": "S1495561975.642395,VS0,VE1337" + }, + "status_code": 200, + "url": "http://www.bbc.com/news/science-environment-23814524" +} \ No newline at end of file diff --git a/test/testdata/0d8cb5efe5e2f466b4b39fcf638b865ea85ee25b.html b/test/testdata/0d8cb5efe5e2f466b4b39fcf638b865ea85ee25b.html new file mode 100644 index 00000000..7e4e8c58 --- /dev/null +++ b/test/testdata/0d8cb5efe5e2f466b4b39fcf638b865ea85ee25b.html @@ -0,0 +1,46 @@ + انتخابات 96 به روایت آمار

irinn | وب سایت شبکه خبر

نسخه آزمایشی       
16:11 - چهارشنبه 03 خرداد 1396
نظامیان صهیونیست یک جوان فلسطینی را در شمال کرانه غربی به ضرب گلوله به شهادت رساندند رژیم صهیونیستی روستای دیراستیا در کرانه غربی رود اردن را منطقه بسته نظامی اعلام کرد نظامیان صهیونیست با یورش به کرانه غربی و نوار غزه ده ها فلسطینی را زخمی کردند انفجار خودروی بمب گذاری شده در جنوب بنغازی لیبی ۶ کشته و ۱۱ زخمی بر جا گذاشت در حمله به یک پایگاه هوایی در جنوب لیبی بیش از ۱۴۰ تن کشته شدند رئیس جمهور ونزوئلا(خطاب به رئیس جمهور آمریکا): دخالت بس است از ونزوئلا بیرون برو آمریکا به بهانه حمایت از دموکراسی تحریم های جدیدی را ضد ونزوئلا وضع کرده است زمین لرزه ۶ ریشتری مرکز فیلیپین را لرزاند
کد خبر: ۴۹۹۶۵۴
تاریخ انتشار: پنجشنبه ۲۸ ارديبهشت ۱۳۹۶ - ۲۲:۵۷
پایگاه اطلاع رسانی شبکه خبر با بررسی همه ابعاد انتخابات 96، آماری تهیه کرده است؛ که این آمار در نوع خود کم‌سابقه و شاید بی‌نظیر است!
بر اساس این گزارش، واجدان شرایط رای دادن: 56 میلیون و 410 هزار و 234 نفر برآورد شده اند.
 رای اولی‌ها: یک میلیون و 350 هزار و 294 نفر
 حائزان شرایط شرکت در انتخابات: متولدان 29 اردیبهشت 1378 و قبل از آن
 عوامل اجرایی و نظارتی انتخابات: حدود یک میلیون و 500 هزار نفر
(حدود یک میلیون 500 هزار نفر عوامل  اجرایی و نظارت بر انتخابات ریاست جمهوری، شوراهای اسلامی شهر و روستاها و میان دوره مجلس شورای اسلامی در چهار حوزه  هستند.)

مسئولان بخش تامین امنیت: 350 هزار نفر، امنیت برگزاری انتخابات را بر عهده دارند.
  عوامل انتظامات: 160 هزار نفر فقط امنیت شعبه اخذی رای را پوشش می دهند. 120 هزار نفر از نیروهای انتظامی و 40 هزار نفر از سپاه و بسیج هستند.

حدود 71 هزار بازرس برای شعبه های اخذ رای وجود دارد. 100 هزار نماینده فرمانداری‌ها و 100 هزار نماینده شورای نگهبان پای صندوق های رای حضور دارند.

هیئت اجرایی: هزار و 60 هیئت اجرایی اعم از بخش و شهرستان.
  از این تعداد ۸ نفر انتخاب می‌شوند؛ به علاوه فرماندار، استاندار و رئیس آموزش و پرورش که به ۱۱ نفر می‌رسند.


 بیشترین شعب اخذ رای: استان‌های تهران، خراسان رضوی و مازندران
 شعب اخذ رای: 63 هزار و 429 شعبه   
  63 هزار و 429 شعبه اخذ رای برای انتخابات 29 اردیبهشت در نظر گرفته شده که از این تعداد بیش از 14 هزار صندوق سیار و بقیه ثابت است.

 تعداد صندوق ها: 117 یا 118 هزار صندوق
 انتخاب رئیس جمهور: یک نفر
 انتخاب نماینده مجلس: 4 نفر
 انتخاب اعضای شوراهای اسلامی: 39 هزار و 575 کرسی
 انتخابات الکترونیکی شوراها: انتخابات شورای اسلامی شهر و روستا در 141 شهر، 9 هزار و 752 شعبه و در 28 استان الکترونیکی برگزار می شود.

حدود ۳۵ هزار صندوق الکترونیک خواهیم داشت.

ایرانیان واجد شرایط مقیم خارج از کشور: دو تا دو و نیم میلیون نفر
 برگزاری انتخابات خارج از کشور: 103 کشور جهان، 133 نمایندگی و 304 شعبه
شعب برخی کشورها: آمریکا 55، عراق 22، امارات متحده 21 و انگلستان 12 شعبه
200 میلیون تعرفه رای برای انتخابات سال 96 پیش‌بینی شده است.
 
وزیر کشور: با توجه به کناره‌گیری برخی نامزدها 200 میلیون تعرفه برای این انتخابات کافیست.
عبدالرضا رحمانی فضلی: در تعرفه‌های ما تقریبا ۱۴ نوع رمزگذاری به‌کار رفته که به هیچ کس اجازه تخلف را نمی‌دهد.

چاپ تعرفه‌های انتخابات شوراها در ۷ رنگ
در جلسه اخیر هیئت مرکزی نظارت بر انتخابات شوراهای اسلامی شهر و روستا و وزارت کشور نحوه تهیه تعرفه‌های انتخاباتی (برگه‌های اخذ رأی) مشخص و مقرر شد، برای استان‌ها، شهرها، بخش‌ها و روستاها که 21،15،11،9،7،5،3 نماینده در شورای شهر دارند تعرفه‌ها با رنگ‌های مختلف چاپ شد.
 
رای‌گیری از  ساعت 8 صبح روز جمعه 29 اردیبهشت تا 24 این روز قانونی است و اگر یک دقیقه از ساعت قانونی رد شود رای‌گیری غیرقانونی است.


تمهیدات ویژه برای حفظ امنیت و سلامت انتخابات
وزیر کشور: ما سه انتخابات شوراهای شهر و روستا، ریاست جمهوری و در 4 استان انتخابات میان دوره‌ای مجلس شورای اسلامی را پیش رو و به تناسب آن 3 قانون برای این انتخابات داریم، برای این 3 انتخابات به همین نسبت 3 هیئت اجرایی و نظارت تشکیل می‌شود.


وی گفت: برای ریاست جمهوری از یک سال قبل با شورای نگهبان وارد مذاکره شدیم که رای‌گیری الکترونیکی را انجام دهیم، اما به دلایلی این اتفاق نیفتاد.

وزیر کشور افزود: انتخابات شورا‌ها فرصت خوبی بود برای آغاز این اتفاق و بنا شد این اتفاق در ۱۴۱ شهر انجام پذیرد.
رحمانی فضلی: تا زمانی که اعلام نتایج تهران حاصل نشود، نتیجه دیگر شهر‌ها را اعلام نخواهیم کرد.

وی گفت: با روش جدید شورای نگهبان در مورد صحت و سلامت شناسنامه ها، این بار تشخیص هویت در انتخابات هم الکترونیکی شده و قانون هم می‌گوید ملاک شماره ملی است و ما اطلاعات ۸۰ میلیون ایرانی را داریم که اگر شماره ملی درست نباشد فرد اجازه رای دادن ندارد.
وزیر کشور افزود: برای نظارت بر شمارش آراء در مرحله شمارش آراء ۶۴ هزار بازرس را در همه شعبه‌ها مستقرشده اند که دقت رابالا ببریم.
رحمانی فضلی گفت: ۶۴ هزار نماینده از شورای نگهبان همراه نمایندگان نامزد‌ها تا لحظه آخر شمارش آراء حضور خواهند داشت.

رئیس هیئت مرکزی نظارت بر انتخابات شوراها: 265 هزار و 121 نفر در انتخابات شوراهای شهر و روستا رقابت می کنند.
محمودی شاه نشین: تعداد داوطلبان روستایی به ازای هر کرسی 1/8 دهم و این تعداد برای شهرها به ازای هر کرسی 6/8 دهم است.
پورعلی مطلق: رای گیری در بیش از 63 هزار و 429 شعبه در سراسر کشور انجام می شود

همراه داشتن شناسنامه و شماره ملی در داخل کشور و گذرنامه در خارج از کشور برای رای دادن الزامی است.

سخنگوی سازمان ثبت احوال: تمام درخواست ها تا 27 اردیبهشت منجر به صدور شناسنامه المثنی می شود.

نظر شما
+ +
+
نام:
+
+
+
+
+
ایمیل:
+
+
+
+
+
* نظر:
+
+
+
+ +
+
+
+ +
+
+
+
پربازدید ها
روز
هفته
ماه
آب و هوا
۱۸°    ۳۴°
\ No newline at end of file diff --git a/test/testdata/0d8cb5efe5e2f466b4b39fcf638b865ea85ee25b.json b/test/testdata/0d8cb5efe5e2f466b4b39fcf638b865ea85ee25b.json new file mode 100644 index 00000000..5302e91a --- /dev/null +++ b/test/testdata/0d8cb5efe5e2f466b4b39fcf638b865ea85ee25b.json @@ -0,0 +1,18 @@ +{ + "encoding": "utf-8", + "headers": { + "Cache-Control": "post-check=0, pre-check=0", + "Connection": "close", + "Content-Encoding": "gzip", + "Content-Length": "18402", + "Content-Type": "text/html; charset=utf-8", + "Date": "Wed, 24 May 2017 12:09:39 GMT", + "Expires": "Sat, 26 Jul 1997 05:00:00 GMT", + "Pragma": "no-cache", + "Server": "sepehr-proxy-1.2-rc3-server4-tabnak", + "X-Cache": "MISS from google.com", + "X-Cache-Lookup": "MISS from google.com:85" + }, + "status_code": 200, + "url": "http://www.irinn.ir/fa/news/499654/%D8%A7%D9%86%D8%AA%D8%AE%D8%A7%D8%A8%D8%A7%D8%AA-96-%D8%A8%D9%87-%D8%B1%D9%88%D8%A7%DB%8C%D8%AA-%D8%A2%D9%85%D8%A7%D8%B1" +} \ No newline at end of file diff --git a/test/testdata/10119f49cdab351169af443f1fb9e07bde39090b.html b/test/testdata/10119f49cdab351169af443f1fb9e07bde39090b.html new file mode 100644 index 00000000..a095c686 --- /dev/null +++ b/test/testdata/10119f49cdab351169af443f1fb9e07bde39090b.html @@ -0,0 +1,727 @@ + + + + + + + + + + + + + + +UAE's Enoc pays Iran $4 billion in oil dues | Iran News | Al Jazeera + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + +
+ + +
+ + +
+ + + +
+
+ + +
+ +
+
body : Layout 27 : Cell 1

NewsIran

UAE's Enoc pays Iran $4 billion in oil dues

+
+

Emirates National Oil Company has paid dues as part of its debts owed to Iran for pre-sanction oil and gas purchases.

+

Business & Economy, Iran, UAE, Middle East

+
Iran re-entered the global economy in January 2016 following years of crippling international sanctions [File: EPA]

Iran has said it had received more than $4bn from Emirates National Oil Company (ENOC), as part of the UAE retailer's settlement of its debts owed to Iran for pre-sanction oil and gas purchases, reported Iranian news website Al Alam.

+

Iran's Supreme Audit Court, which monitors the Oil Ministry's deposits into the state treasury, said that the Central Bank of Iran had received the total amount of $4,105,219,136 as part of Enoc debts over the purchase of gas condensate, the agency said on Sunday.

+

Last year Iran's Ministry of Petroleum confirmed that international oil companies (IOCs) had started paying the amounts owed to Iran for pre-sanction oil purchases.

+

READ MORE: Dawn of a new era as Iran sanctions lifted

+

Debtors include the UAE's Enoc, Anglo Dutch energy giant Shell, Greece's Hellenic Petroleum, Italy's Saras.

+

Banking restrictions had previously prevented the companies from transferring payments to Iran.

+

Iran re-entered the global economy in January 2016 following years of crippling international sanctions, after the UN announced the country had complied with the terms of a landmark deal in July 2015 aimed at scaling down its nuclear programme.

+

The landmark deal Tehran finalised with six world powers allowed it to have immediate access to more than $50bn in long-frozen assets and freedom to sell its oil and purchase goods in the international marketplace.

Source: News agencies

+
+ +
+ +
+
+

+ Content on this website is for general information purposes only. Your comments + are provided by your own free will and you take sole responsibility for any direct + or indirect liability. You hereby provide us with an irrevocable, unlimited, and + global license for no consideration to use, reuse, delete or publish comments, in + accordance with Community Rules & Guidelines and Terms and Conditions. +

+
+ + +
+
MORE FROM AL JAZEERA
Related
+
+ +
+ + +
+
MUST-SEE PROGRAMMES
+
+ +
+ +
+

+  

+
+ +
+
+ + + + + + + + + + + + + + + + +
+ +
+ + + + + + + + + + + +
+ + + +
+ + + + + + + +
+ + diff --git a/test/testdata/10119f49cdab351169af443f1fb9e07bde39090b.json b/test/testdata/10119f49cdab351169af443f1fb9e07bde39090b.json new file mode 100644 index 00000000..b9cc6bf2 --- /dev/null +++ b/test/testdata/10119f49cdab351169af443f1fb9e07bde39090b.json @@ -0,0 +1,20 @@ +{ + "encoding": "ISO-8859-1", + "headers": { + "Accept-Ranges": "bytes", + "Access-Control-Allow-Origin": "http://live.aljazeera.com", + "Cache-Control": "max-age=180, public", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "14054", + "Content-Type": "text/html", + "Date": "Tue, 30 May 2017 04:57:45 GMT", + "Expires": "Tue, 30 May 2017 05:00:45 GMT", + "Publisher": "Al Jazeera (ORYX CMS) - ZUB", + "Server": "Microsoft-IIS/10.0", + "X-Method": "GET", + "X-Powered-By": "VSH-Z-2" + }, + "status_code": 200, + "url": "http://www.aljazeera.com/news/2017/05/uae-enoc-pays-iran-4-billion-oil-dues-170529171315570.html" +} \ No newline at end of file diff --git a/test/testdata/1056a29d30a9039fcd9e22710615c7d7e3e9361b.html b/test/testdata/1056a29d30a9039fcd9e22710615c7d7e3e9361b.html new file mode 100644 index 00000000..40c91ed8 --- /dev/null +++ b/test/testdata/1056a29d30a9039fcd9e22710615c7d7e3e9361b.html @@ -0,0 +1,656 @@ + + + + + + + + + + + + + + + + + + On a New Case of Interference of the Rays of Light on JSTOR + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + +
+ +
+ + + + + + + + + + + +
+
+ + + + + + Have library access? + + Log in through your library + + + + + + + +
+ + +
+ + + + + + +
+
+ + +
+
+ + +
+ + +
+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + + + + diff --git a/test/testdata/1056a29d30a9039fcd9e22710615c7d7e3e9361b.json b/test/testdata/1056a29d30a9039fcd9e22710615c7d7e3e9361b.json new file mode 100644 index 00000000..19985f02 --- /dev/null +++ b/test/testdata/1056a29d30a9039fcd9e22710615c7d7e3e9361b.json @@ -0,0 +1,21 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Type": "text/html; charset=utf-8", + "Date": "Fri, 04 Jun 2021 11:13:18 GMT", + "Server": "Apache/2.4.29 (Ubuntu)", + "Set-Cookie": "AccessSession=H4sIAAAAAAAAAK1Ry27UQBC85yssnzNR97yH22aRAXFMbgihds8MGDnJam0jQZR_x89dLZAb0lymqrpU3fV8VRRlE8viTVH6GKxL0WsArdlbn8kqTmjBSGMwl9eTmFd1QyyICOQCf99MwFVgtdyjfYuyAumq3W3Qzhgvla9gUR83tQGtoq2Vg6AxI0XKihJZK1EquaqHYZNnbZWTSiTQTmifUNSJk5AMYJxmhOiXERr6b9NIprZLM_KD2sUFrZQeTQhOGzNTzWF2V3iDenzuBt3qEh-ax0ubrnu6BHq-_BNz303Qp-J5_J_PiwAQZt8RW7OUegPSCoSga6WCETmQETpIEN7TuGKI2aPnrCJtM_3PQ5qH3h2fhsPJ-gTfUtfwBTeu0zw0v1LV0tdJ0h-HNDIv1__Iqv7MKv_Kmsa2Y2KRlauFZmcEsZcCGP2Ynmsg-P9Zi89LEf2pTDB4LrOl14jhFYL6_jg3tt5gv58T7e62JPv7GaibR2qH-OXc4t1CfLh_v_u4K6dwVy-_AWgjs21SAwAA; Path=/; SameSite=Lax; Secure, AccessSessionSignature=c25f82ddb13822abe6012161bdbaeae5985a921235fa2d68e91e664ea1b463b5; Path=/; SameSite=Lax; Secure, AccessSessionTimedSignature=366b26f6d8a1fbfebe86bf6450035cc5ce58a825abdc3b7ccbc8209466fc6746; Path=/; SameSite=Lax; Secure, UUID=8f463723-e047-48e1-bece-2c00574c10d8; expires=Mon, 03 Jun 2024 11:13:18 GMT; Max-Age=94608000; Path=/; SameSite=None; Secure, csrftoken=R3wgkxs9ydV1V3qZH8hyU99Uxs7IbOe9KLNXChiRX626wb8q2ZE56hwBbliMVyF1; expires=Fri, 03 Jun 2022 11:13:18 GMT; Max-Age=31449600; Path=/; SameSite=Lax; Secure, ReferringRequestId=excelsior:095c6972196af449d128fd7d2d4d4fca; Path=/; SameSite=Lax; Secure, _pxhd=bfc2588208f2eccf03ce709939a0ef3d1f7ddf2f12512797bdf9409a6c0feb64:dc2c88a1-c525-11eb-aa81-df9c70a9b390; Expires=Fri, 01 Jan 2021 00:00:00 GMT; path=/;", + "Vary": "Cookie,Accept-Encoding,Fastly-SSL,Origin,X-Requested-Host", + "Via": "1.1 varnish", + "X-Cache": "MISS", + "X-Cache-Hits": "0", + "X-JSTOR-Restarts": "2", + "X-Served-By": "cache-fra19126-FRA", + "transfer-encoding": "chunked" + }, + "status_code": 200, + "url": "https://www.jstor.org/stable/30078788" +} \ No newline at end of file diff --git a/test/testdata/12042fbb4b8a50c78bf79df514dcf717abc9e45a.html b/test/testdata/12042fbb4b8a50c78bf79df514dcf717abc9e45a.html new file mode 100644 index 00000000..3e491f76 --- /dev/null +++ b/test/testdata/12042fbb4b8a50c78bf79df514dcf717abc9e45a.html @@ -0,0 +1 @@ +{"header":{"type":"esummary","version":"0.3"},"result":{"uids":["3538472"],"3538472":{"uid":"3538472","pubdate":"2012 Jun","epubdate":"","printpubdate":"2012 Jun","source":"Mayo Clin Proc","authors":[{"name":"Sweetser S","authtype":"Author"}],"title":"Evaluating the Patient With Diarrhea: A Case-Based Approach","volume":"87","issue":"6","pages":"596-602","articleids":[{"idtype":"pmid","value":"22677080"},{"idtype":"doi","value":"10.1016/j.mayocp.2012.02.015"},{"idtype":"pmcid","value":"PMC3538472"}],"fulljournalname":"Mayo Clinic Proceedings","sortdate":"2012/06/01 00:00","pmclivedate":"2013/01/11"}}} diff --git a/test/testdata/12042fbb4b8a50c78bf79df514dcf717abc9e45a.json b/test/testdata/12042fbb4b8a50c78bf79df514dcf717abc9e45a.json new file mode 100644 index 00000000..a189fd7f --- /dev/null +++ b/test/testdata/12042fbb4b8a50c78bf79df514dcf717abc9e45a.json @@ -0,0 +1,28 @@ +{ + "encoding": "UTF-8", + "headers": { + "Access-Control-Allow-Origin": "*", + "Access-Control-Expose-Headers": "X-RateLimit-Limit,X-RateLimit-Remaining", + "Cache-Control": "private", + "Connection": "Keep-Alive", + "Content-Security-Policy": "upgrade-insecure-requests", + "Content-Type": "application/json; charset=UTF-8", + "Date": "Thu, 09 Jun 2022 10:37:32 GMT", + "Keep-Alive": "timeout=4, max=40", + "NCBI-PHID": "D0BD347135095BB5000029BABE488B4A.1.1.m_1", + "NCBI-SID": "8A784C81F6E218AD_0658SID", + "Referrer-Policy": "origin-when-cross-origin", + "Server": "Finatra", + "Set-Cookie": "ncbi_sid=8A784C81F6E218AD_0658SID; domain=.nih.gov; path=/; expires=Fri, 09 Jun 2023 10:37:33 GMT", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "Transfer-Encoding": "chunked", + "X-RateLimit-Limit": "10", + "X-RateLimit-Remaining": "9", + "X-Test-Test": "test42", + "X-UA-Compatible": "IE=Edge", + "X-XSS-Protection": "1; mode=block", + "content-encoding": "gzip" + }, + "status_code": 200, + "url": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?api_key=dad47b304cafdc0790b32d335e3e3a403c08&retmode=json&tool=5j9.citer@github.com&email=dalba.wiki@gmail.com&db=pmc&id=3538472" +} \ No newline at end of file diff --git a/test/testdata/1264f1c6b97eb646a5b4dc58ad5398ad33e1bc95.html b/test/testdata/1264f1c6b97eb646a5b4dc58ad5398ad33e1bc95.html new file mode 100644 index 00000000..8b265b12 --- /dev/null +++ b/test/testdata/1264f1c6b97eb646a5b4dc58ad5398ad33e1bc95.html @@ -0,0 +1,11 @@ +@Book{noorlib18454, +Title = {المعجم الموضوعی لاحادیث الامام المهدی عجل الله تعالی فرجه الشریف}, +Year = {}, +Url = {http://www.noorlib.ir/View/fa/Book/BookView/Image/18454}, +publisher = {دار المرتضی}, +address = {بيروت}, +author = {کورانی, علی}, +Series = {المعجم الموضوعي لإحادیث الإمام المهدي (عجل الله فرجه الشریف)}, +Volume = {1} +Language = {عربی} +} \ No newline at end of file diff --git a/test/testdata/1264f1c6b97eb646a5b4dc58ad5398ad33e1bc95.json b/test/testdata/1264f1c6b97eb646a5b4dc58ad5398ad33e1bc95.json new file mode 100644 index 00000000..616dadb9 --- /dev/null +++ b/test/testdata/1264f1c6b97eb646a5b4dc58ad5398ad33e1bc95.json @@ -0,0 +1,16 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "private", + "Content-Disposition": "attachment; filename=\"noorlib-18454.bib\"", + "Content-Length": "480", + "Content-Type": "application/x-bibtex; charset=UTF-8", + "Date": "Tue, 23 May 2017 17:51:52 GMT", + "Server": "Microsoft-IIS/7.5", + "Set-Cookie": "ASP.NET_SessionId=k05g1dakfu1kkooojtyzf5mr; path=/; HttpOnly", + "X-AspNet-Version": "4.0.30319", + "X-Powered-By": "ASP.NET" + }, + "status_code": 200, + "url": "http://www.noorlib.ir/View/HttpHandler/CitationHandler.ashx?id=18454&format=BibTex" +} \ No newline at end of file diff --git a/test/testdata/14f9610681104e51a24848cbbb3de0b6f1297fef.html b/test/testdata/14f9610681104e51a24848cbbb3de0b6f1297fef.html new file mode 100644 index 00000000..7cbf3054 --- /dev/null +++ b/test/testdata/14f9610681104e51a24848cbbb3de0b6f1297fef.html @@ -0,0 +1,1410 @@ + + + + + + + + + + + + روانچی: در ارتباط با مواضع نامناسب اخیر مقامات انگلیسی در مورد ایران گفت‌وگو خواهیم کرد - ایسنا + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + +
+
+
+
+
+
+
+
+ +
    +
  • چهارشنبه / ۲۹ دی ۱۳۹۵ / ۱۴:۵۹
  • +
  • دسته‌بندی: + سیاست خارجی + +
  • +
  • کد خبر: 95102918901
  • +
  • خبرنگار : 71038
  • + +
+
+
+ +
+
+
+ +

با اعلام خبر سفر معاون وزیر خارجه انگلیس به تهران

+

روانچی: در ارتباط با مواضع نامناسب اخیر مقامات انگلیسی در مورد ایران گفت‌وگو خواهیم کرد

+
+
+
+
+ مصاحبه اختصاصی با تخت روانچی +
+

معاون اروپا و آمریکای وزیر امور خارجه با اشاره به دیدار معاون وزیر امور خارجه انگلیس با وی در روز چهارشنبه در تهران، اظهار کرد: در جریان این دیدار در ارتباط با مسائل دوجانبه، منطقه‌ای، بین‌المللی و برجام گفت‌وگو خواهیم کرد.

+ +

مجید تخت روانچی در گفت‌وگو با خبرنگار ایسنا،  با بیان این که این دیدار عصر امروز برگزار می شود ، اعلام کرد: در این ملاقات همچنین نسبت به مواضع نامناسبی که اخیراً توسط مقامات انگلیسی در مورد ایران اعلام شده گفت‌وگو و صحبت خواهیم کرد.

+ +

 ترزا می، نخست وزیر انگلیس، چند هفته پیش  ضمن حضور در نشست سران کشورهای عضو شورای همکاری خلیج فارس از لزوم همکاری با کشورهای این حوزه در برابر فعالیت‌های منطقه‌ای ایران سخن گفته و ادعاهایی را علیه ایران مطرح کرد.

+ +

در این نشست، می ضمن تشریح سیاست‌های دولت خود در زمینه روابط با کشورهای عضو شورای همکاری خلیج فارس، به نقش ایران در منطقه نیز پرداخت و گفت که انگلیس آماده است با کشورهای عضو شورا برای مقابله با آنچه وی «اقدامات تهاجمی ایران در منطقه» خواند، همکاری کند. 

+ +

نخست وزیر انگلیس اظهار کرد : ما باید همچنان به مقابله با دولت‌هایی که نفوذ آن ها بی‌ثباتی منطقه را مشتعل می‌کند، ادامه دهیم و افزود: «بنابراین، می‌خواهم به شما اطمینان دهم که من به وضوح تهدیدی را که ایران متوجه منطقه خلیج فارس و در بُعدی وسیعتر، متوجه خاورمیانه می‌کند، مشاهده می‌کنم».

+ +

 این اظهارات در همان زمان با واکنش شدید جمهوری اسلامی ایران و دستگاه دیپلماسی کشور مواجه شد و بهرام قاسمی سخنگوی وزارت امور خارجه در این ارتباط گفت : کشورهایی که مداخله‌جویی‌های غیر مسئولانه آنها در سایر کشورها موجب گسترش ناامنی، جنگ، خشونت و تروریسم شده است در جایگاهی نیستند که دیگران را به مداخله در امور منطقه متهم نمایند.

+ +

وی با اشاره به سیاست‌های تفرقه افکنانه بریتانیا افزود: این کشور در تلاش برای بازگشت به این منطقه، مجددا به سیاست های تفرقه افکنانه روی آورده است که از دیدگاه جمهوری اسلامی ایران کاری عبث و غیرسازنده است.

+ +

قاسمی اضافه کرد: جمهوری اسلامی ایران، ریشه بخشی از این اظهارات را ناشی از تحولات در روابط این کشور با اتحادیه اروپایی می داند که مشکلات، کمبودها و پیچیدگی هایی را در منافع و جایگاه بین المللی انگلیس ایجاد کرده و باعث شده است نخست وزیر این کشور متناسب با فضای اجلاس شورای همکاری خلیج فارس و برای خوشایند تعدادی از سران کشورهای عضو این شورا، حرف هایی نسنجیده علیه دولت و ملت ایران بر زبان بیاورد.

+ +

سخنگوی وزارت خارجه در پایان گفت: به نظر می رسد هدف از این گونه اظهارات تلاش برای عقد قراردادهای جدید هنگفت تسلیحاتی بین انگلیس و برخی کشورهای عرب حاشیه خلیج فارس و در نهایت، تشدید بحران های ناشی از جنایات جنگی آنها علیه ملت های مظلوم یمن، سوریه، بحرین، عراق و دیگر کشورهای اسلامی منطقه باشد.

+ +

مشروح  گفت‌وگوی مجید تخت روانچی با خبرنگاران هسته‌ای و سیاست خارجی ایسنا طی روزهای آتی ارسال می‌شود .

+ +

انتهای پیام

+ +

+ +
+ + + +
+ +
+ +
+
+ +
+
+
+
+ +
  • در زمینه انتشار نظرات مخاطبان رعایت چند مورد ضروری است:
  • -لطفا نظرات خود را با حروف فارسی تایپ کنید.
  • -«ایسنا» مجاز به ویرایش ادبی نظرات مخاطبان است.
  • - ایسنا از انتشار نظراتی که حاوی مطالب کذب، توهین یا بی‌احترامی به اشخاص، قومیت‌ها، عقاید دیگران، موارد مغایر با قوانین کشور و آموزه‌های دین مبین اسلام باشد معذور است.
  • - نظرات پس از تأیید مدیر بخش مربوطه منتشر می‌شود.
+
+
+
+
+

نظرات

+
+
+ +
+
+
شما در حال پاسخ به نظر «» هستید. + +
+
+
+ + + + +
+
+ +
+
+
+ + + +
+
+
+
+
+ +
+
+
+
+
+
+
+ + +
+
+
+
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/test/testdata/14f9610681104e51a24848cbbb3de0b6f1297fef.json b/test/testdata/14f9610681104e51a24848cbbb3de0b6f1297fef.json new file mode 100644 index 00000000..615648f0 --- /dev/null +++ b/test/testdata/14f9610681104e51a24848cbbb3de0b6f1297fef.json @@ -0,0 +1,29 @@ +{ + "encoding": "UTF-8", + "headers": { + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Type": "text/html;charset=UTF-8", + "Date": "Sat, 27 May 2017 14:57:46 GMT", + "Link": "; rel=\"original\", ; rel=\"timemap\"; type=\"application/link-format\", ; rel=\"timegate\", ; rel=\"first memento\"; datetime=\"Thu, 19 Jan 2017 05:00:01 GMT\", ; rel=\"memento\"; datetime=\"Thu, 19 Jan 2017 05:00:01 GMT\", ; rel=\"last memento\"; datetime=\"Thu, 19 Jan 2017 05:00:01 GMT\"", + "Memento-Datetime": "Thu, 19 Jan 2017 05:00:01 GMT", + "Server": "Tengine/2.1.0", + "Transfer-Encoding": "chunked", + "X-Archive-Guessed-Charset": "UTF-8", + "X-Archive-Orig-accept-ranges": "bytes", + "X-Archive-Orig-age": "25", + "X-Archive-Orig-connection": "close", + "X-Archive-Orig-content-length": "102701", + "X-Archive-Orig-date": "Thu, 19 Jan 2017 04:59:37 GMT", + "X-Archive-Orig-grace": "normal(limited)", + "X-Archive-Orig-server": "Apache-Coyote/1.1", + "X-Archive-Orig-vary": "Accept-Encoding", + "X-Archive-Orig-via": "1.1 varnish-v4", + "X-Archive-Orig-x-varnish": "981271057 985728102", + "X-Archive-Playback": "0", + "X-Page-Cache": "HIT", + "X-location": "All" + }, + "status_code": 200, + "url": "https://web.archive.org/web/20170119050001/http://www.isna.ir/news/95102918901/%D8%B1%D9%88%D8%A7%D9%86%DA%86%DB%8C-%D8%AF%D8%B1-%D8%A7%D8%B1%D8%AA%D8%A8%D8%A7%D8%B7-%D8%A8%D8%A7-%D9%85%D9%88%D8%A7%D8%B6%D8%B9-%D9%86%D8%A7%D9%85%D9%86%D8%A7%D8%B3%D8%A8-%D8%A7%D8%AE%DB%8C%D8%B1-%D9%85%D9%82%D8%A7%D9%85%D8%A7%D8%AA-%D8%A7%D9%86%DA%AF%D9%84%DB%8C%D8%B3%DB%8C-%D8%AF%D8%B1-%D9%85%D9%88%D8%B1%D8%AF" +} \ No newline at end of file diff --git a/test/testdata/1572458f9a8f4e16036fba9b3e6d7e7c9153249a.html b/test/testdata/1572458f9a8f4e16036fba9b3e6d7e7c9153249a.html new file mode 100644 index 00000000..42bb3039 --- /dev/null +++ b/test/testdata/1572458f9a8f4e16036fba9b3e6d7e7c9153249a.html @@ -0,0 +1,2238 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +HuffPost + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + +
+ + +
+
+
+
+
+ +
+
+ +
+ + +
+
+
+ +
+ + +
+ +
+ + + + + + +
+ + +
+ +
+ + + + +
+
+ +
+
+

TOP VIDEOS

+
+
+
+
+ + +
+ +
+
+ + +
+
+
+
+ + +
+ +
+
+ + +
+
+ +
+
+
+
+ + + + + +
+
+
+
+

IN THE NEWS

+
+
+ + + + + + + + + + + + + +
+
+
+
+ +
+ + + +
+ + + + + +
+ + +
+ + + + + + + + + + + +
+ + +
+ + + + + + + + + + + + + + +
+
+ + + + + + + + + + + + + + + + + +
+ + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ + diff --git a/test/testdata/1572458f9a8f4e16036fba9b3e6d7e7c9153249a.json b/test/testdata/1572458f9a8f4e16036fba9b3e6d7e7c9153249a.json new file mode 100644 index 00000000..35924ad6 --- /dev/null +++ b/test/testdata/1572458f9a8f4e16036fba9b3e6d7e7c9153249a.json @@ -0,0 +1,27 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "29", + "Cache-Control": "max-age=30, public, must_revalidate=false", + "Content-Encoding": "gzip", + "Content-Length": "46630", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:53:24 GMT", + "Last-Modified": "Tue, 23 May 2017 17:53:24 GMT", + "Server": "ECD (iad/B9BE)", + "Vary": "Accept-Encoding", + "X-Cache": "HIT", + "X-Content-Type-Options": "nosniff", + "X-EC-Lua": "19365-geo", + "X-Frame-Options": "ALLOWALL", + "X-HP-Trace-ID": "7R74FD9W", + "X-HP-Trace-Project": "HPMW/production/70604bb", + "X-Mobile-URL": "http://m.huffpost.com/", + "X-Request-Id": "ee04722c-97ce-4aaa-b765-ef8dfca761ae", + "X-Runtime": "0.049307", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "http://www.huffingtonpost.com/" +} \ No newline at end of file diff --git a/test/testdata/17f61fda8e3201d799e1a8048d4fa0a0dd69ea5b.html b/test/testdata/17f61fda8e3201d799e1a8048d4fa0a0dd69ea5b.html new file mode 100644 index 00000000..92f388e3 --- /dev/null +++ b/test/testdata/17f61fda8e3201d799e1a8048d4fa0a0dd69ea5b.html @@ -0,0 +1,1380 @@ + + + + + + + Dynamometers Explained - Dealernews | HighBeam Research + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + +
+ +
+ + + + + + +
+ + + + + + + diff --git a/test/testdata/17f61fda8e3201d799e1a8048d4fa0a0dd69ea5b.json b/test/testdata/17f61fda8e3201d799e1a8048d4fa0a0dd69ea5b.json new file mode 100644 index 00000000..9c74cdd3 --- /dev/null +++ b/test/testdata/17f61fda8e3201d799e1a8048d4fa0a0dd69ea5b.json @@ -0,0 +1,21 @@ +{ + "encoding": "utf-8", + "headers": { + "Cache-Control": "no-cache, no-store, must-revalidate", + "Content-Encoding": "gzip", + "Content-Length": "14139", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:54:03 GMT", + "Expires": "-1", + "Pragma": "no-cache", + "Server": "Microsoft-IIS/8.5", + "Set-Cookie": "ASP.NET_SessionId=tsb3df03uw4bkkq0qp00dmpj; path=/; secure; HttpOnly, FirstVisit=repeat; domain=highbeam.com; expires=Sun, 19-Nov-2017 18:54:04 GMT; path=/; secure, RVI=RVI1=[a href=/doc/1P3-3372742961.html]Article: Dynamometers Explained[/a]; domain=highbeam.com; expires=Sun, 19-Nov-2017 18:54:04 GMT; path=/; secure", + "Vary": "Accept-Encoding", + "X-AspNet-Version": "4.0.30319", + "X-AspNetMvc-Version": "4.0", + "X-FRAME-OPTIONS": "SAMEORIGIN", + "X-Powered-By": "ASP.NET" + }, + "status_code": 200, + "url": "https://www.highbeam.com/doc/1P3-3372742961.html" +} \ No newline at end of file diff --git a/test/testdata/17fb0e028bc53ad8a42167dacc182df957a65dda.html b/test/testdata/17fb0e028bc53ad8a42167dacc182df957a65dda.html new file mode 100644 index 00000000..5f1b2092 --- /dev/null +++ b/test/testdata/17fb0e028bc53ad8a42167dacc182df957a65dda.html @@ -0,0 +1,1729 @@ + + + + + + + + + + Epidemics expert questions Marshall's schools advice - InDaily + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+

+ + InDaily + InDaily + +

+ + + + + + + + + +
+ Support independent Journalism + Donate + Subscribe +
+
+ + + + + + +
+ +
+
+ Support independent + journalism +
+ +
+
+ +
+
+
+ +
+

+ + News +

+ + + + +
+ + + +
+
+

Epidemics expert questions Marshall's schools advice

+

+ + News +

+
+

An Australian epidemiologist recognised for her work combating Ebola virus says South Australian parents not working in essential services should take their children out of school, contradicting Premier Steven Marshall’s strong health advice to the public yesterday.

+

+
+ +
+ + + + +
+ +
+ + Print article +
+
+ + + +

At a press conference yesterday, Marshall told reporters “students must remain at school” in the wake of the COVID-19 coronavirus outbreak.

+

“Children should go to school, here in SA and around the country,” he said.

+

“Not to do so doesn’t diminish the risk – it increases the risk and reduces our response as a nation to tackle the coronavirus.”

+

He said that advice from chief medical officers from across the country to the national emergency cabinet was “unequivocal and there was no discussions or dissent from any of the states or commonwealth … we are as one – we want students to go to school”.

+ + + +

At about the same time yesterday, Dr Kamalini Lokuge OAM was speaking at a National Press Club forum on COVID-19 in Canberra.

+

Lokuge, of the National Centre for Epidemiology at the Australian National University, is a medical doctor, an epidemiologist and a senior research fellow leading the ANU’s Humanitarian Health Research Initiative.

+

She was involved in curbing the spread of Ebola across West Africa and has worked with Doctors Without Borders, the World Health Organisation and the International Committee of the Red Cross on humanitarian crises across the globe.

+

Her message to the forum was that schools should play a part in social distancing, with some caveats.

+

“Those people who can take their kids out of school, without having to leave them with grandparents, should,” she said.

+

She said there were “some people who need to send their kids to school – our essential workers; our doctors; our nurses; those who supply our food, our electricity – they need to be able to send their kids to school.

+

“And if we reduce the number of kids in school, we reduce what we call the force of infection in schools.

+

“So, kids are going to have less contact with other kids and therefore there (are at) less risk of those kids, who do need to be in school, passing infection onto their family.”

+

In an interview with InDaily this morning, Lokuge stood by her comments, stressing that the Australia’s education system should be preparing itself to deliver schooling mostly online.

+

“This is my view as an epidemiologist,” she said.

+

“We need to do a whole range of social distancing measures that are going to be very difficult for the community.

+

“While transmission is very low in the community it’s okay (that Australian children are still in school) but if we get widespread community transmission, schools are going to be another point … of infection between people and between households.

+

“Australia is a world leader in remote teaching … we need to develop strategies for online teaching as soon as possible.”

+ + + +

She argued that it was not a question of whether or not to close schools altogether – because the children of parents working in essential services needed to stay in the classroom – but rather a question of how to prevent transmission as much as possible with social distancing measures, and that means withdrawing most other kids from class attendance.

+

She said parents thinking of taking their children out of school should make sure that they are not exposing them to others at higher risk of COVID-19, such as the elderly or people with compromised immune or respiratory systems.

+

And “make sure they are provided with a good learning environment,” she added.

+

During his press conference yesterday, Marshall noted that “parents have the right to make their own decisions” but insisted they also had an obligation to provide adequate education during the layoff period, urging parents to understand that if students were taken out of the school system “they are not in for one or two weeks, they’re not in for one or two months – the expectation is this [situation] will remain in place for six months or more”.

+

“This is not political, it’s not ideological – it’s an evidence-based decision,” he added.

+

A spokesperson for Marshall told InDaily this morning that the Premier was acting on the advice of the country’s leading health experts when he recommended students stay in school.

+

“The Premier could not be clearer,” the spokesperson said.

+

“We are acting on the very clear advice from the Australian Health Protection Principal Committee on this matter – who are the leading experts on infection control.”

+

The AHPPC is the key decision-making committee for health emergencies in Australia, comprised of all state and territory Chief Health Officers, and is chaired by Australian Chief Medical Officer Brendan Murphy.

+

Marshall’s comments were in line with those of Prime Minister Scott Morrison, who told Sky News yesterday afternoon: “There’s only one reason your kids shouldn’t be going to school and that is if they are unwell.”

+

On Tuesday, The Guardian reported that almost 2500 doctors have signed a letter to Australian Health Minister Greg Hunt, saying schools and public places should be closed around the country in an effort to contain the coronavirus.

+

Led by Dr Hemant Garg, the letter states doctors are “dismayed at the disconnect between the actions being taken within the medical community and the recommendation for actions being passed on to the general population”, according to the report.

+

“We should immediately recommend a three to four week closure of schools, cultural and religious places including places of worship, gyms and leisure centres, pubs, bars, theatres, cinemas and concert halls,” the letter states.

+

“This would allow a steady declaration of cases of coronavirus to present to hospitals and fever clinics as their symptomatic phase develops.”

+

However, South Australia’s acting Chief Medical Officer Dr Michael Cusack told FIVEaa radio this morning that “in South Australia at the moment it is the right thing for us to be keeping our schools open”.

+

He said the state’s Chief Public Health Officer Dr Nicola Spurrier had been working with her colleagues interstate “and together they’ve been looking with scientists and international evidence and where things have worked well, where things have worked less well – and on the basis of all the evidence, keeping schools open and children going to school is the right thing for us to be doing at this point in time.

+

“The World Health Organisation did a detailed review (where) they looked at something like 55,000 cases of coronavirus and where children had the disease they had almost universally picked the disease up at home from a close family member as opposed to at school,” he said.

+

“There’s no doubt that children can get the disease, but in terms of the mode of transmission that does not appear to be through the school.”

+

At a press conference this afternoon, Australia’s deputy Chief Medical Officer Professor Paul Kelly said Australia would not be closing schools at this stage because “we are looking for a proportionate response that is sustainable across several months”.

+

“At the moment for us that is not necessary in relation to schools.”

+

He added that there was international evidence that children did not appear to be transmitting the virus to other children, but rather that they were getting the virus from their relatives.

+

Meanwhile, several countries have been implementing strict lockdowns in response to the pandemic.

+

Britain has ordered schools, nurseries and colleges to close for millions of children until further notice after criticism that the government was being too slow to react to the spread of coronavirus.

+ + + +

The UK had previously resisted pressure to follow the lead of Italy, France, and Spain, saying that school closures would not halt the outbreak and would deprive the country of key public sector workers.

+

Most British schools will close from Friday, although some will be asked to stay open to support the children of essential workers like health care employees, UK education minister Gavin Williamson told parliament.

+

“I know the situation has become increasingly challenging. I said before that if the science and the advice changed such that keeping schools open would no longer be in the best interests of children and teachers that we would act,” he said.

+

“We are now at that stage. The spike of the virus is increasing at a faster pace than anticipated.”

+

The shutting of so many schools will have huge economic and social repercussions for the world’s fifth-biggest economy, altering the lives of almost 9 million British children and force parents to stay home from work to look after them.

+

– with AAP

+
+
+ Make a comment + View comment guidelines + +
+ + +
+ +
+
+ +

Make your contribution to independent news

+

A donation of any size to InDaily goes directly to helping our journalists uncover the facts. South Australia needs more than one voice to guide it forward, and we’d truly appreciate your contribution. Please click below to donate to InDaily.

+
+ + Donate here + +
+ Powered by + PressPatron +
+ +
+
+ + + + + + +
+
+ + +
+
+

More News stories

+
+ +
+ +
+
+ +
Loading next article
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/testdata/17fb0e028bc53ad8a42167dacc182df957a65dda.json b/test/testdata/17fb0e028bc53ad8a42167dacc182df957a65dda.json new file mode 100644 index 00000000..2d8913e2 --- /dev/null +++ b/test/testdata/17fb0e028bc53ad8a42167dacc182df957a65dda.json @@ -0,0 +1,21 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "max-age=600, must-revalidate", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Type": "text/html; charset=UTF-8", + "Date": "Fri, 08 Jan 2021 12:55:28 GMT", + "Keep-Alive": "timeout=20", + "Link": "; rel=\"https://api.w.org/\", ; rel=\"alternate\"; type=\"application/json\", ; rel=shortlink", + "Server": "nginx", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding, Accept-Encoding, Accept-Encoding,Cookie", + "X-Cache": "HIT: 1", + "X-Cache-Group": "normal", + "X-Cacheable": "SHORT", + "X-Powered-By": "WP Engine" + }, + "status_code": 200, + "url": "https://indaily.com.au/news/2020/03/19/epidemics-expert-contradicts-marshalls-schools-advice/" +} \ No newline at end of file diff --git a/test/testdata/17fb77a7df637fd9d5af2fc80c14f789df0de8a5.html b/test/testdata/17fb77a7df637fd9d5af2fc80c14f789df0de8a5.html new file mode 100644 index 00000000..bae6ea05 --- /dev/null +++ b/test/testdata/17fb77a7df637fd9d5af2fc80c14f789df0de8a5.html @@ -0,0 +1,409 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Sudan Tech Sanctions Harm Innovation and Development: US Government and Corporations Must Act | Electronic Frontier Foundation + + + + + + + + + + + + + Skip to main content + + + + + + + +
+ +
+

+
+
+ +
+ + +
+
+ +
+
+ DEEPLINKS BLOG
+ +
+

Sudan Tech Sanctions Harm Innovation and Development: US Government and Corporations Must Act

+
+
June 26, 2014
+ +
+
+
+
+ + +
+

It’s no secret that EFF is strongly opposed to the United States’ piecemeal approach to updating sanction provisions for the five U.S.-embargoed countries of Sudan, Syria, Cuba, North Korea, and Iran.  We’ve noted that the fundamental problem with the United States’ reform method is that it’s “largely reactionary and ultimately prioritizes certain countries over others for reasons that are, to put it charitably, hard to discern.” For example, according to an article published by the Open Technology Institute, the Office of Foreign Assets Control (OFAC) issued Iran a new General License D-1—which replaces the old General License D—making it acceptable for U.S. companies to offer technology tools to Iran such as laptops and anti-virus software.  Similar allowances have been made for Syria. Despite years of advocacy, Syrians did not enjoy greater access to technology until after civil war broke out in the country. Recognizing the need for communications technologies, the Department of Treasury issued a general license (§ 542.511) allowing for the access of “instant messaging, chat and email, social networking, sharing of photos and movies, web browsing, and blogging ... provided that such services are publicly available at no cost to the user.”

+

Sudanese citizens have not enjoyed the same provisions.  In fact, U.S. sanctions in Sudan actually “inadvertently aid the regime by blocking access to critical personal communications tools.”  The simplest explanation for why sanction reforms have not yet occurred in Sudan seem to be a simple lack of political attention. In Iran,  a greater capacity and market demand for technology led to a reconsideration of sanctions, while in Syria, the civil war triggered an advocacy effort to ensure access on the ground to key communications technologies.  Unfortunately in Sudan—where 21 percent of the country’s 37 million citizens are online—people remain cut off from many important technologies, from medical resource sites to massive open online courses (MOOCs) and the Google Play store.

+

As we’ve written before, sanctions are only part of the problem.  Since OFAC restrictions limit access to goods, technologies, and services from the U.S. or by a U.S.-person, corporate lawyers are often overly cautious, resulting in overbroad restrictions on access.  For example, in 2009 Linkedin, in an effort to protect itself from liability, made the decision to delete the accounts of users in Syria, a decision that also affected usability in Iran, North Korea, Cuba, and Sudan. It wasn’t until after the company was called out for being overly cautious that they reinstated service to Syrian users, admitting overcompliance with export controls restrictions.  SourceForge took similar action in 2010, and Apple and Airbnb have both been called out for restrictions placed on Iranians.

+

Demand for many of the banned technologies and websites are high. Dalia Haj Omar, a Sudanese activist and blogger, told us via e-mail that MOOCs and other online educational programs are “in great demand, especially from a younger population that is turning to online education,” in part because of a 1989 decision by the government to Arabize school curricula.  “Many youth realize they can't compete regionally or nationally if they don't have better education,” says Haj Omar. “Some universities are also turning to MOOCs to supplement their curriculums, since access to hard copy books is hard and expensive.”

+

Sudanese activists are calling for a general license similar to those issued for Iran and Syria.  Such a measure would provide residents of the country with much-desired access to sites like Mathworks.com, which provides engineers and scientists with software to discover, research, and innovate; anti-virus software updates from companies like Norton and AVG; and developer sites like SourceForge.

+

In the meantime, companies can help ease the pain of deprivation by applying for individual licenses.  A company that wishes to export to Sudan can file an online application with OFAC for a license.  Alternatively, companies may also request “interpretative guidance” as to whether or not they require a license.

+

Is your company looking to apply for a license? EFF wants to help!

+

We challenge those companies who are concerned about these restrictions to take the simple steps necessary to apply for a license.  In fact, this is so important to us that EFF is willing to help companies that want to take these steps but don’t have the resources to do so. Please contact EFF's Legal Director, Cindy@eff.org, if you'd like our help.

+

In limiting access to these sites, the Department of Treasury is unjustly preventing Sudanese from accessing information and technologies that are necessary for the advancement, innovation, and democracy of the country.  And the fact that users in other U.S.-sanctioned countries sometimes have access to these technologies, while Sudan is left on the sidelines to watch, is just a slap in the face.

+ +
+ +
+
+ +
+
+ +
+
+
+
+
+
+ + + + + + JavaScript license information
+ + + diff --git a/test/testdata/17fb77a7df637fd9d5af2fc80c14f789df0de8a5.json b/test/testdata/17fb77a7df637fd9d5af2fc80c14f789df0de8a5.json new file mode 100644 index 00000000..838d6e02 --- /dev/null +++ b/test/testdata/17fb77a7df637fd9d5af2fc80c14f789df0de8a5.json @@ -0,0 +1,34 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "0", + "Cache-Control": "public, max-age=1800", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Language": "en", + "Content-Length": "9466", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:55:06 GMT", + "Etag": "\"1495562105-1\"", + "Expires": "Sun, 19 Nov 1978 05:00:00 GMT", + "Fastly-Debug-Digest": "0592febd108b431e7f8412dc3d37ad1e53f08d1225cb5019c1466290960d48fe", + "Last-Modified": "Tue, 23 May 2017 17:55:05 GMT", + "Link": "; rel=\"canonical\",; rel=\"shortlink\",; rel=\"publisher\"", + "Server": "nginx", + "Strict-Transport-Security": "max-age=31536000; includeSubdomains; preload", + "Vary": "Cookie,fastly-ssl,Accept-Encoding", + "Via": "1.1 varnish, 1.1 varnish, 1.1 varnish", + "X-Cache": "MISS, MISS", + "X-Cache-Hits": "0, 0", + "X-Content-Type-Options": "nosniff", + "X-Drupal-Cache": "MISS", + "X-Frame-Options": "SAMEORIGIN", + "X-Generator": "Drupal 7 (http://drupal.org)", + "X-Served-By": "cache-sjc3137-SJC, cache-iad2638-IAD", + "X-Timer": "S1495562106.658405,VS0,VE1099", + "X-UA-Compatible": "IE=edge,chrome=1" + }, + "status_code": 200, + "url": "https://www.eff.org/deeplinks/2014/06/sudan-tech-sanctions-harm-innovation-development-us-government-and-corporations-must-act" +} \ No newline at end of file diff --git a/test/testdata/1819f33f317f43c5cc82884b0670bde143816f85.html b/test/testdata/1819f33f317f43c5cc82884b0670bde143816f85.html new file mode 100644 index 00000000..533f716c --- /dev/null +++ b/test/testdata/1819f33f317f43c5cc82884b0670bde143816f85.html @@ -0,0 +1,62 @@ +{ + "header": { + "type": "esummary", + "version": "0.3" + }, + "result": { + "uids": [ + "2562006" + ], + "2562006": { + "uid": "2562006", + "pubdate": "2008 Aug 22", + "epubdate": "2008 Aug 22", + "printpubdate": "2008 Oct 15", + "source": "Bioinformatics", + "authors": [ + { + "name": "Bannen RM", + "authtype": "Author" + }, + { + "name": "Suresh V", + "authtype": "Author" + }, + { + "name": "Phillips GN Jr", + "authtype": "Author" + }, + { + "name": "Wright SJ", + "authtype": "Author" + }, + { + "name": "Mitchell JC", + "authtype": "Author" + } + ], + "title": "Optimal design of thermally stable proteins", + "volume": "24", + "issue": "20", + "pages": "2339-2343", + "articleids": [ + { + "idtype": "pmid", + "value": "18723523" + }, + { + "idtype": "doi", + "value": "10.1093/bioinformatics/btn450" + }, + { + "idtype": "pmcid", + "value": "PMC2562006" + } + ], + "fulljournalname": "Bioinformatics", + "sortdate": "2008/08/22 00:00", + "pmclivedate": "2009/02/25" + } + } +} + diff --git a/test/testdata/1819f33f317f43c5cc82884b0670bde143816f85.json b/test/testdata/1819f33f317f43c5cc82884b0670bde143816f85.json new file mode 100644 index 00000000..46286ade --- /dev/null +++ b/test/testdata/1819f33f317f43c5cc82884b0670bde143816f85.json @@ -0,0 +1,26 @@ +{ + "encoding": "UTF-8", + "headers": { + "Access-Control-Allow-Origin": "*", + "Cache-Control": "private", + "Connection": "Keep-Alive", + "Content-Security-Policy": "upgrade-insecure-requests", + "Content-Type": "application/json; charset=UTF-8", + "Date": "Fri, 01 Mar 2019 07:22:42 GMT", + "Keep-Alive": "timeout=4, max=40", + "NCBI-PHID": "322CBF38DE76A3650000206D63887E96.1.1.m_1", + "NCBI-SID": "AAE5B6BEDCB8ED3A_AC5ESID", + "Server": "Finatra", + "Set-Cookie": "ncbi_sid=AAE5B6BEDCB8ED3A_AC5ESID; domain=.nih.gov; path=/; expires=Sun, 01 Mar 2020 07:22:42 GMT", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "Transfer-Encoding": "chunked", + "X-RateLimit-Limit": "3", + "X-RateLimit-Remaining": "3", + "X-UA-Compatible": "IE=Edge", + "X-XSS-Protection": "1; mode=block", + "content-encoding": "gzip", + "l5d-success-class": "1.0" + }, + "status_code": 200, + "url": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?api_key=dad47b304cafdc0790b32d335e3e3a403c08&retmode=json&tool=5j9.citer@github.com&email=dalba.wiki@gmail.com&db=pmc&id=2562006" +} \ No newline at end of file diff --git a/test/testdata/18e867d4b3e19cdf8ce4f40799aa3e7e1a46ba7e.html b/test/testdata/18e867d4b3e19cdf8ce4f40799aa3e7e1a46ba7e.html new file mode 100644 index 00000000..359d7314 --- /dev/null +++ b/test/testdata/18e867d4b3e19cdf8ce4f40799aa3e7e1a46ba7e.html @@ -0,0 +1,9 @@ +TY - BOOK +T1 - So You Want to Sing World Music: A Guide for Performers +A1 - Hoch, M. +SN - 9781538112281 +T3 - So You Want to Sing +UR - https://books.google.com/books?id=OlCwDwAAQBAJ +Y1 - 2019 +PB - Rowman & Littlefield Publishers +ER - diff --git a/test/testdata/18e867d4b3e19cdf8ce4f40799aa3e7e1a46ba7e.json b/test/testdata/18e867d4b3e19cdf8ce4f40799aa3e7e1a46ba7e.json new file mode 100644 index 00000000..69e12eea --- /dev/null +++ b/test/testdata/18e867d4b3e19cdf8ce4f40799aa3e7e1a46ba7e.json @@ -0,0 +1,21 @@ +{ + "encoding": null, + "headers": { + "Alt-Svc": "h3-29=\":443\"; ma=2592000,h3-T051=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"", + "Cache-Control": "private, max-age=0", + "Content-Disposition": "attachment; filename=So_You_Want_to_Sing_World_Music.ris", + "Content-Length": "252", + "Content-Type": "application/x-research-info-systems", + "Date": "Wed, 05 May 2021 15:22:52 GMT", + "Expires": "Wed, 05 May 2021 15:22:52 GMT", + "P3P": "CP=\"This is not a P3P policy! See g.co/p3phelp for more info.\"", + "Server": "OFE/0.1", + "Set-Cookie": "NID=215=H4Gem5lKvx1QoS3hzHx6-q2VT5zzhxMQrCfTM60mPj8IPzPsdAHQK6cXI7UOlCsyg5iEzLe-uBiXcxtgRwI8AC5JemyLyWZOR0DU2vIYXtwdBvyuVyZPgUl58EpeG5BIMeXzq5H4Q4_HaRQYMGjbuJaD7oCrzdTxjW30zt3Bv5Q; expires=Thu, 04-Nov-2021 15:22:52 GMT; path=/; domain=.google.com; Secure; HttpOnly; SameSite=none", + "Strict-Transport-Security": "max-age=604800", + "X-Content-Type-Options": "nosniff", + "X-Frame-Options": "SAMEORIGIN", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://books.google.com/books/download/?id=OlCwDwAAQBAJ&output=ris" +} \ No newline at end of file diff --git a/test/testdata/1a33b865b87875d535e04b2405c29d0026497b02.html b/test/testdata/1a33b865b87875d535e04b2405c29d0026497b02.html new file mode 100644 index 00000000..9647e2a4 --- /dev/null +++ b/test/testdata/1a33b865b87875d535e04b2405c29d0026497b02.html @@ -0,0 +1,2827 @@ + + + + + + + + + + + + + +We could see the whale's eyes, mouth... the barnacles on its back - Telegraph + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + +
+ + +
+
+ +
Telegraph.co.uk
+
+ +
+
+
+

+ Tuesday 23 May 2017

+
+ +
+
+ +
+
+
+
+ + + +
+
+ +
+
    +
+
+
+
+
+ + +
+
+
Advertisement
+
+ +
+ + + + + + +
+
+
+
    +
  1. Home»
  2. +
  3. News»
  4. +
  5. Health
+
+
+
+ + +
+
+ +
+
+ + + +
+ +

We could see the whale's eyes, mouth... the barnacles on its back

+ + + +
+
+
+
+
+
+
+ Ben Fogle +
+ +
+ Ben Fogle is down to his last two pairs of shortsMore pictures 
+
+
+
+
+
+ +
+ +
+ +

James Cracknell, Olympic gold medalist, and Ben Fogle, television presenter, spent last week huddled in the cabin of their boat with the sea anchor down. This year, the Atlantic Rowing Race from the Canary Islands to Antigua has been plagued by the worst weather in its history and, for a week, the pair have been unable to make progress. On Tuesday, the wind began to change. Ben Fogle takes up the story with Cassandra Jardine....

We got away at midnight, but it was not until Tuesday midday that the wind finally moved to the north-east and we started to make real progress. With the wind and the sea behind us, we've increased our speed to three knots an hour. It's fantastic, having spent the previous week being bored out of our minds. Some of our pack of cards had gone missing and we hadn't packed any rainy-day things such as jigsaws, so there was nothing to do.

The best moment was our amazing encounter with a whale. I saw this huge white thing coming towards us. It passed so close that we could feel its back scraping the underside of the boat. It was incredibly graceful and as it moved under the boat, we could see its huge shadow, like a submarine. It was so close that we could see its eyes, its mouth, even the barnacles on its back. One flip and it could have turned us over. Thankfully, it didn't think we were another whale and try to mate with us.

We can also be glad that we haven't had a shark encounter. One of the other boats has; they had to hide in the cabin while it attacked their boat. As we get closer to the Caribbean, there will be more sharks and we won't be punching them on the nose, we'll also be hiding. We have to go into the water once a week to maintain the boat, so it's an alarming thought.

It's great to be rowing again, but although the old blisters had time to dry up while we were on the sea anchor, new blisters have now formed on top of the old ones. I have 12 of them, but our bottoms are in the worst state as this is a relentless process, sitting in the same place for hours every day. It's like having bedsores - and it's only going to get worse.

Before long we may well be forced to go naked, because I am down to two pairs of shorts and James has had to borrow my last pair of pants because things just seem to get lost. If we lose those, we will have no protection from the seats and the weather is getting hotter as we go further west.

But our main problem is food. When you are bored, you just want to eat and we can't, although we think about food all the time. We've been at sea for three weeks but have rations for only 50 days. We are likely to be here for another month, so must cut back on our calories and we are both getting noticeably thinner. I am going to have to take my watch off soon because it is so loose.

So you can imagine my frustration when I burnt my lips on the precious half cup of hot chocolate that we allow ourselves each day, and spilled it all over the boat. I'm sure when we have arrived in Antigua and I see my girlfriend, Marina, and James sees his wife, Bev, we will be able to laugh about it, but on the Atlantic it's hard to cope with the mood swings.

Even though we are moving again, our spirits wax and wane as we have so much time to think. We've lost our position at the front of the rowing pairs. We've a long way still to go at sea and already Christmas is upon us. Everyone at home is getting together, wrapping presents. I imagine the morning frost. There's a lot to regret and miss...

Tonight, when James isn't looking, I'm going to put up some Christmas decorations that I sneaked on board. I have tinsel, a pretend Christmas tree and some Santa hats for us to wear. I shall have to sneak off when James is sleeping to decorate the VHF aerial and any other bits that need cheering up.

+

+
+ +
+
+ +
+
+ +
+ + +
+
+ +
+
+ + +
+

In Health

+
+
+
+ +
+ Asco +   +
+ +

+ Miniature horse therapy +

+ +
+
+
+
+ +
+ An astonishing image of a pregnant pony uterus has been selected as the overall winner for the 2015 Wellcome Image Awards. The photograph was taken by Michael Frank, and is of an historic specimen from the Lanyon Anatomy Museum of the Royal Veterinary College in London. It shows the preserved uterus of a New Forest pony, approximately five months into the pregnancy +   +
+ +

+ Wellcome Image Awards +

+ +
+
+
+
+ +
+ A young women has had to have a metal spoon fished out of her stomach after accidentally swallowing it while eating ice cream.
+Zhang Weiwei, the 22-year-old varsity student from Wuhan University in Wuhan, central China’s Hubei Province, was on her way back from a meal with friends when the incident happened.
+Weiwei had bought an ice cream and was chatting and walking back to her dorm room when another friend saw her and jumped on her back to greet her.
+Weiwei got such a fright that she swallowed the entire 14cm metal spoon. +   +
+ +

+ Weird X-rays +

+ +
+
+
+
+ +
+ For the past two years Russian photojournalist Vladimir Yakovlev travelled around the world, searching for people who have discovered new found hobbies and pleasure in their older age. With the series The Age Of Happiness, Yakovlev hopes to change the usual perception of life after retirement and promote positive ageing. On his travels he met some extraordinary characters over 60-year-old - some very close to the 100 milestone -   who enjoy each day and inspire others to make their lives equally fulfilling. +   +
+ +

+ Life begins at 70 +

+ +
+
+ +
+
+ +
+ The Hepatitis C Trust +   +
+ +

+ Secret Postcards +

+ +
+
+
+
+ +
+ +   +
+ +

+ Celebs in specs +

+ +
+
+
+
+
+
+ +
+ +
+ + + + +
+
+
+ +

+ Top news galleries +

+
+ + + + + + + + + + + +
+
+
+ + + + + + + + + + +
+ +
+ + + +
+
+
+ +
+
+
+
+
+
+
+
+
+ +
Advertisement
+
+
+ +
+
+ +
+
+
+ +

+ Latest Video» +

+
+ + +
+ +
+ + + + + + + +
+
+ +
+
Scientist in lab + +
+ + Sponsored +

+ When media meets medicine +

+ +
+
+ +
+
+
+
+
+
+ + + +
+ +

+ More from the web +

+
+ + +
+ +
+
+
+
+ +
Advertisement
+
+
+ +
+ +
+
+ +
Advertisement
+
+
+
+ +
+
+ + + + + + + +
+
+ + + +
+ +

+ More from the web +

+
+ + +
+ +
+
+
+
+
+
+
+ + + +
+ +

+ More from the web +

+
+ + +
+ +
+
+
+ + +
+
+ +
+
+ +
+
+ +
+
+
+
+ + + +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+
+ +
+ +
+ + +
+ +
+
+ + + + + + +
+ + + + + + +
+ + + + + + +
+ + + + +
+
+ +
+

© Copyright of Telegraph Media Group Limited 2017

+

Terms and Conditions

+

Today's News

+

Archive

+

Style Book

+

Weather Forecast

+
+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + diff --git a/test/testdata/1a33b865b87875d535e04b2405c29d0026497b02.json b/test/testdata/1a33b865b87875d535e04b2405c29d0026497b02.json new file mode 100644 index 00000000..cc436f8e --- /dev/null +++ b/test/testdata/1a33b865b87875d535e04b2405c29d0026497b02.json @@ -0,0 +1,18 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "max-age=604800", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Language": "en-GB", + "Content-Length": "22511", + "Content-Type": "text/html;charset=UTF-8", + "Date": "Tue, 23 May 2017 17:53:16 GMT", + "ETag": "3334755-1495561996670", + "Server": "nginx", + "Vary": "Accept-Encoding", + "X-UA-Compatible": "IE=Edge" + }, + "status_code": 200, + "url": "http://www.telegraph.co.uk/news/health/3334755/We-could-see-the-whales-eyes-mouth...-the-barnacles-on-its-back.html" +} \ No newline at end of file diff --git a/test/testdata/1a56323aeb042a71e42768fc788a559460b88299.html b/test/testdata/1a56323aeb042a71e42768fc788a559460b88299.html new file mode 100644 index 00000000..5859f685 --- /dev/null +++ b/test/testdata/1a56323aeb042a71e42768fc788a559460b88299.html @@ -0,0 +1,1564 @@ + + + + + + US 'received Qatar assurances' on Afghan prisoner deal - BBC News + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+
+
+ + + + + + +
+ +
+ + + Asia + + + + +
+ +
+ + Asia + + + + +
+ +
+

US 'received Qatar assurances' on Afghan prisoner deal

+ + + +
+
+
+
Media playback is unsupported on your device
+
+
+
+
+
+
Media captionThe US president, who was joined at the White House by Sgt Bergdahl's parents, Robert and Jani, said ''he was never forgotten''
+

US President Barack Obama says he received security guarantees from Qatar over five Guantanamo Bay prisoners who were transferred to secure the release of a US soldier in Afghanistan.

US Army Sergeant Bowe Bergdahl, 28, was handed to US forces after being held for nearly five years by the Taliban.

He has left Afghanistan and is en route to a US military hospital in Germany.

Five Afghan detainees were released from the US prison in Cuba and handed to Qatar, which mediated the deal.

Sgt Bergdahl, who is said to be in good condition, was the only US soldier being held by the Taliban in Afghanistan.

His parents said they were "joyful and relieved" to hear of their son's release.


Analysis: David Loyn, BBC News, Kabul

+ + + +
+ + + Image copyright + AFP + +
+ +
+ Image caption + + A video grab image from 2010 showed Sgt Bergdahl in captivity + +
+ +

Negotiations for the US-Taliban prisoner swap began three years ago. US negotiators met Taliban leaders face to face in Qatar, but Taliban sources told me that the talks did not move forward because the US were pushing for a wider peace process, while the Taliban wanted to limit the talks to a prisoner swap.

The Afghan government blocked further progress a year ago, enraged when the Taliban opened a political office in Qatar. But using Qatar as a mediator, the US continued talks in secret. A US source said that the breakthrough came recently when hardline Taliban elements agreed to the swap. The US believes that Bergdahl was held across the frontier in Pakistan for most of his captivity.

It is unclear what impact the release will have on a wider peace process. The Afghan High Peace Council want talks with the Taliban to happen inside Afghanistan not outside, and do not want to involve the Americans.


+ + + +
+ + + Image copyright + Getty Images + +
+ +
+ Image caption + + Sgt Bergdahl's parents said they could not wait to wrap their arms around their only son + +
+ +

Hours after the release, President Obama told reporters the Qatari government had given the US assurances "that it will put in place measures to protect our national security".

He also thanked the Qatari authorities for their role in acting as a go-between during indirect US-Taliban negotiations that led to the deal.

The exchanged prisoners are thought to be the most senior Afghans still held at Guantanamo. Under the deal, they will be banned from leaving Qatar for at least a year.

The Taliban said they welcomed their release with "great happiness".

"While Sgt Bergdahl was gone he was never forgotten," President Obama said, adding that the US had an "ironclad commitment" to bringing home its prisoners of war.

He was joined by Sgt Bergdahl's parents, Robert and Jani, at the White House on Saturday. They offered thanks to those who took part in securing their son's freedom.

In an emotional speech, Robert Bergdahl said his son was having trouble speaking English after his rescue.

The BBC's Beth McLeod in Washington says the soldier is being taken to a US military medical centre in Germany, where he will receive medical treatment and begin the process of reconnecting with his family through telephone calls and video conferences.

+
+
Media playback is unsupported on your device
+
+
+
+
+
+
Media captionDavid Loyn reports on reaction from the Taliban's leadership to the release of Sgt Bowe Bergdahl
+

Who are the Guantanamo detainees?

+ + + +
+ + + Image copyright + AFP + +
+ +
+ Image caption + + The five released inmates had all been held at Guantanamo since 2002 + +
+ +

Mohammad Fazl served as the Taliban's deputy defence minister during America's military campaign in 2001. Accused of possible war crimes, including the murder of thousands of Shia Muslims.

Khirullah Khairkhwa was a senior Taliban official serving as interior minister and governor of Herat, Afghanistan's third largest city. Alleged to have had direct links to Osama bin Laden.

Abdul Haq Wasiq was the Taliban's deputy minister of intelligence. Said to have been central in forming alliances with other Islamic fundamentalist groups to fight against US and coalition forces.

Mullah Norullah Noori was a senior Taliban military commander and a governor. Also accused of being involved in the mass killings of Shia Muslims.

Mohammad Nabi Omari held multiple Taliban leadership roles, including chief of security. Alleged to have been involved in attacks against US and coalition forces.


Officials said the Taliban had handed Sgt Bergdahl over on Saturday evening, local time, in eastern Afghanistan, in an exchange that involved several dozen US special forces.

A senior official told the BBC that, once aboard the US helicopter, Sgt Bergdahl wrote "SF?" - asking if they were special operations forces - on a paper plate and showed it to the pilots, who replied: "Yes, we've been looking for you for a long time."

+ + + +
+ + + Image copyright + AP + +
+ +
+ Image caption + + New signs hang in the soldier's hometown of Hailey after his release was announced on Saturday + +
+ +

The senior official said: "At that point, Sgt Bergdahl broke down".

The soldier, of Hailey, Idaho, was captured on 30 June 2009, about two months after arriving in eastern Afghanistan.

In January, the US military obtained a new video of Sgt Bergdahl, giving his family renewed hope of his eventual return.

Throughout his captivity, the soldier's hometown had continued to remember him with special events and yellow ribbons tied to poles and trees.

"I'm beyond thrilled," Stefanie O'Neill, a family friend in Hailey, told Reuters on Saturday. "It's probably the happiest day of my life, besides when my two kids were born."

She said a Bring Bowe Back vigil planned for 28 June would now be called Bowe is Back.

+
+
+
+

Related Topics

+ +
+ + + +
+

More on this story

+ + + + + + + +
+ +
+ + + + + +
+ + + + +
+ +
+ + + +
+ + + + + + + + + + + + + + + + + diff --git a/test/testdata/1a56323aeb042a71e42768fc788a559460b88299.json b/test/testdata/1a56323aeb042a71e42768fc788a559460b88299.json new file mode 100644 index 00000000..47eada73 --- /dev/null +++ b/test/testdata/1a56323aeb042a71e42768fc788a559460b88299.json @@ -0,0 +1,31 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "0", + "Cache-Control": "private, max-age=60, stale-while-revalidate", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Language": "en", + "Content-Length": "37838", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:52:53 GMT", + "Server": "Apache", + "Set-Cookie": "BBC-UID=9dec646b9c063a14d996724eb3495e99dfa884913d6be73fda1f6e913a6ef6670Mozilla%2F5.0%20%28Windows%20NT%2010.0%3B%20Win64%3B%20x64%3B%20rv%3A50.0%29%20Gecko%2F20100101%20Firefox%2F50.0; expires=Sat, 22 May 2021 17:52:53 GMT; path=/; domain=.bbc.com", + "Vary": "Accept-Encoding", + "Via": "1.1 varnish", + "X-Cache": "MISS", + "X-Cache-Action": "MISS", + "X-Cache-Age": "0", + "X-Cache-Hits": "0", + "X-Fastly-Cache-Status": "MISS-CLUSTER", + "X-LB-NoCache": "true", + "X-News-Cache-Id": "70217", + "X-News-Data-Centre": "telhc", + "X-PAL-Host": "pal199.back.live.telhc.local:80", + "X-Served-By": "cache-iad2648-IAD", + "X-Timer": "S1495561972.400831,VS0,VE1133" + }, + "status_code": 200, + "url": "http://www.bbc.com/news/world-asia-27653361" +} \ No newline at end of file diff --git a/test/testdata/1cbc2ff0526a0c00d2b3ea958743391cc86fec6f.html b/test/testdata/1cbc2ff0526a0c00d2b3ea958743391cc86fec6f.html new file mode 100644 index 00000000..2f9a2ba3 --- /dev/null +++ b/test/testdata/1cbc2ff0526a0c00d2b3ea958743391cc86fec6f.html @@ -0,0 +1,35 @@ +
+ \ No newline at end of file diff --git a/test/testdata/1cbc2ff0526a0c00d2b3ea958743391cc86fec6f.json b/test/testdata/1cbc2ff0526a0c00d2b3ea958743391cc86fec6f.json new file mode 100644 index 00000000..144157f9 --- /dev/null +++ b/test/testdata/1cbc2ff0526a0c00d2b3ea958743391cc86fec6f.json @@ -0,0 +1,21 @@ +{ + "encoding": "utf-8", + "headers": { + "CF-Cache-Status": "DYNAMIC", + "CF-RAY": "7445347e3c2a9b37-FRA", + "Cache-Control": "private, no-cache, no-store, max-age=0, must-revalidate", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Security-Policy": "frame-ancestors 'none'", + "Content-Type": "text/html; charset=utf-8", + "Date": "Fri, 02 Sep 2022 09:26:36 GMT", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=15724800; includeSubDomains", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding", + "X-Frame-Options": "SAMEORIGIN", + "X-Powered-By": "Next.js" + }, + "status_code": 200, + "url": "https://www.worldcat.org/title/809771201" +} \ No newline at end of file diff --git a/test/testdata/1e1a00c41d410522d44c599a5c079fb1ad00a463.html b/test/testdata/1e1a00c41d410522d44c599a5c079fb1ad00a463.html new file mode 100644 index 00000000..b11706fc --- /dev/null +++ b/test/testdata/1e1a00c41d410522d44c599a5c079fb1ad00a463.html @@ -0,0 +1 @@ +{"indexed":{"date-parts":[[2022,4,3]],"date-time":"2022-04-03T18:05:00Z","timestamp":1649009100157},"reference-count":43,"publisher":"Medknow","issue":"3","content-domain":{"domain":[],"crossmark-restriction":false},"published-print":{"date-parts":[[2020]]},"DOI":"10.4103\/npmj.npmj_69_20","type":"journal-article","created":{"date-parts":[[2020,7,16]],"date-time":"2020-07-16T14:24:24Z","timestamp":1594909464000},"page":"242","source":"Crossref","is-referenced-by-count":1,"title":"Management of a giant prostatic enlargement: Case report and review of the literature","prefix":"10.4103","volume":"27","author":[{"given":"RufusWale","family":"Ojewola","sequence":"first","affiliation":[]},{"given":"KehindeHabeeb","family":"Tijani","sequence":"additional","affiliation":[]},{"given":"AdedejiLukman","family":"Fatuga","sequence":"additional","affiliation":[]},{"given":"ChigozieInnocent","family":"Onyeze","sequence":"additional","affiliation":[]},{"given":"ChikeJohn","family":"Okeke","sequence":"additional","affiliation":[]}],"member":"2581","reference":[{"key":"key-10.4103\/1117-1936.289919-1","unstructured":"Joseph CP. Neoplasms of the prostate gland. In: Tanagho EA, McAninch JW, editors. Smith 's General Urology. 15th ed.. New York: Lange Medica Books\/McGraw-Hill; 2000. p. 348-74."},{"key":"key-10.4103\/1117-1936.289919-2","first-page":"208","volume-title":"Management of the complications of BPH\/BOO","author":"Speakman","year":"2014","journal-title":"Indian J Urol","ISSN":"http:\/\/id.crossref.org\/issn\/0970-1591","issn-type":"print"},{"key":"key-10.4103\/1117-1936.289919-3","first-page":"336","volume-title":"A case of giant prostatic hyperplasia","author":"Fishman","year":"1993","journal-title":"Urology"},{"key":"key-10.4103\/1117-1936.289919-4","first-page":"474","volume-title":"The development of human benign prostatic hyperplasia with age","author":"Berry","year":"1984","journal-title":"J Urol"},{"key":"key-10.4103\/1117-1936.289919-5","first-page":"1793","volume-title":"Update on AUA guideline on the management of benign prostatic hyperplasia","author":"McVary","year":"2011","journal-title":"J Urol"},{"key":"key-10.4103\/1117-1936.289919-6","first-page":"53","volume-title":"A case of giant prostatic hyperplasia","author":"Wang","year":"2016","journal-title":"Asian J Urol"},{"key":"key-10.4103\/1117-1936.289919-7","first-page":"795","volume-title":"Giant hypertrophy of the prostate: 2,410 grams of weight and 24 cm in diameter","author":"Medina","year":"1997","journal-title":"Arch Esp Urol"},{"key":"key-10.4103\/1117-1936.289919-8","first-page":"1009","volume-title":"A giant prostatic hyperplasia treated by open surgery","author":"Ogawa","year":"2012","journal-title":"Int J Gen Med"},{"key":"key-10.4103\/1117-1936.289919-9","first-page":"33","volume-title":"Giant benign prostatic hyperplasia in a Pakistani patient","author":"Khan","year":"2014","journal-title":"Urol Case Rep"},{"key":"key-10.4103\/1117-1936.289919-10","first-page":"e3","volume-title":"Giant prostatic hyperplasia: Case report of 3987 mL","author":"Dom\u00ednguez","year":"2016","journal-title":"Urology"},{"key":"key-10.4103\/1117-1936.289919-11","first-page":"101051","volume-title":"Giant benign prostatic hyperplasia: A case report","author":"Aghamir","year":"2020","journal-title":"Urol Case Rep"},{"key":"key-10.4103\/1117-1936.289919-12","first-page":"777","volume-title":"Massive benign prostatic hyperplasia","author":"Tolley","year":"1987","journal-title":"J R Soc Med"},{"key":"key-10.4103\/1117-1936.289919-13","first-page":"81","volume-title":"Giant prostate; the largest recorded","author":"Ockerblad","year":"1946","journal-title":"J Urol"},{"key":"key-10.4103\/1117-1936.289919-14","first-page":"14","volume-title":"Case Report: Giant Benign Prostatic Hyperplasia in a Ghanaian.J Med Biomed Sci","author":"Appiah","year":"2014","journal-title":""},{"key":"key-10.4103\/1117-1936.289919-15","first-page":"e253","volume-title":"Giant Prostatic Hyperplasia: Fourth largest prostate reported in medical literature","author":"Maliakal","year":"2014","journal-title":"Sultan Qaboos Univ Med J"},{"key":"key-10.4103\/1117-1936.289919-16","first-page":"489","volume-title":"Giant prostatic hyperplasia: Case report and literature review","author":"\u00dc\u00e7er","year":"2011","journal-title":"Dicle Med J"},{"key":"key-10.4103\/1117-1936.289919-17","first-page":"454","volume-title":"Largest recorded prostate","author":"Nelson","year":"1940","journal-title":"Urol Cutan Rev"},{"key":"key-10.4103\/1117-1936.289919-18","first-page":"420","volume-title":"Giant prostatic hyperplasia: Report of a previously asymptomatic man presenting with gross hematuria and hypovolemic shock","author":"Wroclawski","year":"2015","journal-title":"Einstein (Sao Paulo)"},{"key":"key-10.4103\/1117-1936.289919-19","first-page":"309","volume-title":"One-stage suprapubic prostatectomy for a gland weighing 713 grams (one and one-half pounds)","author":"Gilbert","year":"1939","journal-title":"Urol Cutan Rev"},{"key":"key-10.4103\/1117-1936.289919-20","first-page":"8000","volume-title":"Retropubic prostatectomy for giant benign prostatic hyperplasia","author":"Lacy","year":"2015","journal-title":"Can J Urol"},{"key":"key-10.4103\/1117-1936.289919-21","first-page":"509","volume-title":"The largest surgically removed hypertrophied prostate","author":"Wadstein","year":"1938","journal-title":"JAMA"},{"key":"key-10.4103\/1117-1936.289919-22","first-page":"77","volume-title":"A prostate of world record size","author":"Lantzius-Beninga","year":"1966","journal-title":"Z Urol Nephrol"},{"key":"key-10.4103\/1117-1936.289919-23","first-page":"583","volume-title":"Enucleation of a giant prostatic hyperplasia in Ghana: A case report and mini literature review","author":"Egote","year":"2018","journal-title":"Case Rep Clin Med"},{"key":"key-10.4103\/1117-1936.289919-24","first-page":"769","volume-title":"Giant prostatic hypertrophy","author":"Ashamalla","year":"1972","journal-title":"Arch Surg"},{"key":"key-10.4103\/1117-1936.289919-25","first-page":"525","volume-title":"Haemorrhage and post operative obstruction in suprapubic prostatectomy: And an open operation for their prevention","author":"Thomson-Walker","year":"1920","journal-title":"Br J Surg"},{"key":"key-10.4103\/1117-1936.289919-26","first-page":"587","volume-title":"Giant prostatic hyperplasia: Case report","author":"Yilmaz","year":"2006","journal-title":"Int Urol Nephrol"},{"key":"key-10.4103\/1117-1936.289919-27","first-page":"571","volume-title":"Retropubic prostatectomy; early technical difficulties; report of removal of giant prostate","author":"Bacon","year":"1949","journal-title":"J Urol"},{"key":"key-10.4103\/1117-1936.289919-28","first-page":"1583","volume-title":"Prostate artery embolization for giant prostatic hyperplasia","author":"Bhatia","year":"2015","journal-title":"J Vasc Interv Radiol"},{"key":"key-10.4103\/1117-1936.289919-29","first-page":"1967","volume-title":"How large is the hyperplastic prostate.Report of the largest hypertro-phied prostate ever surgically removed?","author":"Middleton","year":"1937","journal-title":"JAMA"},{"key":"key-10.4103\/1117-1936.289919-30","first-page":"467","volume-title":"The largest BPH in Japan: Case report and review of the literature","author":"Kitagawa","year":"1980","journal-title":"Rinsho Hinyokika"},{"key":"key-10.4103\/1117-1936.289919-31","first-page":"717","volume-title":"Minimally invasive simple prostatectomy for a case of giant benign prostatic hyperplasia","author":"Zeng","year":"2017","journal-title":"Asian J Androl","ISSN":"http:\/\/id.crossref.org\/issn\/1008-682X","issn-type":"print"},{"key":"key-10.4103\/1117-1936.289919-32","first-page":"232","volume-title":"Giant prostatic hyperplasia: Surgical management of a case","author":"Sood","year":"2006","journal-title":"J Postgrad Med","ISSN":"http:\/\/id.crossref.org\/issn\/0022-3859","issn-type":"print"},{"key":"key-10.4103\/1117-1936.289919-33","first-page":"1","volume-title":"Giant benign prostatic hyperplasia in a Nigerian: Report of a case","author":"Akpo","year":"2010","journal-title":"Internet J Urol"},{"key":"key-10.4103\/1117-1936.289919-34","first-page":"276","volume-title":"Huge benign prostatic hyperplasia","author":"Hosseini","year":"2004","journal-title":"Urol J"},{"key":"key-10.4103\/1117-1936.289919-35","first-page":"200","volume-title":"Lower urinary tract symptoms: Prevalence, perceptions, and healthcare-seeking behavior amongst Nigerian men","author":"Ojewola","year":"2016","journal-title":"World J Mens Health"},{"key":"key-10.4103\/1117-1936.289919-36","first-page":"1965","volume-title":"Prevalence of urinary symptoms and other urological conditions in Spanish men 50 years old or older","author":"Hunter","year":"1996","journal-title":"J Urol"},{"key":"key-10.4103\/1117-1936.289919-37","first-page":"182","volume-title":"Effects of finasteride on vascular endothelial growth factor","author":"H\u00e4ggstr\u00f6m","year":"2002","journal-title":"Scand J Urol Nephrol"},{"key":"key-10.4103\/1117-1936.289919-38","first-page":"1359","volume-title":"Benign prostatic hyperplasia","author":"Thorpe","year":"2003","journal-title":"Lancet"},{"key":"key-10.4103\/1117-1936.289919-39","unstructured":"European Association of Urology. Guidelines on Treatment of Male Lower Urinary Tract Symptoms Including Benign Prostatic Hyperplasia; March, 2019. Available from: https:\/\/uroweb.org\/guideline\/treatm ent-of-non-neurogenic-male-luts. [Last accessed on 2020 May 02]."},{"key":"key-10.4103\/1117-1936.289919-40","first-page":"86","volume-title":"Perioperative outcomes of robotic and laparoscopic simple prostatectomy: A European-American multi-institutional analysis","author":"Autorino","year":"2015","journal-title":"Eur Urol"},{"key":"key-10.4103\/1117-1936.289919-41","first-page":"7","volume-title":"Robotic-assisted laparoscopic simple prostatectomy: An alternative minimal invasive approach for prostate adenoma","author":"Uffort","year":"2010","journal-title":"J Robot Surg"},{"key":"key-10.4103\/1117-1936.289919-42","first-page":"139","volume-title":"Management and outcomes of Gleason six prostate cancer detected on needle biopsy: A single-surgeon experience over 6 years","author":"March","year":"2017","journal-title":"Prostate Int"},{"key":"key-10.4103\/1117-1936.289919-43","first-page":"2106","volume-title":"Guideline for the management of clinically localized prostate cancer: 2007 update","author":"Thompson","year":"2007","journal-title":"J Urol"}],"container-title":"Nigerian Postgraduate Medical Journal","original-title":[],"language":"en","deposited":{"date-parts":[[2020,7,16]],"date-time":"2020-07-16T14:28:29Z","timestamp":1594909709000},"score":1,"resource":{"primary":{"URL":"http:\/\/www.npmj.org\/text.asp?2020\/27\/3\/242\/289919"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020]]},"references-count":43,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2020]]}},"alternative-id":["289919"],"URL":"http:\/\/dx.doi.org\/10.4103\/npmj.npmj_69_20","relation":{},"ISSN":["1117-1936"],"subject":["General Medicine"],"container-title-short":"Niger Postgrad Med J","published":{"date-parts":[[2020]]}} \ No newline at end of file diff --git a/test/testdata/1e1a00c41d410522d44c599a5c079fb1ad00a463.json b/test/testdata/1e1a00c41d410522d44c599a5c079fb1ad00a463.json new file mode 100644 index 00000000..691429bb --- /dev/null +++ b/test/testdata/1e1a00c41d410522d44c599a5c079fb1ad00a463.json @@ -0,0 +1,24 @@ +{ + "encoding": null, + "headers": { + "access-control-allow-headers": "X-Requested-With, Accept, Accept-Encoding, Accept-Charset, Accept-Language, Accept-Ranges, Cache-Control", + "access-control-allow-origin": "*", + "access-control-expose-headers": "Link", + "connection": "close", + "content-encoding": "gzip", + "content-length": "3245", + "content-type": "application/vnd.citationstyles.csl+json", + "date": "Sat, 27 Aug 2022 12:19:16 GMT", + "link": "; rel=\"canonical\"", + "permissions-policy": "interest-cohort=()", + "server": "Jetty(9.4.40.v20210413)", + "vary": "Accept, Accept-Encoding", + "x-api-pool": "public", + "x-rate-limit-interval": "1s", + "x-rate-limit-limit": "50", + "x-ratelimit-interval": "1s", + "x-ratelimit-limit": "50" + }, + "status_code": 200, + "url": "https://api.crossref.org/v1/works/10.4103%2Fnpmj.npmj_69_20/transform" +} \ No newline at end of file diff --git a/test/testdata/2073465ca8895fa9b55f883607b6c3fc4bcb3c0f.html b/test/testdata/2073465ca8895fa9b55f883607b6c3fc4bcb3c0f.html new file mode 100644 index 00000000..555b6d14 --- /dev/null +++ b/test/testdata/2073465ca8895fa9b55f883607b6c3fc4bcb3c0f.html @@ -0,0 +1,99 @@ +USA TODAY: Latest World and US News - USATODAY.com
Ukraine: Maps & Graphics
Featured Videos
More Top Stories
Discover
A mentally ill man died in prison. His widow wants justice
\ No newline at end of file diff --git a/test/testdata/2073465ca8895fa9b55f883607b6c3fc4bcb3c0f.json b/test/testdata/2073465ca8895fa9b55f883607b6c3fc4bcb3c0f.json new file mode 100644 index 00000000..d6c52b0e --- /dev/null +++ b/test/testdata/2073465ca8895fa9b55f883607b6c3fc4bcb3c0f.json @@ -0,0 +1,36 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "64", + "Cache-Control": "no-store", + "Connection": "keep-alive", + "Content-Encoding": "br", + "Content-Length": "51534", + "Content-Security-Policy": "upgrade-insecure-requests;frame-ancestors 'none';object-src 'none'", + "Content-Security-Policy-Report-Only": "script-src https: blob: 'unsafe-inline' 'unsafe-eval' 'self';base-uri 'self';report-uri https://reporting-api.gannettinnovation.com;report-to default", + "Content-Type": "text/html; charset=utf-8", + "Cross-Origin-Opener-Policy": "same-origin", + "Cross-Origin-Resource-Policy": "same-origin", + "Date": "Tue, 15 Mar 2022 14:57:21 GMT", + "Feature-Policy": "camera 'none';display-capture 'none';geolocation 'none';microphone 'none';payment 'none';usb 'none';xr-spatial-tracking 'none'", + "Gannett-Cam-Experience-Id": "control:15", + "NEL": "{\"report_to\":\"default\",\"max_age\":31557600,\"include_subdomains\":true,\"success_fraction\":0.005}", + "Origin-Agent-Cluster": "?1", + "Permissions-Policy": "camera=(),display-capture=(),geolocation=(),microphone=(),payment=(),usb=(),xr-spatial-tracking=()", + "Referrer-Policy": "strict-origin-when-cross-origin", + "Report-to": "{\"max_age\":31557600,\"include_subdomains\":true,\"endpoints\":[{\"url\":\"https://reporting-api.gannettinnovation.com\"}]}", + "Set-Cookie": "gup_anonid=9daf215e-19c0-468c-b65e-3b59715d20e7; Domain=.usatoday.com; Max-Age=31536000; Path=/; SameSite=Lax; Secure, gup_clientid=0a4a17d9-59a9-4b1b-a2f4-59b9c425ff6c; Domain=.usatoday.com; Max-Age=31536000; Path=/; SameSite=Lax; Secure, gnt_ub=75; domain=.usatoday.com; path=/; secure; samesite=lax; max-age=31536000;, gnt_sb=15; domain=.usatoday.com; path=/; secure; samesite=lax; max-age=31536000;, gnt_eid=control:15; domain=.usatoday.com; path=/; secure; samesite=lax; max-age=5184000;, gnt_d=%7B%22w%22%3A%7B%22t%22%3A%2258%22%2C%22f%22%3A%221-q1a2z32cb0f2f2%22%2C%22c%22%3A%22Sunny%22%7D%2C%22z%22%3A%2222102%22%2C%22c%22%3A%22McLean%22%2C%22s%22%3A%22VA%22%7D; domain=.usatoday.com; path=/; samesite=lax; secure; priority=high;", + "Strict-Transport-Security": "max-age=63072000", + "Vary": "X-AbVariant,X-AbVCfg,X-AltUrl,Accept-Encoding,User-Agent", + "X-Cache": "HIT, HIT", + "X-Content-Type-Options": "nosniff", + "X-Frame-Options": "deny", + "X-Timer": "S1647356241.176968,VS0,VE7", + "X-XSS-Protection": "1; mode=block", + "etag": "W/\"30685-2ymAR6UVCsR5xVWb34VCsMPefsw\"", + "link": ";rel=preload;as=image;nopush" + }, + "status_code": 200, + "url": "https://www.usatoday.com/" +} \ No newline at end of file diff --git a/test/testdata/2455ea251162486387e2e10d3a43a3d4eb421dd2.html b/test/testdata/2455ea251162486387e2e10d3a43a3d4eb421dd2.html new file mode 100644 index 00000000..e61ba268 --- /dev/null +++ b/test/testdata/2455ea251162486387e2e10d3a43a3d4eb421dd2.html @@ -0,0 +1,81 @@ + + + + ABC (Australian Broadcasting Corporation) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Skip to main content

ABC Home

Breaking News Ticker

Promotion

Freeman

One race, 20 million memories

Stream nowon ABC TV and iview
+ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/testdata/2455ea251162486387e2e10d3a43a3d4eb421dd2.json b/test/testdata/2455ea251162486387e2e10d3a43a3d4eb421dd2.json new file mode 100644 index 00000000..6705e62b --- /dev/null +++ b/test/testdata/2455ea251162486387e2e10d3a43a3d4eb421dd2.json @@ -0,0 +1,30 @@ +{ + "encoding": "utf-8", + "headers": { + "Application": "core", + "Branch": "master-core", + "Build": "41", + "Cache-Control": "public, max-age=3", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "36567", + "Content-Security-Policy": "upgrade-insecure-requests;", + "Content-Type": "text/html; charset=utf-8", + "Date": "Fri, 25 Sep 2020 15:04:42 GMT", + "ETag": "W/\"23fab-KPZOOKoI6/wB8yvbzwhOgyPjNqk-gzip\"", + "Environment": "production", + "Expires": "Fri, 25 Sep 2020 15:04:45 GMT", + "Product": "presentation-layer", + "Referrer-Policy": "no-referrer-when-downgrade", + "Server": "Apache/2.4.46 (Unix)", + "Set-Cookie": "ABCGuestID=82.178.158.102.145361601046282684; expires=Mon, 31-Dec-2038 23:59:59 GMT; path=/; domain=.abc.net.au, ABC_LD=int; path=/; domain=.abc.net.au, ABC_FF=desktop; expires=Fri, 25-Sep-2020 17:04:42 GMT; path=/", + "Vary": "Accept-Encoding, Origin, User-Agent", + "X-Content-Type-Options": "nosniff", + "X-DNS-Prefetch-Control": "off", + "X-Download-Options": "noopen", + "X-Frame-Options": "SAMEORIGIN", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://www.abc.net.au/" +} \ No newline at end of file diff --git a/test/testdata/2674dc9c21a2cb7a5b5800dc7c432ad41f6d3429.html b/test/testdata/2674dc9c21a2cb7a5b5800dc7c432ad41f6d3429.html new file mode 100644 index 00000000..5b9e9129 --- /dev/null +++ b/test/testdata/2674dc9c21a2cb7a5b5800dc7c432ad41f6d3429.html @@ -0,0 +1,5179 @@ + + + + + + + The New York Times - Breaking News, World News & Multimedia + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + +
+
+ + + + +
+
+ + + + + + + + + +
+
+ +
+ +
+ +
+

Top News

+ +
+ +
+
+
Developing
+
+
+ +
+ + + +
+
+ +
+
+ +
+
+ +

Assailant Died in Blast That Killed 22 in Manchester

+ + + +

  • The suspect was identified as Salman Abedi, a Briton of Libyan descent who lived near the arena. The police said his ID was found at the scene.
  • +
  • The British government did not make any immediate comment on the Islamic State’s claim of responsibility.

+ +

+  Comments +

+ +
+ + +
+ + + +
+
+ +
+
+ +
+ +
+
+ +
See the aftermath of the bombing, in photographs. + +
+ +
+
+ +
+
+ + +
+
+ +

On the Scene in Manchester

+ + + +

The Times has reporters in Manchester, where witnesses described the carnage, and the police said they had carried out raids.

+ + +
+
+
+ +
+
+
+
+
+
+ +

+ +
+
+ +
+
+ + +

+

+ + +
+
+ +
+
+
+ + + +
+ + + + + + + + + +
+ +
+
+
+
+ + +
+ +
+ +
+ +
+ +
+
+ +

Russia Contacts With Trump Team Worried Ex-C.I.A. Chief

+ + + +

  • John O. Brennan told senators on Tuesday that he became concerned last year that the Russian government was trying to influence members of the Trump campaign.
  • +
  • It is the first time the former C.I.A. director has publicly acknowledged that he was concerned about possible ties between Russia and Trump associates.

+ + +
+ + +
+
+
+ +

+ + +

+

+ + + +
+
+ +
+
+
+
+
+

Related Article

+ +
+
+
+ +

Budget Slashes Aid to Poor and Offers Huge Tax Cuts

+ + + +

President Trump’s budget proposal calls for spending more than $2.6 billion for border security — including $1.6 billion to begin work on a border wall — and slashing more than $800 billion from Medicaid.

+ +

+  Comments +

+ +
+
+ +
+ +
+ +
+
+ + + + +
+
+
+

Got a confidential news tip?

+

The New York Times offers several ways to get in touch with and provide materials to our journalists. Learn more.

+
+
+
+ + +
+ +
+ +
+ +
+ + + +
+ +
+ + + + +
+
+ + The Daily Logo + +
+

Audio

+

+ + Listen to ‘The Daily’ + +

+

The latest from President Trump’s trip abroad; the continuing saga of Michael Flynn; and developments after the bombing in Manchester, England.

+
+ Audio +
+
+
+
+ +
+
+ + +
+ + + +
+
+ + + +
+
+
+
+ +

Circa Now

+

The Tricky Etiquette of Co-Working Spaces

+ +
+ +
+ + + +

+ To get a glimpse of what manners will be like in the office of the future, it behooves us to look at the co-working spaces of today.

+ + +
+
+ +

Tech Tip

+

How to Expand Wi-Fi in Your Home

+ +
+ +
+ + + +

+ If parts of your house are not getting a decent Wi-Fi signal from your router, hardware, software and maybe an empty beer can might help.

+ + +
+
+ + + + + + +
+
+
+ + +
+
+ +
+
+ A Los Angeles interchange. California can write its own auto emissions standards because of a waiver granted under the Clean Air Act. + + + Credit + Melissa Lyttle for The New York Times +
+
+ +

Fighting Trump on Climate, California Becomes a Global Force

+ +

The state has been at the leading edge of the resistance to President Trump. But of all the battles, none has the global implications of climate change.

+ + + + + +
+
+
+
+ +

Sir Roger Moore, Who Played a Wry James Bond, Dies

+ +
+
+ +
+
+ + + +

+ The British actor brought tongue-in-cheek humor to the James Bond persona in seven films. His family announced his death in a statement on Twitter. He was 89.

+ + +
+
+
+
+ +

Jared Kushner’s Other Real Estate Empire

+ +
+
+ +
+
+ + + +

+ Baltimore-area renters complain about a property owner they say is neglectful and litigious. Few know their landlord is the president’s son-in-law.

+ +

+  Comments +

+ +
+
+
+
+ +

Firebrand Sheriff, Voted Out of Office, Has No Regrets

+ +
+ +
+ + + +

+ Joe Arpaio, the former Arizona sheriff known for being tough on inmates and accused of targeting Latinos, reflects on his reputation and his decades in law enforcement.

+ + +
+
+
+ + + +
+ + +
+
+
+ + + + + +
+
+
+ +
+
+

Morning Briefing: Australia Edition

+

The news and stories that matter to readers in Australia. Sign up to get it by email, Monday through Friday.

+ +
+
+
+ + + +
+ + + + +
+
+
+ +
+
+

Morning Briefing: Asia Edition

+

The news and stories that matter to readers in Asia. Sign up to get it by email, Monday through Friday.

+ +
+
+
+ + + + +
+ + + + + +
+
+
+ +
+
+

Morning Briefing: Europe Edition

+

The news and stories that matter to readers in Europe. Sign up to get it by email, Monday through Friday.

+
+
+
+ + + +
+ + + + +
+
+
+ +
+
+

Morning Briefing

+

The news and stories that matter. Delivered to your inbox Monday through Friday.

+ + +
+
+
+ + +
+ +
+ +
+ + +
+ +
+ +
+
+ + + + + +
+
+
+ +
+

+

+
+ +
+
+ +
+
+ +
+
+
+
+ +
+
+
+

+

+
+ +
+
+ +
+
+ +
+
+
+ +
+ +
+ +
+ + + +
+ + +
+ +
+ +
+ +
+ +
+ +
+
+
+
+ +

Manchester, United in Grief and Kindness

+ +
+ +
+ + + +

+ This was an attack on the city’s very soul. But terror’s spite only redoubles people’s decency.

+ + +
+ +
+ +
+
+ +
+
+
+
+ +

Beware of Sheriff David Clarke

+ +
+ +
+ + +

+ We in Milwaukee are relieved to be rid of him, but worried about the damage he could do in the Trump administration.

+ + +
+ + + +
+ +
+
+ +
+ +
+
+
+
+ +
+ +
+ +

User Subscriptions

+ + + + + +
+ +
+ +
+ + + +
+ +
+ + + + +
+
+
+

Watching

+
+
+
+
+
+ + + + +
+ +
+ +
+ + +
+ +
+ + +
+ +
+ +
+ + + + +
+
+
+
Loading...
+
+ +
+
+ +
+ +
+ + + +
+ +
+ +
+ +
+
+

Sections

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ +
+ +
+ + +
+
+
+
+ +
+

+ Real Estate » +

+ + +
+ + +
+
+
+
+
+
+
+
+ +
+
+
+
+ + +
+ + +
+
+
+
+
+
+ + + +
+
Loading...
+
+
+ + + + +
+ +
+ + + + +
+
+
+
+

Go to Home Page »

+

+ Site Index + + The New York Times + +

+ +
+ + + +
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/test/testdata/2674dc9c21a2cb7a5b5800dc7c432ad41f6d3429.json b/test/testdata/2674dc9c21a2cb7a5b5800dc7c432ad41f6d3429.json new file mode 100644 index 00000000..ed0e2981 --- /dev/null +++ b/test/testdata/2674dc9c21a2cb7a5b5800dc7c432ad41f6d3429.json @@ -0,0 +1,30 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "156", + "Cache-Control": "no-cache", + "Connection": "close", + "Content-Encoding": "gzip", + "Content-Length": "51336", + "Content-Security-Policy": "default-src data: 'unsafe-inline' 'unsafe-eval' https:; script-src data: 'unsafe-inline' 'unsafe-eval' https: blob:; style-src data: 'unsafe-inline' https:; img-src data: https: blob:; font-src data: https:; connect-src https: wss:; media-src https: blob:; object-src https:; child-src https: data: blob:; form-action https:; block-all-mixed-content;", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:53:30 GMT", + "Server": "Apache", + "Set-Cookie": "nyt-a=7fa51849d1553600c660c351c80a912910cb1686763bb2e8d61ebb4b6adeb687; Expires=Wed, 23 May 2018 17:53:30 GMT; Path=/; Domain=.nytimes.com", + "Vary": "Host, Accept-Encoding, Fastly-SSL", + "X-API-Version": "F-5-5", + "X-Age": "3", + "X-Cache": "HIT", + "X-Cache-Hits": "43", + "X-ESI": "1", + "X-Frame-Options": "DENY", + "X-Origin-Time": "2017-05-23 13:50:54 EDT", + "X-PageType": "homepage", + "X-Served-By": "cache-iad2139-IAD", + "X-Timer": "S1495562011.960848,VS0,VE0", + "ntCoent-Length": "235547" + }, + "status_code": 200, + "url": "https://www.nytimes.com/" +} \ No newline at end of file diff --git a/test/testdata/2a65476f9d13cb10f1deb219ec20e9bdde891f2f.html b/test/testdata/2a65476f9d13cb10f1deb219ec20e9bdde891f2f.html new file mode 100644 index 00000000..43d3f4f6 --- /dev/null +++ b/test/testdata/2a65476f9d13cb10f1deb219ec20e9bdde891f2f.html @@ -0,0 +1 @@ +{"indexed":{"date-parts":[[2022,5,20]],"date-time":"2022-05-20T12:31:19Z","timestamp":1653049879444},"reference-count":4,"publisher":"Springer Science and Business Media LLC","issue":"7","license":[{"start":{"date-parts":[[2002,7,1]],"date-time":"2002-07-01T00:00:00Z","timestamp":1025481600000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"published-print":{"date-parts":[[2002,7]]},"DOI":"10.1038\/nrd842","type":"journal-article","created":{"date-parts":[[2002,7,28]],"date-time":"2002-07-28T21:36:31Z","timestamp":1027892191000},"page":"491-492","source":"Crossref","is-referenced-by-count":59,"title":"Selective anticancer drugs","prefix":"10.1038","volume":"1","author":[{"given":"Joshua H.","family":"Atkins","sequence":"first","affiliation":[]},{"given":"Leland J.","family":"Gershell","sequence":"additional","affiliation":[]}],"member":"297","reference":[{"key":"BFnrd842_CR1","doi-asserted-by":"publisher","first-page":"493","DOI":"10.1038\/nrd839","volume":"1","author":"R Capdeville","year":"2002","unstructured":"Capdeville, R. et al. Glivec (STI571, imatinib), a rationally developed targeted anticancer drug. Nature Rev. Drug Discov. 1, 493\u2013502 (2002).","journal-title":"Nature Rev. Drug Discov."},{"key":"BFnrd842_CR2","first-page":"2958","volume":"7","author":"F Ciardello","year":"2001","unstructured":"Ciardello, F. & Tortora, G. A novel approach in the treatment of cancer: targeting the epidermal growth factor receptor. Clin. Cancer Res. 7, 2958\u20132970 (2001).","journal-title":"Clin. Cancer Res."},{"key":"BFnrd842_CR3","doi-asserted-by":"publisher","first-page":"117","DOI":"10.1016\/S1535-6108(02)00039-9","volume":"1","author":"LK Shawver","year":"2002","unstructured":"Shawver, L. K., Slamon, D. & Ullrich, A. Smart drugs: tyrosine kinase inhibitors in cancer therapy. Cancer Cell 1, 117\u2013123 (2002).","journal-title":"Cancer Cell"},{"key":"BFnrd842_CR4","doi-asserted-by":"publisher","first-page":"23","DOI":"10.3322\/canjclin.52.1.23","volume":"52","author":"A Jem","year":"2002","unstructured":"Jem, A. et al. Cancer statistics, 2002. CA Cancer J. Clin. 52, 23\u201347 (2002)","journal-title":"CA Cancer J. Clin."}],"container-title":"Nature Reviews Drug Discovery","original-title":[],"language":"en","link":[{"URL":"http:\/\/www.nature.com\/articles\/nrd842.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/www.nature.com\/articles\/nrd842","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/www.nature.com\/articles\/nrd842.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,12,2]],"date-time":"2021-12-02T09:28:59Z","timestamp":1638437339000},"score":1,"resource":{"primary":{"URL":"http:\/\/www.nature.com\/articles\/nrd842"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2002,7]]},"references-count":4,"journal-issue":{"issue":"7","published-print":{"date-parts":[[2002,7]]}},"alternative-id":["BFnrd842"],"URL":"http:\/\/dx.doi.org\/10.1038\/nrd842","relation":{},"ISSN":["1474-1776","1474-1784"],"subject":["Drug Discovery","Pharmacology","General Medicine"],"container-title-short":"Nat Rev Drug Discov","published":{"date-parts":[[2002,7]]}} \ No newline at end of file diff --git a/test/testdata/2a65476f9d13cb10f1deb219ec20e9bdde891f2f.json b/test/testdata/2a65476f9d13cb10f1deb219ec20e9bdde891f2f.json new file mode 100644 index 00000000..b70c6b83 --- /dev/null +++ b/test/testdata/2a65476f9d13cb10f1deb219ec20e9bdde891f2f.json @@ -0,0 +1,24 @@ +{ + "encoding": null, + "headers": { + "access-control-allow-headers": "X-Requested-With, Accept, Accept-Encoding, Accept-Charset, Accept-Language, Accept-Ranges, Cache-Control", + "access-control-allow-origin": "*", + "access-control-expose-headers": "Link", + "connection": "close", + "content-encoding": "gzip", + "content-length": "1385", + "content-type": "application/vnd.citationstyles.csl+json", + "date": "Thu, 09 Jun 2022 10:37:17 GMT", + "link": "; rel=\"canonical\", ; version=\"vor\"; type=\"application/pdf\"; rel=\"item\", ; version=\"vor\"; type=\"text/html\"; rel=\"item\", ; version=\"vor\"; type=\"application/pdf\"; rel=\"item\", ; version=\"tdm\"; rel=\"license\"", + "permissions-policy": "interest-cohort=()", + "server": "Jetty(9.4.40.v20210413)", + "vary": "Accept, Accept-Encoding", + "x-api-pool": "public", + "x-rate-limit-interval": "1s", + "x-rate-limit-limit": "50", + "x-ratelimit-interval": "1s", + "x-ratelimit-limit": "50" + }, + "status_code": 200, + "url": "https://api.crossref.org/v1/works/10.1038%2Fnrd842/transform" +} \ No newline at end of file diff --git a/test/testdata/2b322ef125a11d78fe8815231ea9684a584a0e9a.html b/test/testdata/2b322ef125a11d78fe8815231ea9684a584a0e9a.html new file mode 100644 index 00000000..01a10297 --- /dev/null +++ b/test/testdata/2b322ef125a11d78fe8815231ea9684a584a0e9a.html @@ -0,0 +1 @@ +CNN - Breaking News, U.S., World, Weather, Entertainment & Video News
\ No newline at end of file diff --git a/test/testdata/2b322ef125a11d78fe8815231ea9684a584a0e9a.json b/test/testdata/2b322ef125a11d78fe8815231ea9684a584a0e9a.json new file mode 100644 index 00000000..c926fb08 --- /dev/null +++ b/test/testdata/2b322ef125a11d78fe8815231ea9684a584a0e9a.json @@ -0,0 +1,28 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "398", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "33925", + "Content-Type": "text/html; charset=utf-8", + "Date": "Mon, 08 Jan 2018 15:09:45 GMT", + "Fastly-Debug-Digest": "67f9f2e80720128694de4d55b6bd4871f6e8fb89010fb7129c4ddfb8175ec498", + "Set-Cookie": "countryCode=IR; Domain=.cnn.com; Path=/, geoData=tehran|07|0|IR|AS; Domain=.cnn.com; Path=/, tryThing00=2069; Domain=.cnn.com; Path=/; Expires=Sun Apr 01 2018 00:00:00 GMT", + "Vary": "Accept-Encoding, Fastly-SSL, Fastly-SSL", + "Via": "1.1 varnish, 1.1 varnish", + "X-Cache": "HIT, HIT", + "X-Cache-Hits": "1, 3", + "X-Served-By": "cache-iad2151-IAD, cache-hhn1539-HHN", + "X-Timer": "S1515424185.039635,VS0,VE0", + "access-control-allow-origin": "*", + "cache-control": "max-age=60", + "content-security-policy": "default-src 'self' blob: https://*.cnn.com:* http://*.cnn.com:* *.cnn.io:* *.cnn.net:* *.turner.com:* *.turner.io:* *.ugdturner.com:* courageousstudio.com *.vgtf.net:*; script-src 'unsafe-eval' 'unsafe-inline' 'self' *; style-src 'unsafe-inline' 'self' blob: *; child-src 'self' blob: *; frame-src 'self' *; object-src 'self' *; img-src 'self' data: blob: *; media-src 'self' data: blob: *; font-src 'self' data: *; connect-src 'self' *; frame-ancestors 'self' *.cnn.com:* *.turner.com:* courageousstudio.com;", + "x-content-type-options": "nosniff", + "x-servedByHost": "::ffff:172.17.47.18", + "x-xss-protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://edition.cnn.com/" +} \ No newline at end of file diff --git a/test/testdata/2bac30e4721173648785a6d5d88262dbc56e33b1.html b/test/testdata/2bac30e4721173648785a6d5d88262dbc56e33b1.html new file mode 100644 index 00000000..8749c355 --- /dev/null +++ b/test/testdata/2bac30e4721173648785a6d5d88262dbc56e33b1.html @@ -0,0 +1,1017 @@ + + + +DealBook - The New York Times + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+Edition: U.S. / Global +
+ +
+
+
+ +
+
+
+
+ + + + + +
+
+
+ + + +
+

+Wednesday, May 24, 2017 +

+
+ +

+ +Business Day + +Business +

+ +
+
+ +
+
+
+
+ +
+ +Follow us + + + + +
+
+ + +
+
+
+
+
+ +
+
+
+

+After Complaints, Fannie Mae Will Stop Selling Homes to Vision Property

+ +
+

+Fannie Mae said it had stopped selling properties to the firm after conducting a review of the firm’s rent-to-own program.

+
+
+ +
+
+

+Unity Technologies, Whose Engine Is Behind Pokémon Go, Agrees to Funding

+ +

+If a new investment round closes, it could bring Unity, which makes software at the heart of video games, $400 million and value it at about $2.6 billion.

+
+
+
+ +Speaker Paul D. Ryan at a news conference on Capitol Hill on Tuesday. He said Congress would take the president’s budget “and then work on our own budget, which is the case every single year.” + +
+

+Republicans, Pushing Aside Trump’s Budget, Find Few Alternatives

+ +

+The budget battle ahead is likely to mirror the party’s health care fight, in which concessions to moderates alienate conservatives and vice versa.

+
+
+
+ +An Uber driver in the Bronx. A lawsuit last year said that Uber was committing a form of wage theft. + +
+

+Uber to Repay Millions to Drivers, Who Could Be Owed Far More

+ +

+The company concedes taking tens of millions in excess commissions. A lawsuit alleges improper tax deductions that are costing drivers even more.

+
+
+
+ +Eric Ueland, right, a Senate Budget Committee staff member, distributing the 2018 federal budget proposal on Capitol Hill on Tuesday. + +
+

+Trump’s Problematic Math: Budget Plan Adds Growth, but Doesn’t Subtract Cost

+ +

+The White House is projecting faster growth as a consequence of tax cuts. But it does not project the cost of those tax cuts, that is, the loss in tax revenue.

+
+
+
+ +Target’s headquarters in Minneapolis. A settlement by the company ended an investigation into how the data of millions of customers was compromised in 2013. + +
+

+Target to Pay $18.5 Million to 47 States in Security Breach Settlement

+ +

+The agreement, which includes the District of Columbia, ends an investigation into how hackers obtained information about tens of millions of people in 2013.

+
+
+
+ +Soybean meal produced by Glencore. A deal between Glencore and Bunge would add to the consolidation reshaping big agricultural companies. + +
+

+Glencore Makes Informal Takeover Approach to Bunge

+ +

+The talks are at an early stage, but a deal would make Glencore a large player in the United States agricultural commodities market.

+
+
+
+ +The Trump administration promises average annual economic growth of 3 percent, but the Federal Reserve and the Congressional Budget Office have projected a pace of less than 2 percent in the long run. + +
+

+Economists See Little Magic in Tax Cuts to Promote Growth

+ +

+President Trump’s budget hews to a longtime Republican premise, but past efforts to enlarge the pie even while slicing it have fallen short of hopes.

+
+
+

+Obama’s Fiduciary Rule, After a Delay, Will Go Into Effect

+ +

+New protections requiring financial advisers to put customers’ interests first will take effect in June, even as regulators continue to review them.

+
+
+
+ +A Chinese magazine poster featuring President Trump at a newsstand in Shanghai in March. Mr. Trump could have up to 116 trademarks in China.       + +
+

+
Trump Adds Another Chinese Trademark to His Portfolio

+ +

+The president’s trove of trademarks in China and elsewhere have sparked criticism over potential conflicts, and run counter to his nationalistic agenda.

+
+
+
+ +An employee at an avocado packaging plant in Michoacán State, Mexico. The United States buys most of Mexico’s avocados, which would be subject to a 20 percent tax under a proposed plan. + +
+

+Border Tax’s Apparent Demise Jeopardizes G.O.P. Overhaul Plan

+ +

+Both Democrats and Republicans, and even the White House, have derided a key provision of House Speaker Paul D. Ryan’s plan for rewriting the tax code in 2017.

+
+
+
+ +The Fiat Chrysler display at the North American International Auto Show in Detroit in January. The emissions issue has essentially stopped Fiat Chrysler’s domestic sales of diesel-powered Ram trucks and Grand Cherokees. + +
+

+U.S. Sues Fiat Chrysler, Accusing It of Using Software to Pass Emissions Tests

+ +

+The move occurred days after the company acted to modify software that the government had said concealed the actual emissions from about 104,000 vehicles.

+
+
+
+ +President Trump’s budget proposes a major restructuring of the way electricity is bought and sold in Western states, which rely on hydroelectric power generated by government structures like the Hoover Dam in Nevada. + +
+

+Trump Budget Proposes Deep Cuts in Energy Innovation Programs

+ +

+The spending plan also calls for raising billions of dollars by opening up public lands to oil and gas drilling and selling oil from the Strategic Petroleum Reserve.

+
+
+
White Collar Watch
+
+ +Michael T. Flynn, President Trump’s former national security adviser, has refused to turn over documents to the Senate Intelligence Committee. + +
+

+Fifth Amendment Makes it Hard to Build a Case Against Flynn

+ +

+The special counsel investigating the Russia matter must find an end-run to obtain documents that Michael T. Flynn refuses to turn over.

+
+
+
+ + + +
+

+Movers: Trump’s Budget and Fed Speakers

+ +

+We’re following major developments in the markets throughout the day. Check here for the latest updates.

+
+
+
+ +Workers at the Government Publishing Office in Washington preparing the 2018 budget documents last week. + +
+

+Morning Agenda: A Budget With a Big If

+ +

+President Trump’s $4.1 trillion spending plan for 2018 calls for increases in spending on the military and on border security, with huge tax cuts.

+
+
+
+ +The Park Lane offers wide-angle views of Central Park. The developers had hoped to build a supertower with ultraluxury apartments there. + +
+

+Malaysian Money. Opulent Ideas. But Now, for Park Lane, a Forced Sale.

+ +

+The Park Lane Hotel in Manhattan was to become an ultraluxury tower. Now, amid global intrigue and money laundering charges, it will go to the highest bidder.

+
+
+
+
+
+
Insight & Analysis
+
White Collar Watch
+
+ +The former F.B.I. director James B. Comey, left, with his predecesssor, Robert S. Mueller III, in 2013. Mr. Mueller has been appointed as special counsel to investigate possible Russian involvement in the 2016 presidential election. + +
+

+Lawyers Are the Big Winners in the Inquiry Into the Election

+ +

+Legal fees required to deal with the investigation into Russian involvement in the election will be steep. Many of those connected to it may not be able to afford them.

+
+
+
White Collar Watch
+
+ +Some have seen parallels between President Trump’s firing of James B. Comey and the Watergate cover-up that toppled President Richard M. Nixon. + +
+

+Why Obstruction of Justice Is a Hard Crime to Prove

+ +

+Any inquiry into possible obstruction by the president or others will confront Supreme Court decisions that have been notably hostile to cases that push the limits of the law.

+
+
+
+ +
Special Sections
+
+
+
+ +President Trump, at a forum with administration officials and business leaders at the White House in February, is building a government rich in corporate experience. + +
+
Special Section
+
+DealBook: Business Goes to Washington
+ +

+“We are absolutely destroying these horrible regulations that have been placed on your heads.” — President Trump.

+
+
+
+
+ +
Editors’ Picks
+
+
+
+ + + +
+
+The Housing Trap
+ +

+Seller financing can become a money trap for low-income buyers.

+
+
+
+ +
+
+
+
+
+ +
+
+ +
+
+
+ +

Markets »

+
+
+ +
+
+
+ + + + +
+ +
+
+
+ +
+
+
+ + +
+

+Subscribe to +DealBook Emails and Alerts +

+
+

Sign up for the DealBook Newsletter, delivered every morning and afternoon, and receive breaking news alerts throughout the day.

+
+

Subscribe

+
+ +
+
+

Columns

+
+
+ +thumbnail + +
+
Another View
+

Opinion Contributors

+
+
+
+ +thumbnail + +
+
Book Entry
+

Jonathan A. Knee

+
+
+
+ +thumbnail + +
+
Breakingviews
+

Reuters Commentary

+
+
+
+ +thumbnail +
+
Deal Professor
+

Steven Davidoff Solomon

+
+
+
+ +thumbnail + +
+
DealBook Column
+

Andrew Ross Sorkin

+
+
+
+ +thumbnail + +
+
In Debt
+

Stephen J. Lubben

+
+
+
+ +thumbnail + +
+
Street Scene
+

William D. Cohan

+
+
+
+ +thumbnail + +
+
White Collar Watch
+

Peter J. Henning

+
+
+
+ + +

Most Popular - Business Day

+
+ +
+ + + + + + + + + + +
+ +
+
+
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+
+ +
+
+ +
+
+ +
+ +
+
+
+ +
+ +
+ +
+
+
+
+
+
+ +
+
+ + + + + + + + + diff --git a/test/testdata/2bac30e4721173648785a6d5d88262dbc56e33b1.json b/test/testdata/2bac30e4721173648785a6d5d88262dbc56e33b1.json new file mode 100644 index 00000000..00707f90 --- /dev/null +++ b/test/testdata/2bac30e4721173648785a6d5d88262dbc56e33b1.json @@ -0,0 +1,28 @@ +{ + "encoding": "ISO-8859-1", + "headers": { + "Accept-Ranges": "bytes", + "Cache-Control": "no-cache", + "Connection": "close", + "Content-Encoding": "gzip", + "Content-Type": "text/html", + "Cteonnt-Length": "129584", + "Date": "Wed, 24 May 2017 07:34:59 GMT", + "Expires": "Thu, 01 Dec 1994 16:00:00 GMT", + "Pragma": "no-cache", + "Server": "Apache", + "Transfer-Encoding": "chunked", + "Vary": "Host,Accept-Encoding,Fastly-SSL", + "X-API-Version": "F-4", + "X-Cache": "MISS, MISS from google.com", + "X-Cache-Hits": "0", + "X-Cache-Lookup": "MISS from google.com:85", + "X-Frame-Options": "DENY", + "X-Origin-Time": "2017-05-24 03:34:59 EDT", + "X-PageType": "legacy", + "X-Served-By": "cache-sjc3121-SJC", + "X-Timer": "S1495611299.204278,VS0,VE31" + }, + "status_code": 200, + "url": "http://www.nytimes.com/pages/business/dealbook/index.html" +} \ No newline at end of file diff --git a/test/testdata/2eb8bc5e9578b30fc9c368eda08048c2208b2232.html b/test/testdata/2eb8bc5e9578b30fc9c368eda08048c2208b2232.html new file mode 100644 index 00000000..9a94c167 --- /dev/null +++ b/test/testdata/2eb8bc5e9578b30fc9c368eda08048c2208b2232.html @@ -0,0 +1,3551 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Management of a giant prostatic enlargement: Case report and review of the literature - PubMed + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Skip to main page content + + + + +
+ + +
+
+
+ + + + + + + + + + + + + +
+
+
+
+ + + + + + +
+ + + +
+ +
+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ +
+ + + + + + + + +
+ + + + + + + + + + + + +
+ + +
+ +
+ +
Review
+ + + + +
+ +. 2020 Jul-Sep;27(3):242-247. + +
+ + + + + doi: 10.4103/npmj.npmj_69_20. + + + + + + + + +
+ + +

+ + + + + + + Management of a giant prostatic enlargement: Case report and review of the literature + + + + +

+ + + + + + + +
+ + + + Affiliations + + + + + +
+ + + + + + + + + + + + Free article + + +
+
+ +
Review
+ + + + +

+ + + + + + + Management of a giant prostatic enlargement: Case report and review of the literature + + + + +

+ + + +
+ + + + Rufus Wale Ojewola et al. + + + + + + + Niger Postgrad Med J. + + + + 2020 Jul-Sep. + + + + +
+ + + Free article + +
+ + + + +
+ + + +
+ +
+ +
+ + +
+ + + +
+ + + + + + + +
+ +

+ Abstract + +

+ + + +
+ + + + + +

+ + Giant prostatic enlargement often referred to as giant prostatic hyperplasia (GPH) is a rare condition described as a massive prostatic enlargement >500 g. Up until now, the total number of GPH reported worldwide in medical literature is < 30. To the best of our knowledge, only one case of a giant prostate has been reported in Nigeria. We report a case of a giant prostatic enlargement treated by open simple retropubic prostatectomy in a 73-year-old man who was suffering from lower urinary tract symptoms and persistent visible (gross) haematuria necessitating repeated blood transfusions. Transrectal ultrasound (TRUS) scan revealed a markedly enlarged prostate measuring 565 ml with a suspicious nodule and prostate-specific antigen level of 48.5 ng/ml. He had a 20-core TRUS-guided prostatic biopsy which showed benign prostatic hyperplasia. We performed a retropubic open simple prostatectomy for complete enucleation of the adenoma. Specimen weighed 512.5 g with dimensions of 17 cm × 16 cm and a volume of 528 ml. Histological examination showed prostatic fibromuscular hyperplasia with a focus of adenocarcinoma. The patient had an uneventful post-operative recovery and was discharged within a week post-surgery. Urethral catheter was removed after 2 weeks with satisfactory outcome. +

+ + + + + + +
+ + + + + + + + + + + + + + + + +

+ + + Keywords: + + + Benign prostatic hyperplasia; giant prostatic enlargement; giant prostatic hyperplasia; prostatectomy. +

+ + + + + +
+ + + + + + + + +
+

+ Conflict of interest statement +

+ +
+ + + +

None

+ + +
+
+ + + + + + + + + +
+

+ Similar articles +

+ + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+

+ LinkOut - more resources +

+ + +
+ + +
+ + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/testdata/2eb8bc5e9578b30fc9c368eda08048c2208b2232.json b/test/testdata/2eb8bc5e9578b30fc9c368eda08048c2208b2232.json new file mode 100644 index 00000000..c293b014 --- /dev/null +++ b/test/testdata/2eb8bc5e9578b30fc9c368eda08048c2208b2232.json @@ -0,0 +1,23 @@ +{ + "encoding": "utf-8", + "headers": { + "Alt-Svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "Cache-Control": "private", + "Content-Type": "text/html; charset=utf-8", + "Date": "Sat, 27 Aug 2022 12:11:25 GMT", + "Referrer-Policy": "same-origin", + "Server": "nginx", + "Set-Cookie": "pm-csrf=N4q0ezMvkEALHh16rjyA4xiocaKsqGgw6GN2jUlcBctkdmwts8wskLubauxFXLeF; expires=Sat, 26 Aug 2023 12:11:25 GMT; HttpOnly; Max-Age=31449600; Path=/; SameSite=Lax; Secure, pm-sessionid=levdbe6vccvp87sqslxqqmi04ds09m9q; expires=Sat, 27 Aug 2022 20:11:25 GMT; HttpOnly; Max-Age=28800; Path=/; Secure, ncbi_sid=599900A430A07A63_1038SID; Domain=.nih.gov; expires=Sun, 27 Aug 2023 12:11:25 GMT; HttpOnly; Max-Age=31536000; Path=/; Secure", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "Transfer-Encoding": "chunked", + "Vary": "Origin", + "Via": "1.1 google", + "X-Content-Type-Options": "nosniff", + "X-Frame-Options": "DENY", + "X-UA-Compatible": "IE=Edge", + "X-XSS-Protection": "1; mode=block", + "content-encoding": "gzip" + }, + "status_code": 200, + "url": "https://pubmed.ncbi.nlm.nih.gov/32687126/" +} \ No newline at end of file diff --git a/test/testdata/317fabb623b41b049c7b8d01791f9ae63a4c834c.html b/test/testdata/317fabb623b41b049c7b8d01791f9ae63a4c834c.html new file mode 100644 index 00000000..466f864c --- /dev/null +++ b/test/testdata/317fabb623b41b049c7b8d01791f9ae63a4c834c.html @@ -0,0 +1,59 @@ + + + + +Black Convicts + + + + + + + + + + + +
+ + + + + + + +
+ + + + + + + + + + +
+

Black Convicts

+

When James Brown arrived in Van Diemen's Land in 1833 he told the muster master: 'I was taken when a child as a Slave from the Congo River and sold to a Spanish Slaver. Captured by a British King's Ship & liberated at Sierra Leone. Brought away from thence as servant'. Brown was one of several hundred black convicts transported to Van Diemen's Land.

+

After 1830, the West Indies slave colonies sought transportation as a means to control a dangerously restive slave population excited by rumours of impending emancipation. The same year as Brown, Alexander Simpson arrived in Van Diemen's Land and told the Muster Master his offence was: 'Mutiny & exciting the Slaves to rebellion. I was a slave myself'. Indeed, Simpson was a participant in the largest slave rebellion in the British Empire at Montego Bay, Jamaica, in 1831.

+

A few black women transported included Maria, a slave from the Bay of Honduras who stabbed a man who may have been trying to rape her, and Priscilla from Jamaica who was convicted of trying to poison her mistress. Other African convicts were the indigenous Khoi (called Hottentots by the British) of the Cape Colony who were mostly convicted of banditry and cattle stealing, though their real crime was resistance to colonial rule.

+

One of Tasmania's last convict bushrangers was Peter Haley, a Khoi man from the Cape Colony, executed in 1857. William Cuffay, son of a West Indian slave in London, was transported as one of the leaders of a republican Chartist conspiracy in London in 1848.

+

Further reading: C Pybus, 'A touch of the tar', London Papers in Australian Studies 3, 2001.

+
Cassandra Pybus
+
+
+ + diff --git a/test/testdata/317fabb623b41b049c7b8d01791f9ae63a4c834c.json b/test/testdata/317fabb623b41b049c7b8d01791f9ae63a4c834c.json new file mode 100644 index 00000000..92b0f84a --- /dev/null +++ b/test/testdata/317fabb623b41b049c7b8d01791f9ae63a4c834c.json @@ -0,0 +1,25 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "max-age=0, private", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Type": "text/html; charset=UTF-8", + "Date": "Fri, 26 Aug 2022 09:01:34 GMT", + "ETag": "W/\"9e01c8-10ce-4c34227183b80\"", + "Last-Modified": "Mon, 25 Jun 2012 01:46:38 GMT", + "Matrix-Upstream": "web-golive-upstream", + "Server": "Apache", + "Set-Cookie": "SQ_SYSTEM_SESSION=true; path=/; domain=secure.utas.edu.au; expires=Fri, 26-Aug-2022 09:01:34 GMT; secure; HttpOnly, BIGipServerweb6.its.utas.edu.au_80=235274762.20480.0000; path=/; Httponly; Secure", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding, Accept-Encoding", + "Via": "1.1 squizedge.net", + "X-DEBUG-Phase": "1", + "X-DEBUG-Referer": "0", + "X-FRAME-OPTIONS": "SAMEORIGIN", + "X-Request-ID": "63697026-20e8-47fc-a65c-b68bc75b3961", + "X-upgrade-enabled": "off" + }, + "status_code": 200, + "url": "https://www.utas.edu.au/library/companion_to_tasmanian_history/B/Black%20Convicts.htm" +} \ No newline at end of file diff --git a/test/testdata/31c4bf240990f94c08ad0678c0affc55a58f587e.html b/test/testdata/31c4bf240990f94c08ad0678c0affc55a58f587e.html new file mode 100644 index 00000000..a59bd04a --- /dev/null +++ b/test/testdata/31c4bf240990f94c08ad0678c0affc55a58f587e.html @@ -0,0 +1,11 @@ +@Book{noorlib6120, +Title = {ایران در زمان ساسانیان: تاریخ ایران ساسانی تا حمله عرب و وضع دولت و ملت در زمان ساسانیان}, +Year = {1368}, +Url = {http://www.noorlib.ir/View/fa/Book/BookView/Image/6120}, +publisher = {دنیای کتاب}, +address = {تهران - ایران}, +author = {رشید یاسمی, غلامرضا and کریستن سن, آرتور امانویل}, +Series = {ایران در زمان ساسانیان: تاریخ ایران ساسانی تا حمله عرب و وضع دولت و ملت در زمان ساسانیان}, +Volume = {1} +Language = {فارسی} +} \ No newline at end of file diff --git a/test/testdata/31c4bf240990f94c08ad0678c0affc55a58f587e.json b/test/testdata/31c4bf240990f94c08ad0678c0affc55a58f587e.json new file mode 100644 index 00000000..73095461 --- /dev/null +++ b/test/testdata/31c4bf240990f94c08ad0678c0affc55a58f587e.json @@ -0,0 +1,16 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "private", + "Content-Disposition": "attachment; filename=\"noorlib-6120.bib\"", + "Content-Length": "646", + "Content-Type": "application/x-bibtex; charset=UTF-8", + "Date": "Tue, 23 May 2017 17:51:51 GMT", + "Server": "Microsoft-IIS/7.5", + "Set-Cookie": "ASP.NET_SessionId=ky0tkllarud3api10xrnkky4; path=/; HttpOnly", + "X-AspNet-Version": "4.0.30319", + "X-Powered-By": "ASP.NET" + }, + "status_code": 200, + "url": "http://www.noorlib.ir/View/HttpHandler/CitationHandler.ashx?id=6120&format=BibTex" +} \ No newline at end of file diff --git a/test/testdata/34fadf2b072dd824a3271bc272e6b324e81ba968.html b/test/testdata/34fadf2b072dd824a3271bc272e6b324e81ba968.html new file mode 100644 index 00000000..4917bb4b --- /dev/null +++ b/test/testdata/34fadf2b072dd824a3271bc272e6b324e81ba968.html @@ -0,0 +1 @@ +{"indexed":{"date-parts":[[2022,3,31]],"date-time":"2022-03-31T06:07:08Z","timestamp":1648706828600},"reference-count":9,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"2","license":[{"start":{"date-parts":[[1985,6,1]],"date-time":"1985-06-01T00:00:00Z","timestamp":486432000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"}],"content-domain":{"domain":[],"crossmark-restriction":false},"published-print":{"date-parts":[[1985,6]]},"DOI":"10.1109\/tim.1985.4315297","type":"journal-article","created":{"date-parts":[[2008,7,21]],"date-time":"2008-07-21T21:28:29Z","timestamp":1216675709000},"page":"185-187","source":"Crossref","is-referenced-by-count":14,"title":"Near-Zero Bias Arrays of Josephson Tunnel Junctions Providing Standard Voltages up to 1 V","prefix":"10.1109","volume":"IM-34","author":[{"given":"Jurgen","family":"Niemeyer","sequence":"first","affiliation":[]},{"given":"Johann H.","family":"Hinken","sequence":"additional","affiliation":[]},{"given":"Richard L.","family":"Kautz","sequence":"additional","affiliation":[]}],"member":"263","reference":[{"key":"ref4","first-page":"119","article-title":"Experimental observations of microwave induced dc voltage phenomenon","author":"guang-ji","year":"1983","journal-title":"Proc Joint Sino-Japanese Seminar on Jos Eff Phys and Appl"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1109\/TMAG.1981.1060950"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/TIM.1984.4315230"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1063\/1.336221"},{"key":"ref8","year":"0"},{"key":"ref7","first-page":"375","article-title":"Simplified analysis and synthesis of fin-line tapers","volume":"37","author":"hinken","year":"1983","journal-title":"Arch Elek \ufffdbertragung"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1063\/1.89520"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1147\/rd.242.0195"},{"key":"ref1","first-page":"120","article-title":"The realization of the Josephson potentiometer","author":"koyanagi","year":"1981","journal-title":"Proc Int Conf on Prec Meas and Fund Const"}],"container-title":"IEEE Transactions on Instrumentation and Measurement","original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx5\/19\/4315277\/04315297.pdf?arnumber=4315297","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,11,29]],"date-time":"2021-11-29T20:46:34Z","timestamp":1638218794000},"score":1,"resource":{"primary":{"URL":"http:\/\/ieeexplore.ieee.org\/document\/4315297\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[1985,6]]},"references-count":9,"journal-issue":{"issue":"2"},"URL":"http:\/\/dx.doi.org\/10.1109\/TIM.1985.4315297","relation":{},"ISSN":["0018-9456","1557-9662"],"subject":["Electrical and Electronic Engineering","Instrumentation"],"container-title-short":"IEEE Trans. Instrum. Meas.","published":{"date-parts":[[1985,6]]}} \ No newline at end of file diff --git a/test/testdata/34fadf2b072dd824a3271bc272e6b324e81ba968.json b/test/testdata/34fadf2b072dd824a3271bc272e6b324e81ba968.json new file mode 100644 index 00000000..f8b73dab --- /dev/null +++ b/test/testdata/34fadf2b072dd824a3271bc272e6b324e81ba968.json @@ -0,0 +1,24 @@ +{ + "encoding": null, + "headers": { + "access-control-allow-headers": "X-Requested-With, Accept, Accept-Encoding, Accept-Charset, Accept-Language, Accept-Ranges, Cache-Control", + "access-control-allow-origin": "*", + "access-control-expose-headers": "Link", + "connection": "close", + "content-encoding": "gzip", + "content-length": "1377", + "content-type": "application/vnd.citationstyles.csl+json", + "date": "Thu, 09 Jun 2022 10:35:30 GMT", + "link": "; rel=\"canonical\", ; version=\"vor\"; rel=\"item\", ; version=\"vor\"; rel=\"license\"", + "permissions-policy": "interest-cohort=()", + "server": "Jetty(9.4.40.v20210413)", + "vary": "Accept, Accept-Encoding", + "x-api-pool": "public", + "x-rate-limit-interval": "1s", + "x-rate-limit-limit": "50", + "x-ratelimit-interval": "1s", + "x-ratelimit-limit": "50" + }, + "status_code": 200, + "url": "https://api.crossref.org/v1/works/10.1109%2FTIM.1985.4315297/transform" +} \ No newline at end of file diff --git a/test/testdata/3a00b911128333cc3f5269954ef3652d3b57ad46.html b/test/testdata/3a00b911128333cc3f5269954ef3652d3b57ad46.html new file mode 100644 index 00000000..91452334 --- /dev/null +++ b/test/testdata/3a00b911128333cc3f5269954ef3652d3b57ad46.html @@ -0,0 +1,2267 @@ + + + خبرگزاری ایسنا | صفحه اصلی | ISNA News Agency + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + +
+
+
+
+
+ +
+
+ +
+
+
+
+

اجتماعی +

+
+
+ +
+
+
+
+

علمی و دانشگاهی +

+
+
+ +
+
+
+
+

فرهنگی و هنری +

+
+
+ +
+
+
+
+

سیاسی +

+
+
+ +
+
+
+
+

اقتصادی +

+
+
+ +
+
+
+
+

بین‌الملل +

+
+
+ +
+
+
+
+

ورزشی +

+
+
+ +
+
+
+
+

استان‌ها +

+
+
+ +
+
+ +
+
+

رسانه‌های دیگر +

+
+
+ +
+
+
+
+

بازار +

+
+
+ +
+
+
+ +
+
+
+ +
+
+ + + + + + + + + \ No newline at end of file diff --git a/test/testdata/3a00b911128333cc3f5269954ef3652d3b57ad46.json b/test/testdata/3a00b911128333cc3f5269954ef3652d3b57ad46.json new file mode 100644 index 00000000..f940b324 --- /dev/null +++ b/test/testdata/3a00b911128333cc3f5269954ef3652d3b57ad46.json @@ -0,0 +1,24 @@ +{ + "encoding": "UTF-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "2", + "Cache-Control": "max-age=60", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "27862", + "Content-Type": "text/html;charset=UTF-8", + "Date": "Tue, 23 May 2017 17:54:08 GMT", + "Expires": "Tue, 23 May 2017 17:55:09 GMT", + "Server": "Apache-Coyote/1.1", + "Vary": "Accept-Encoding", + "Via": "1.1 varnish-v4", + "X-Content-Type-Options": "nosniff", + "X-Frame-Options": "SAMEORIGIN", + "X-Varnish": "287492040 287633622", + "X-XSS-Protection": "1; mode=block", + "grace": "none" + }, + "status_code": 200, + "url": "http://www.isna.ir/" +} \ No newline at end of file diff --git a/test/testdata/3a4f93ccfc201e8d1c57e1553cbcf2ee915a5c0d.html b/test/testdata/3a4f93ccfc201e8d1c57e1553cbcf2ee915a5c0d.html new file mode 100644 index 00000000..4abc914d --- /dev/null +++ b/test/testdata/3a4f93ccfc201e8d1c57e1553cbcf2ee915a5c0d.html @@ -0,0 +1,570 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Magiran | روزنامه سرمایه (1386/03/01): دکتر طاهر صباحی، محقق و مجموعه دار فرش: بازار جهانی با تولید فرش هنری نصیب ایران می شود + + + + + + + + + + + + + + +
+ +
+ + +
+
+
+ +
+
+ آرشیو سه‌شنبه ۱ خرداد ۱۳۸۶، شماره ۴۶۲ +
+
+
هنر
+
+
۸
+
+
+
+
+ +
+
+ +
+
+
+
+

دکتر طاهر صباحی، محقق و مجموعه دار فرش: بازار جهانی با تولید فرش هنری نصیب ایران می شود

+
+
+
+ + آزاده شهمیر نوری + + +
+
+ +
+

دکتر طاهر صباحی سال هاست در اروپا به عنوان یکی از کارشناسان معتبر فرش فعالیت می کند. او تا به حال 18 عنوان کتاب به زبان های ایتالیایی، فرانسوی و انگلیسی درباره هنر فرش ایرانی منتشر کرده است. صباحی علاوه بر این 14 سال است، نشریه ای با عنوان «گره» به دو زبان انگلیسی و ایتالیایی منتشر می کند.

او که 67 سال سن دارد، همچنین صاحب مجموعه ای نفیس و بی نظیر از فرش های ایرانی است. با دکتر صباحی پیرامون فرش هنری ایران و فعالیت هایش در این حوزه، گفت وگو کرده ایم:

چطور شد از اروپا سر درآوردید؟

مادرم دوست داشت، پزشک شوم. وقتی تحصیلاتم را در رشته پزشکی تمام کردم چون مادرم نگران بود، مرا به سربازی بفرستند، برنگشتم و همان جا در رشته داروسازی ادامه تحصیل دادم. بعد هم وارد کار فرش شدم.

پدر شما در کار فرش بود؟

نه. اما این شاید حسی بوده که از پدربزرگم به من رسیده است. پدرم به این حرفه علاقه ای نداشت، پسرم هم علاقه مند نیست، اما از همین حالا می بینم، نوه چهار ساله ام چقدر فرش را دوست دارد. یک بار فرشی برایش بردم تا در را باز کرد، سریع آن را گرفت و پرسید این مال من است؟ حالا آن را در اتاقش پهن کرده، رویش می نشیند، بازی می کند و می خوابد. خوشحالم او هم علاقه به فرش را از پدر بزرگش ارث برده است. فکر می کنم، احساسات فرد به یک موضوع را باید از همان کودکی بیدار کرد.

مجموعه شخصی تان را در ایتالیا نگهداری می کنید؟

بله من در تورینو، شهری در شمال ایتالیا زندگی می کنم و آن جا موزه کوچکی از مجموعه فرش های شخصی ام ساخته ام و به نمایش در آورده ام. در طول سال بسیاری از علاقه مندان و محققان فرش به موزه می آیند تا مجموعه ام را از نزدیک ببینند.

شما مجموعه دار هستید، همین طور محقق و تاجر، آیا این حرفه های متفاوت فرش، به هم ارتباطی دارد؟

به نظر من این ها همه لازم و ملزوم یکدیگر هستند. بازار در عین حال که محل خرید و فروش فرش است، محل تبادل اطلاعات هم محسوب می شود و پر از خبرها و نکاتی است که به درد یک محقق می خورد. تجار پیر بازار، هر کدام برای خودشان یک دایره المعارف هستند. در واقع یک نویسنده، هر چقدر کتاب بخواند و بنویسد اما از پشت میزش جدا نشود، کتاب هایش به اندازه کسی که می رود، می بیند، می خرد و می فروشد و با این کار زندگی می کند، نمی تواند موفق باشد. من برای نوشتن کتاب هایم سفرهای زیادی کرده ام، موزه های دنیا را دیده ام، به تماشای محل های بافت فرش از قفقاز تا ترکیه رفته ام، دیده هایم را با کتاب های موجود، مقایسه کرده ام و به نکات تازه ای رسیده ام. برای نمونه برادران کاستلی دو تاجر معروف بودند که به سفارش آن ها فرش در کرمان بافته می شد. ما فکر می کردیم آن ها حدود سال 1910 یا 1920 در ایران بودند، اما فرشی از واشنگتن برایم پیدا کردند و آن را خریدم که روی آن تاریخ 1886 بافته شده و به امضای برادران کاستلی بود. پیدا کردن همین یک فرش تاریخ را 40 سال عقب کشید. این تحقیقات میدانی که الزام بازار خرید و فروش فرش هست، خیلی به نویسنده کمک می کند. مجموعه داری هم نوعی بازی و عشق و علاقه به فرش هاست. شما دیگر وارد دنیای فرش های زیبا می شوید. مقابل آن ها می ایستید و حاضرید برای آن فرش همه چیزتان را بدهید تا زیبایی آن به شما تعلق پیدا کند.

سازمان های مرتبط با فرش در ایران، تا به حال از شما نخواسته اند که کتاب و یا تحقیقی در این زمینه انجام دهید و یا 18 عنوان کتابی که منتشر کرده اید به زبان فارسی برگردانید.

تا امروز که چنین اتفاقی نیفتاده است، اما پژوهشکده هنرهای سنتی میراث فرهنگی و گردشگری ابراز تمایل کرده که این کتاب را منتشر کند، اما من چشمم آب نمی خورد. متاسفانه در ایران ارزش چنین تحقیقاتی را نمی دانند.

برای مجموعه تان از چه زمان شروع به خرید فرش کرده اید؟

ما شش خواهر و برادر بودیم البته من و برادرم ناصر بسیار شیطان بودیم. مادرم برای این که کم تر اذیت کنیم، تابستان ها ما را با پدرم به بازار می فرستاد. پدرم ما را سبزه میدان پیاده می کرد. خودش می رفت تا ماشین را پارک کند و ما پیاده می رفتیم تا نوروز خان. سر راهمان یک دکان فرش فروشی بود که دو تخته پاتختی لاکی کاشان زیبا داشت و من هر روز محو تماشای آن ها می شدم. یک روز صبح با تپش قلب از خواب بیدار شدم و فکر کردم، نکند که این فرش ها را بفروشد؟ قلکم را شکستم و همه پول هایم را ریختم در دستمال و بردم سر سفره صبحانه به پدرم دادم و از او خواستم که آن ها را برایم بخرد. پدرم تعجب کرد و گفت دو تا؟ بالاخره پدرم را بردم و او با صاحب دکان صحبت کرد. خلاصه تا وقتی فروشنده آن ها را از دیوار نکند، خیالم راحت نشد. در حقیقت اولین خریدم را در 11 سالگی انجام دادم. اگرچه تا همین اواخر که پدرم زنده بود آن ها را نتوانستم به مجموعه ام اضافه کنم، چون فرش ها را به من نمی داد، تا این که دو قالیچه ابریشمی تبریز برایش خریدم و فرش های خودم را پس گرفتم.

چه نوع فرش هایی در مجموعه دارید؟

تجارت باعث شده بتوانم با فرش فروش ها ارتباط برقرار کنم. من تاجران زیادی را می شناسم و با آن ها دوست هستم، به همین واسطه فرش های زیبایی در مجموعه جمع آوری کرده ام. یادم می آید چند سال پیش به یکی از دوستانم سرزدم. او گفت فرش زیبایی دارم که حتما باید آن را ببینی. شاگردش فرش را پهن کرد. خیلی تعجب کردم چون فرش امضای حاجی جلیلی داشت. فرش حاجی جلیلی زیاد دیده بودم، اما هیچ کدام امضا نداشتند. برادران حاجی جلیلی در مرند فرش تولید می کردند و فرش هایشان بسیار زیباست. تعداد فرش های مجموعه ام را نشمرده ام، اما خیلی زیاد است به طوری که می توانم آن ها را به گونه های مختلف طبقه بندی کنم. فرش های ازبک، ترکمن، خورجین های زیبای مشرق زمین، فرش های بلوچستان و زابل، کرمان، سوزنی های ازبکستان، واگیره، رو اسبی، تبریز و انواع دیگری که هر کدام برای خودش یک مجموعه است.

واگیره چه نوع فرشی است؟

واگیره، الگوهای قالیبافی است که در قدیم استفاده می شده است. نقش های قالیبافی به سه قسمت تقسیم می شود، یکی ذهنی است که در هر قومی سینه به سینه توسط پیرزن ها به جوان ها منتقل می شود، دیگری روی کاغذ حک می شده و فرش را از روی آن می بافتند و دیگری واگیره نام دارد، فرش کوچکی که شامل حاشیه، گل های مختلف نقش ترنج وسط و... می شود و بافنده می تواند از درون آن نقش هایی را که می خواهد در فرش ببافد، جدا کند. 20 سال پیش درباره واگیره ها کتابی نوشتم.

در حال حاضر کتابی در دست انتشار دارید؟

به سفارش سازمان دایره المعارف اسلامی مشغول تکمیل بخش فرش دایره المعارف اسلامی هستم که در بخش حرف «ف» در دانشنامه ایران منتشر می شود. در بخش فرش تمام اصطلاحات اشخاص و فرش های معروف را به تفصیل توضیح می دهم. هر اطلاعی که برای شناختن فرش لازم است، در این کتاب هست.

به نظر شما برای نجات فرش ایران چه باید کنیم. ما تنها کاری که در بازار رقابت با فرش های خارجی می کنیم، متهم کردن کشورهای دیگر به کپی برداری از ایران است.

به تازگی یک سازمان مثل سازمان اوپک که برای نفت تشکیل دادیم از سوی کشورهای پاکستان، هند، چین، ترکیه و نپال برای فرش تشکیل شده است. این سازمان به سمتی می رود که منافع همه در آن حفظ شود. اگر ایران عضو این سازمان شود تنها یک رای دارد، پس حرفی نمی تواند بزند و اگر عضو نشود آن وقت آن ها همه متحد می شوند تا با ما رقابت کنند. شرایط دشواری است. ما باید چاره دیگری پیدا کنیم.

چه چاره ای؟

دنیا پر از گرسنه است. با ماهی 15 هزارتومان و با پشم ارزان می توان به یک بافنده سفارش فرش داد و از آن جایی که شرایط ایران با کشورهای رقیب فرق دارد و بافندگانش به اندازه آن ها محتاج نیستند که به هر قیمتی فرش ببافند، پس ما هرگز در تولید فرش ارزان نمی توانیم با آن ها رقابت کنیم. در دو سال آینده با خروج افغانی ها از ایران و بالا رفتن هزینه کارگر، قیمت در همه امور و رشته ها بالا می رود. پس باید به فکر تولیدات مرغوب و نمایشگاهی باشیم که کشورهای دیگر به فکر تولید آن نیستند.چرا به سراغ موزه های دنیا نمی رویم که تکه پاره های فرش های نفیس ما را نگاه می دارند و همان ها را نمی بافیم. بازار رقابت و تجارت فرش از دست ما رفته، اما می توانیم به عنوان کالای هنری آن را تولید کنیم و بازار را دوباره به دست بیاوریم. کالایی که حتما خریدار خودش را هم دارد.

+
+
+
+
+ +
+
+ +
+
+
+
+
+ + +
+ + روزنامه سرمایه، شماره 462 + +
+
+
+ +
+
+ +
+
+ + روزنامه سرمایه، شماره 14958 + + +
+
+ صاحب امتیاز و مدیر مسئول : + حسین عبده تبریزی +
+ تلفن روزنامه: + ۰۲۱-۲۲۰۵۵۰۸۵ +
+
+
+
+ +
+
+ + + +
+
+ + + +
+
+ + + +
+
+ + + +
+
+
+ +
+
+ + + +
+
روزنامه‌های عضو
+
+
+
+
+ + شرق + +
+
+
+
+ + ایران + +
+
+
+
+ + کیهان + +
+
+
+
+ + رسالت + +
+
+
+
+ + جام جم + +
+
+
+
+ + اعتماد + +
+
+
+
+ + دنیای اقتصاد + +
+
+
+
+
+ + +
+
+ + +
+
+ + +
+ +
+ درخواست پشتیبانی - گزارش اشکال +
+
+ + + + + + + + + + + + + + + + + + diff --git a/test/testdata/3a4f93ccfc201e8d1c57e1553cbcf2ee915a5c0d.json b/test/testdata/3a4f93ccfc201e8d1c57e1553cbcf2ee915a5c0d.json new file mode 100644 index 00000000..4823431e --- /dev/null +++ b/test/testdata/3a4f93ccfc201e8d1c57e1553cbcf2ee915a5c0d.json @@ -0,0 +1,29 @@ +{ + "encoding": "utf-8", + "headers": { + "AR-ATIME": "0.045", + "AR-CACHE": "BYPASS", + "AR-Request-ID": "54e6b5af1d977d1d7dc248a8adcc3fac", + "AR-SID": "2020", + "Accept-Ranges": "bytes", + "Cache-Control": "max-age=0", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "15534", + "Content-Security-Policy": "upgrade-insecure-requests", + "Content-Type": "text/html; charset=utf-8", + "Date": "Sat, 04 Jun 2022 08:54:01 GMT", + "Expires": "Sat, 04 Jun 2022 08:54:01 GMT", + "Keep-Alive": "timeout=65", + "Pragma": "no-cache", + "Server": "ArvanCloud", + "Strict-Transport-Security": "max-age=15768000", + "Vary": "Accept-Encoding, Accept-Encoding", + "X-Content-Type-Options": "nosniff", + "X-Frame-Options": "SAMEORIGIN", + "X-Powered-By": "My Little Pony", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://www.magiran.com/article/1410487" +} \ No newline at end of file diff --git a/test/testdata/3c3c8c1e47b7e028bdabfb866f7bc63c8e898086.html b/test/testdata/3c3c8c1e47b7e028bdabfb866f7bc63c8e898086.html new file mode 100644 index 00000000..0939091f --- /dev/null +++ b/test/testdata/3c3c8c1e47b7e028bdabfb866f7bc63c8e898086.html @@ -0,0 +1,81 @@ + + + + + + +London Development Centre: Support, time, recovery (STR) workers + + + + + + + + + +
+ + +
Care Services Improvement Partnership
+
+

Support, time, recovery (STR) workers

+ +
+
+
 
+
+

Support, Time, Recovery (STR) workers support people with mental health problems in their recovery by spending time with them and providing practical help. STR workers recognise each person’s strengths and needs to work flexibly with the individual, their supporters and other services. ‘Expertise of experience’ is valued, both in the way the STR worker supports the individual, and by the expectation that 20% of the STR workforce will have expertise via the experience of having used mental health services themselves.

The Support Time Recovery worker role was designed by the NHS as a result of feedback from people using mental health services who said they wanted workers who would spend more time supporting them in their recovery. STR workers receive training and supervision in order to reflect on and develop their practice. By December 2006 there will be over 3,000 STR workers throughout England. Some of these will be brand new jobs. Some will be existing job roles which have been ‘re-configured’ to fit the STR criteria.

The London Development Centre offers support for the implementation of STR workers across London. This includes a network to support the implementation of STR workers. The network meets regularly and is for anyone involved in implementing this new role. In addition, The London Development Centre offers 2 day recovery and STR induction training for STR workers and their managers in Central London. It is also possible for the STR Project Lead to advise on training needs and provide bespoke training for teams. For further information on any of the above, please contact Mike Firn (020 7307 2447).

Download a checklist describing the roles and characteristics of an STR worker.

FAQs
How does an STR worker support people in their recovery?
What advantages are there in becoming an STR worker?
What’s in it for the people using STR services?
What are the challenges to STR workers and their teams new to the role?
What is the training pathway for STR workers?

Accelerated Development Programme

Example of ADP Action Plan

To support the process of implementation, steering groups have been established in most areas. You can download examples of terms of reference of STR steering groups:

Terms of reference 1
Terms of reference 2

STR Pursuits, the game
You can download materials to play this game which you can use to help people think about putting the new roles in place.

The game is for 6 players or teams. The coloured cards correspond to different areas for consideration such as 'Designing the Role' and 'Service Users'. Each player or team takes responsibilty for one of these areas, using the game sheets provided to record any relevant points made in the course of the game. The game is played by each player or team in turn picking a card at random, then attempting to answer the question posed on the card with the help of the other players.

STR Pursuits questions          
Game cards         
Game sheets

Other useful documents
Induction for STR workers
WRAP agreement

Last updated on: 12th February 2007

+
+
+
+

Department of HealthWe help to improve services and achieve better outcomes forchildren and families, adults and older people including those with mental health problems, physical or learning disabilities or people in the criminal justice system. We work with and are funded by the Department of Health.

+ + +
+
+ + diff --git a/test/testdata/3c3c8c1e47b7e028bdabfb866f7bc63c8e898086.json b/test/testdata/3c3c8c1e47b7e028bdabfb866f7bc63c8e898086.json new file mode 100644 index 00000000..6669c28c --- /dev/null +++ b/test/testdata/3c3c8c1e47b7e028bdabfb866f7bc63c8e898086.json @@ -0,0 +1,25 @@ +{ + "encoding": "ISO-8859-1", + "headers": { + "Cache-Control": "max-age=1800", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Type": "text/html", + "Date": "Wed, 24 May 2017 04:23:42 GMT", + "Link": "; rel=\"original\", ; rel=\"timemap\"; type=\"application/link-format\", ; rel=\"timegate\", ; rel=\"first memento\"; datetime=\"Wed, 15 Feb 2006 00:34:34 GMT\", ; rel=\"prev memento\"; datetime=\"Fri, 16 Feb 2007 08:38:57 GMT\", ; rel=\"memento\"; datetime=\"Sun, 29 Apr 2007 19:38:49 GMT\", ; rel=\"next memento\"; datetime=\"Tue, 11 Dec 2007 23:44:46 GMT\", ; rel=\"last memento\"; datetime=\"Tue, 16 Aug 2016 04:37:25 GMT\"", + "Memento-Datetime": "Sun, 29 Apr 2007 19:38:49 GMT", + "Server": "Tengine/2.1.0", + "Transfer-Encoding": "chunked", + "X-Archive-Guessed-Content-Type": "text/html", + "X-Archive-Guessed-Encoding": "ASCII", + "X-Archive-Orig-connection": "close", + "X-Archive-Orig-date": "Sun, 29 Apr 2007 19:38:49 GMT", + "X-Archive-Orig-server": "Apache/2.0.52 (Red Hat)", + "X-Archive-Orig-x-powered-by": "PHP/5.2.0", + "X-Archive-Playback": "0", + "X-Page-Cache": "HIT", + "X-location": "All" + }, + "status_code": 200, + "url": "https://web.archive.org/web/20070429193849id_/http://www.londondevelopmentcentre.org/page.php?s=1&p=2462" +} \ No newline at end of file diff --git a/test/testdata/3f1e14ba615b5a9ddd2af44b30e504b8ee288dc7.html b/test/testdata/3f1e14ba615b5a9ddd2af44b30e504b8ee288dc7.html new file mode 100644 index 00000000..928f1252 --- /dev/null +++ b/test/testdata/3f1e14ba615b5a9ddd2af44b30e504b8ee288dc7.html @@ -0,0 +1,57 @@ + Washington Post: Breaking News, World, US, DC News & Analysis - The Washington Post
The Washington Post
Describing a previously undisclosed high-level conversation between Washington and Moscow, John Brennan testified that in a phone conversation with the head of Russia’s domestic spy service he said that “American voters would be outraged by any Russian attempt to interfere in the election.”
President Trump made the appeals to the director of national intelligence and the director of the National Security Agency, after then-FBI Director James B. Comey announced his agency’s probe. Each refused to comply with the request, according to current and former officials.
The director of national intelligence said it would be inappropriate to discuss conversations with the president.
(Video: Victoria Walker, Amber Ferguson/The Post; photo: Getty)
Manchester police named the suspected attacker as 22-year-old Salman Abedi but declined to provide other details. A senior European intelligence official said the attacker was a British citizen of Libyan descent.
Eyewitness accounts, police statements photos and video footage paint a picture of a grisly scene of chaos and gore, in which the glee of music fans — many of them teenagers, some younger still — turned to horror.
Some of these programs — including Medicaid and the modern version of food stamps — provide benefits to up to a fifth of all Americans. The $4.094 trillion budget proposal for the fiscal year that begins in October marks the president's first exercise in spelling out how he wants the government to change.
The Philippine president issued the decree in response to fighting in Marawi City, where militants linked to the Islamic State were battling security forces.
Roger Moore
1927–2017
The British leading man often defined his acting through a raised eyebrow and a quip.
Perspective
What some call “fake news” helped Trump gain the White House. Now it’s getting worse.
The advance is a step toward what industry insiders have long described as the “holy grail” of digital advertising, but it is also likely to renew concerns over whether technology companies know too much about people’s lives.
A great glazed doughnut is best eaten fresh. So here's how to make a perfect batch at home.
  • 1 hour ago
Share news tips with us confidentially

Do you have information the public should know? Here are some ways you can securely send information and documents to Post journalists.

Learn more

Video
Trump's new budget proposal: What's in and what's out
Play Video 2:08
Trump asked intelligence officials to deny connections with Russia
Play Video 1:52
World leaders condemn Manchester attack
Play Video 2:25
At least 22 dead after explosion at Ariana Grande concert in England
Play Video 2:44
DJIA 0.22%
NASDAQ -0.02%
Last Update: 1:38 PM 05/23/2017(DJIA&NASDAQ)
In February 2016, presidential candidate Donald Trump promised $6 million in donations, including $1 million from his own pocket, to charities along his campaign trail. Months later, he had donated far less than he pledged. Post reporter David A. Fahrenthold went in search of the missing money and found a bigger story than he ever expected.
From Our Advertisers
This content is paid for by the advertiser and published by WP BrandStudio. The Washington Post newsroom was not involved in the creation of this content. Learn more about WP BrandStudio.
\ No newline at end of file diff --git a/test/testdata/3f1e14ba615b5a9ddd2af44b30e504b8ee288dc7.json b/test/testdata/3f1e14ba615b5a9ddd2af44b30e504b8ee288dc7.json new file mode 100644 index 00000000..64b44d3a --- /dev/null +++ b/test/testdata/3f1e14ba615b5a9ddd2af44b30e504b8ee288dc7.json @@ -0,0 +1,25 @@ +{ + "encoding": "UTF-8", + "headers": { + "Access-Control-Allow-Origin": "*", + "Age": "44", + "Cache-Control": "s-maxage=120", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "49526", + "Content-Security-Policy": "upgrade-insecure-requests", + "Content-Type": "text/html;charset=UTF-8", + "Date": "Tue, 23 May 2017 17:54:33 GMT", + "PB-PID": "pPcRfs1Puwpjmp", + "PB-RID": "rtxZn328UTjtkq", + "Server": "nginx", + "Set-Cookie": "de=;Expires=Thursday, 23-May-2019 17:55:17 GMT; path=/; domain=.washingtonpost.com, client_region=0;Expires=Tuesday, 23-May-2017 18:05:17 GMT; path=/; domain=.washingtonpost.com, X-WP-Split=X;Expires=Thursday, 01-January-1970 00:00:00 GMT; path=/; domain=.washingtonpost.com, devicetype=0;Expires=Friday, 23-June-2017 04:24:17 GMT; path=/; domain=.washingtonpost.com, osfam=0;Expires=Friday, 23-June-2017 04:24:17 GMT; path=/; domain=.washingtonpost.com, rpld1=0:wmflabs.org|20:usa|21:ca|22:san francisco|23:37.785591|24:-122.435661|;Expires=Tuesday, 23-May-2017 18:55:17 GMT; path=/; domain=.washingtonpost.com", + "X-Backend": "http://pagebuilder-app.wpprivate.com", + "X-Instart-Debug-Header": "auth_status:200, origin:origin-web.washingtonpost.com, cache key modifier:, num_auth_cookies:6", + "X-Instart-Request-ID": "9459106932934454950:VNQ01-NPPRY42:1495562117:165", + "X-Served-By": "pb", + "x-instart-cache-id": "29:620706499373658602::1495562073" + }, + "status_code": 200, + "url": "https://www.washingtonpost.com/" +} \ No newline at end of file diff --git a/test/testdata/4308ae5993b46b0550ec84cd9940a0bfbd9c896a.html b/test/testdata/4308ae5993b46b0550ec84cd9940a0bfbd9c896a.html new file mode 100644 index 00000000..599df015 --- /dev/null +++ b/test/testdata/4308ae5993b46b0550ec84cd9940a0bfbd9c896a.html @@ -0,0 +1,681 @@ + + + + + + راز-گل-سرخ:-نقد-و-گزیده-شعرهای-سهراب-سپهری | نگاه | خانه کتاب و ادبیات ایران + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + +
+
+
+
+ + + + + + + +
+ + + + + +
+ + + ورود + + ثبت نام + + + + + + +
+
+
+
+ + +
+ + + + +
+
+
+
+
+ راز گل سرخ: نقد و گزیده شعرهای سهراب سپهری | خانه کتاب و ادبیات ایران +
+
+
+ صفحات اولیه کتاب +

+ راز گل سرخ: نقد و گزیده شعرهای سهراب سپهری

+

+ + + شعر فارسی - قرن 14 + + + شعر فارسی - قرن 14 - تاریخ و نقد + + + شاعران ایرانی - سرگذشتنامه + + + سپهری، سهراب، 1307 - 1359 - سرگذشتنامه + + +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
پدیدآور + + به‌اهتمام : + + معصومی ، سحر + + +
ناشر + + + + نگاه + + (برای تماس با ناشر و خرید کتاب کلیک کنید) + + + + +
شابک978-964-6736-34-4
تاریخ نشر + +13861102 +
قیمت +32,000
کد دیویی8fa1.62‌
زبان کتابفارسی
محل نشرتهران - تهران
توضیحات + جلد - + 240 صفحه - + تالیف - + چاپ 6 +
+
+
+
+
+
+
+
+
+
معرفی مختصر کتاب
+

+ +

+
+
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/testdata/4308ae5993b46b0550ec84cd9940a0bfbd9c896a.json b/test/testdata/4308ae5993b46b0550ec84cd9940a0bfbd9c896a.json new file mode 100644 index 00000000..898f4bda --- /dev/null +++ b/test/testdata/4308ae5993b46b0550ec84cd9940a0bfbd9c896a.json @@ -0,0 +1,22 @@ +{ + "encoding": "utf-8", + "headers": { + "AR-ATIME": "0.323", + "AR-CACHE": "BYPASS", + "AR-PoweredBy": "Arvan Cloud (arvancloud.com)", + "AR-Request-ID": "1c2e756f40db42c8790de8b390027d91", + "AR-SID": "2012", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Security-Policy": "upgrade-insecure-requests", + "Content-Type": "text/html; charset=utf-8", + "Date": "Sat, 27 Aug 2022 23:42:44 GMT", + "Keep-Alive": "timeout=65", + "Server": "ArvanCloud", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding, Accept-Encoding", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://ketab.ir/book/d2515c41-fd82-41f3-bfad-9262d3b43a3d" +} \ No newline at end of file diff --git a/test/testdata/457ad9f6675d5581bea5d37077b53516388b452c.html b/test/testdata/457ad9f6675d5581bea5d37077b53516388b452c.html new file mode 100644 index 00000000..e5c4732e --- /dev/null +++ b/test/testdata/457ad9f6675d5581bea5d37077b53516388b452c.html @@ -0,0 +1 @@ +{"result":{"total":1531275,"groups":{"author":{"total":24,"items":[{"entity_type":"Author","author_title":"هشت ، بورکهارد","id":"Author-173267","url":"3f8685d8-37bb-4a21-a2b0-f6ffb302bb57"}]},"printableBook":{"total":1531240,"items":[{"book_subject":null,"book_parent_subject":["ادبیات"],"image":"https://pic.ketab.ir/DataBase/BookImages/96/96522287.jpg","book_print_version":3,"book_cover_price":0,"book_author":["خاقانی ، بدیل‌بن‌علی"],"book_page_count":836,"url":"d3c92d14-e702-45fa-b44a-83092702ddf1","entity_type":"PrintableBook","book_title":"دیوان خاقانی شروانی","book_issue_year":1396,"id":"Book-2151489","book_cover_type":"گالینگور","book_publisher":"نگاه","book_volume_number":0}]},"publisher":{"total":11,"items":[{"image":"https://pic.ketab.ir/DataBase/Publishers/Arms/8225.jpg","entity_type":"Publisher","publisher_manager_fullname":" ","publisher_title":"پنجاه و نه","id":"Publisher-8225","url":"8d29e008-fc3d-41ee-873c-c6d8c07a74b4"}]}},"from":0},"facets":{"book_issue_year":{"1396":97900,"1385":51817,"1395":88789,"1384":51098,"1394":81050,"1383":40142,"1393":73236,"1382":35530,"1392":65745,"1391":63237,"1390":67754,"1401":38907,"1400":110315,"1389":64275,"1399":93748,"1388":59910,"1398":104028,"1387":56105,"1397":99596,"1386":55509},"book_parent_subject":{"آموزشی":139277,"فلسفه":65715,"ادبیات":251354,"کودک":219033,"کمک درسی":137898,"علوم طبیعی و ریاضیات":23815,"هنر":45309,"کمک درسی کودک":31623,"علوم اجتماعی":140913,"دانشگاهی":2,"دین":206781,"تاریخ و جغرافیا":67458,"علوم عملی":173605,"کلیات":2,"زبان":28455},"book_print_version_type":{"چاپ مجدد":673356,"چاپ اول":857884},"book_publisher":{"سمت":12585,"نشر نی":5692,"دانشگاه پیام نور":10679,"پرتقال(وابسته به موسسه انتشاراتی سرزمین بچه های خوشحال)":5849,"بین المللی گاج":17440,"موسسه فرهنگی مدرسه برهان":11064,"نشر چشمه":7743,"آیندگان هزار نکته وابسته به موسسه فرهنگی آیندگان هزار نکته":7281,"آموزشی تالیفی ارشدان":5816,"خیلی سبز":15916,"موسسه بوستان کتاب":6567,"قدیانی":13896,"مبتکران":14666,"شرکت انتشارات کانون فرهنگی آموزش":18792,"شرکت نشر قطره":6203,"مدرسان شریف":14274,"شرکت انتشارات سوره مهر":7121,"امیرکبیر":6473,"موسسه چاپ و انتشارات دانشگاه تهران":5727,"افق":8746},"book_author":{"احمدی‌جزی ، کامران":1039,"تریسی ، برایان":2179,"نصری ، کمیل":1197,"صادقی ، داریوش":1161,"بازرگانی ، بهمن":1803,"هیات ‌مولفان":1928,"شعبانی ، اسدالله":1205,"فتاحی ، حسین":1767,"مطهری ، مرتضی":3596,"طباطبایی ، سیدمحمدحسین":1181,"شیخی ، مژگان":1048,"قاسم‌نیا ، شکوه":2245,"سبحانی‌تبریزی ، جعفر":1040,"مولوی ، جلال‌الدین‌محمدبن‌محمد":1804,"هیات مولفان کانون فرهنگی اموزش":1645,"کوییلو ، پایولو":1156,"سپهری ، نیما":1033,"دپارتمان ا‌یندگان":2407,"عمیق ، مجید":1072,"رحماندوست ، مصطفی":1013,"موحدی ، محمود":1698,"کشاورز ، ناصر":2219,"مکارم‌شیرازی ، ناصر":2098,"ال‌احمد ، جلال":1667,"حافظ ، شمس‌الدین‌محمد":2624,"حیدری‌ابهری ، غلامرضا":1098,"حامی ، فرهاد":1151,"قمی ، عباس":5903,"قرایتی ، محسن":1637,"کیانی ، مصطفی":1110,"مجلسی ، محمدباقربن‌محمدتقی":1511,"قراچه‌داغی ، مهدی":2287,"عبدالمحمدی ، علیرضا":1146,"جوادی‌املی ، عبدالله":1359,"وحیدی‌صدر ، مهدی":1081,"اخلاصمندمنفرد ، علیرضا":1698,"نامی ، حسین":1435,"الهی‌قمشه‌ای ، مهدی":7675,"گروه مولفان":1728,"نجف‌خانی ، محبوبه":1227,"گراس ، تونی":1329,"فردوسی ، ابوالقاسم":1391,"استاین ، ار.ال.":1225,"سیاری ، مجید":1127,"سعدی ، مصلح‌بن‌عبدالله":1921,"طالب‌تبار ، حمید":1051,"اناری ، شهاب":1380,"اعضای هیات علمی سنجش تکمیلی":1332,"محمدی‌ری‌شهری ، محمد":1270,"میرزایی‌دلاویز ، محمود":1126},"listModel":{"book_issue_year":[{"label":"1396","value":97900},{"label":"1385","value":51817},{"label":"1395","value":88789},{"label":"1384","value":51098},{"label":"1394","value":81050},{"label":"1383","value":40142},{"label":"1393","value":73236},{"label":"1382","value":35530},{"label":"1392","value":65745},{"label":"1391","value":63237},{"label":"1390","value":67754},{"label":"1401","value":38907},{"label":"1400","value":110315},{"label":"1389","value":64275},{"label":"1399","value":93748},{"label":"1388","value":59910},{"label":"1398","value":104028},{"label":"1387","value":56105},{"label":"1397","value":99596},{"label":"1386","value":55509}],"book_print_version_type":[{"label":"چاپ مجدد","value":673356},{"label":"چاپ اول","value":857884}],"book_parent_subject":[{"label":"آموزشی","value":139277},{"label":"فلسفه","value":65715},{"label":"ادبیات","value":251354},{"label":"کودک","value":219033},{"label":"کمک درسی","value":137898},{"label":"علوم طبیعی و ریاضیات","value":23815},{"label":"هنر","value":45309},{"label":"کمک درسی کودک","value":31623},{"label":"علوم اجتماعی","value":140913},{"label":"دانشگاهی","value":2},{"label":"دین","value":206781},{"label":"تاریخ و جغرافیا","value":67458},{"label":"علوم عملی","value":173605},{"label":"کلیات","value":2},{"label":"زبان","value":28455}],"book_publisher":[{"label":"سمت","value":12585},{"label":"نشر نی","value":5692},{"label":"دانشگاه پیام نور","value":10679},{"label":"پرتقال(وابسته به موسسه انتشاراتی سرزمین بچه های خوشحال)","value":5849},{"label":"بین المللی گاج","value":17440},{"label":"موسسه فرهنگی مدرسه برهان","value":11064},{"label":"نشر چشمه","value":7743},{"label":"آیندگان هزار نکته وابسته به موسسه فرهنگی آیندگان هزار نکته","value":7281},{"label":"آموزشی تالیفی ارشدان","value":5816},{"label":"خیلی سبز","value":15916},{"label":"موسسه بوستان کتاب","value":6567},{"label":"قدیانی","value":13896},{"label":"مبتکران","value":14666},{"label":"شرکت انتشارات کانون فرهنگی آموزش","value":18792},{"label":"شرکت نشر قطره","value":6203},{"label":"مدرسان شریف","value":14274},{"label":"شرکت انتشارات سوره مهر","value":7121},{"label":"امیرکبیر","value":6473},{"label":"موسسه چاپ و انتشارات دانشگاه تهران","value":5727},{"label":"افق","value":8746}],"book_author":[{"label":"احمدی‌جزی ، کامران","value":1039},{"label":"تریسی ، برایان","value":2179},{"label":"نصری ، کمیل","value":1197},{"label":"صادقی ، داریوش","value":1161},{"label":"بازرگانی ، بهمن","value":1803},{"label":"هیات ‌مولفان","value":1928},{"label":"شعبانی ، اسدالله","value":1205},{"label":"فتاحی ، حسین","value":1767},{"label":"مطهری ، مرتضی","value":3596},{"label":"طباطبایی ، سیدمحمدحسین","value":1181},{"label":"شیخی ، مژگان","value":1048},{"label":"قاسم‌نیا ، شکوه","value":2245},{"label":"سبحانی‌تبریزی ، جعفر","value":1040},{"label":"مولوی ، جلال‌الدین‌محمدبن‌محمد","value":1804},{"label":"هیات مولفان کانون فرهنگی اموزش","value":1645},{"label":"کوییلو ، پایولو","value":1156},{"label":"سپهری ، نیما","value":1033},{"label":"دپارتمان ا‌یندگان","value":2407},{"label":"عمیق ، مجید","value":1072},{"label":"رحماندوست ، مصطفی","value":1013},{"label":"موحدی ، محمود","value":1698},{"label":"کشاورز ، ناصر","value":2219},{"label":"مکارم‌شیرازی ، ناصر","value":2098},{"label":"ال‌احمد ، جلال","value":1667},{"label":"حافظ ، شمس‌الدین‌محمد","value":2624},{"label":"حیدری‌ابهری ، غلامرضا","value":1098},{"label":"حامی ، فرهاد","value":1151},{"label":"قمی ، عباس","value":5903},{"label":"قرایتی ، محسن","value":1637},{"label":"کیانی ، مصطفی","value":1110},{"label":"مجلسی ، محمدباقربن‌محمدتقی","value":1511},{"label":"قراچه‌داغی ، مهدی","value":2287},{"label":"عبدالمحمدی ، علیرضا","value":1146},{"label":"جوادی‌املی ، عبدالله","value":1359},{"label":"وحیدی‌صدر ، مهدی","value":1081},{"label":"اخلاصمندمنفرد ، علیرضا","value":1698},{"label":"نامی ، حسین","value":1435},{"label":"الهی‌قمشه‌ای ، مهدی","value":7675},{"label":"گروه مولفان","value":1728},{"label":"نجف‌خانی ، محبوبه","value":1227},{"label":"گراس ، تونی","value":1329},{"label":"فردوسی ، ابوالقاسم","value":1391},{"label":"استاین ، ار.ال.","value":1225},{"label":"سیاری ، مجید","value":1127},{"label":"سعدی ، مصلح‌بن‌عبدالله","value":1921},{"label":"طالب‌تبار ، حمید","value":1051},{"label":"اناری ، شهاب","value":1380},{"label":"اعضای هیات علمی سنجش تکمیلی","value":1332},{"label":"محمدی‌ری‌شهری ، محمد","value":1270},{"label":"میرزایی‌دلاویز ، محمود","value":1126}]}},"spelling":null} \ No newline at end of file diff --git a/test/testdata/457ad9f6675d5581bea5d37077b53516388b452c.json b/test/testdata/457ad9f6675d5581bea5d37077b53516388b452c.json new file mode 100644 index 00000000..2e25cf90 --- /dev/null +++ b/test/testdata/457ad9f6675d5581bea5d37077b53516388b452c.json @@ -0,0 +1,21 @@ +{ + "encoding": "utf-8", + "headers": { + "AR-ATIME": "3.916", + "AR-CACHE": "BYPASS", + "AR-PoweredBy": "Arvan Cloud (arvancloud.com)", + "AR-Request-ID": "af6c4392d3ae66747c7a5ffc3548260f", + "AR-SID": "2012", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Type": "application/json; charset=utf-8", + "Date": "Sat, 27 Aug 2022 23:41:52 GMT", + "Keep-Alive": "timeout=65", + "Server": "ArvanCloud", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding, Accept-Encoding", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://msapi.ketab.ir/search/?query=978-964-6736-71-9&limit=1" +} \ No newline at end of file diff --git a/test/testdata/461414338328cffba0654600ebd1efe3beec0706.html b/test/testdata/461414338328cffba0654600ebd1efe3beec0706.html new file mode 100644 index 00000000..19240a76 --- /dev/null +++ b/test/testdata/461414338328cffba0654600ebd1efe3beec0706.html @@ -0,0 +1,346 @@ + + + + + + +نهاد دایگانی در دورۀ ساسانیان + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + +
+
+
+
+ +
+ + + + + +
+ +
+ +
+
+
+
+
+ + + + +
+ +
+ + + diff --git a/test/testdata/461414338328cffba0654600ebd1efe3beec0706.json b/test/testdata/461414338328cffba0654600ebd1efe3beec0706.json new file mode 100644 index 00000000..50b8bb4f --- /dev/null +++ b/test/testdata/461414338328cffba0654600ebd1efe3beec0706.json @@ -0,0 +1,21 @@ +{ + "encoding": "ISO-8859-1", + "headers": { + "Cache-Control": "no-store, no-cache, must-revalidate, post-check=0, pre-check=0", + "Connection": "close", + "Content-Encoding": "gzip", + "Content-Length": "6967", + "Content-Type": "text/html", + "Date": "Sun, 28 May 2017 05:54:52 GMT", + "Expires": "Thu, 19 Nov 1981 08:52:00 GMT", + "Pragma": "no-cache", + "Server": "Apache/2.4.7 (Ubuntu)", + "Set-Cookie": "juFirstLang=fa; expires=Tue, 27-Jun-2017 05:46:07 GMT; Max-Age=2592000; path=/; httponly, juSecondLang=en; expires=Tue, 27-Jun-2017 05:46:07 GMT; Max-Age=2592000; path=/; httponly, PHPSESSID=l8rn1s0k503pouqv7gdj4snh13; path=/; HttpOnly", + "Vary": "Accept-Encoding", + "X-Cache": "MISS from google.com", + "X-Cache-Lookup": "MISS from google.com:86", + "X-Powered-By": "PHP/5.5.9-1ubuntu4.5" + }, + "status_code": 200, + "url": "http://socialhistory.ihcs.ac.ir/article_319_84.html" +} \ No newline at end of file diff --git a/test/testdata/46b02c583dae5038c3d7690809de7be6037287e5.html b/test/testdata/46b02c583dae5038c3d7690809de7be6037287e5.html new file mode 100644 index 00000000..b84ad077 --- /dev/null +++ b/test/testdata/46b02c583dae5038c3d7690809de7be6037287e5.html @@ -0,0 +1,9 @@ +TY - BOOK +T1 - New Approach to Legal Translation +A1 - Sarcevic, S. +A1 - Šar?evi?, S. +SN - 9789041104014 +UR - https://books.google.com/books?id=i8nZjjo_9ikC +Y1 - 1997 +PB - Springer Netherlands +ER - diff --git a/test/testdata/46b02c583dae5038c3d7690809de7be6037287e5.json b/test/testdata/46b02c583dae5038c3d7690809de7be6037287e5.json new file mode 100644 index 00000000..3434e5be --- /dev/null +++ b/test/testdata/46b02c583dae5038c3d7690809de7be6037287e5.json @@ -0,0 +1,21 @@ +{ + "encoding": null, + "headers": { + "Alt-Svc": "h3-29=\":443\"; ma=2592000,h3-27=\":443\"; ma=2592000,h3-25=\":443\"; ma=2592000,h3-T050=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"", + "Cache-Control": "private, max-age=0", + "Content-Disposition": "attachment; filename=New_Approach_to_Legal_Translation.ris", + "Content-Length": "217", + "Content-Type": "application/x-research-info-systems", + "Date": "Sat, 11 Jul 2020 09:16:24 GMT", + "Expires": "Sat, 11 Jul 2020 09:16:24 GMT", + "P3P": "CP=\"This is not a P3P policy! See g.co/p3phelp for more info.\"", + "Server": "OFE/0.1", + "Set-Cookie": "NID=204=0P8ngW5E3G8OddbHI0EBVu1KXS5Z1qDQOGB2NndolWNPgbr7c15fm0KeGehPXABexZ3uNpYNfHDkshYXBF0XM9TxnRH0RN7q8BUW5gk7fbrOvlLIhHutzR7xhcSR8ptyrYtMyxwwaQaji4Bvrve7kyoHry5_CiA1vC_szh48qPM; expires=Sun, 10-Jan-2021 09:16:24 GMT; path=/; domain=.google.com; Secure; HttpOnly; SameSite=none", + "Strict-Transport-Security": "max-age=604800", + "X-Content-Type-Options": "nosniff", + "X-Frame-Options": "SAMEORIGIN", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://books.google.com/books/download/?id=i8nZjjo_9ikC&output=ris" +} \ No newline at end of file diff --git a/test/testdata/49a7146ea16042067a505241b1f2e4f7f7f2f1ca.html b/test/testdata/49a7146ea16042067a505241b1f2e4f7f7f2f1ca.html new file mode 100644 index 00000000..310e1782 --- /dev/null +++ b/test/testdata/49a7146ea16042067a505241b1f2e4f7f7f2f1ca.html @@ -0,0 +1,6615 @@ + + + + + + + + + + + + + HuffPost Canada - Canadian News Stories, Breaking News, Opinion + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ + + + + + + + +
+ + + + +
+ + + + +
+ + + +
+ +
+
+ + + +
+ + + + + + + + +
+ + + +
+
+ + +
+ + +
+ + +
+
+ + + + +
+
+ +
+ + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + +
+
+ + +
+ +
+
+ + + +
+ +
+
+
+
+
+ +
+ +
+ + +
+ +
+
+
+

What Is Appropriate And What Is Cultural Appropriation?

+
+ +
+ + +
+ + Indigenous Pow Wow Canada + + NurPhoto via Getty Images +
+ + +

For numbers of year now, there has been a movement that seeks to "indigenize" education in Canada. This means that our institutions will have to create an appropriate curriculum for non-indigenous and indigenous educators alike to deliver to a very diverse student body. Can this be done? If so, who will get to say what is appropriate and what is appropriation?

+ + + +
+
+
+ +
+ + + + + + + + + +
+ + +
+
+ + + +
+ + + +
+ + +
+
+ + + + + + + + + + + + + +
+ + +

Separatism And Scandal: Bernier's Unlikely Road To Redemption

Maxime Bernier
Sean Kilpatrick/CP
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

Summer Temperatures Will Feel Like A Teeter-Totter: Meteorologist

Seesaw
Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

These Are The Lives Cut Tragically Short In Manchester

Athena Image
Twitter/DarrenDuffy8
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

Environment Canada Reminds Albertans Of Fleeting Joy Of Patio Season

Alberta Lightning
Nigel Midwinter via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

Trump: Israel And Neighbours Can Unite On 'Common Cause' Of Iran

Donald Trump Israel Iran
JACK GUEZ via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

Trump 'Fireworks' Expected At NATO, G7 Summits

Donald Trump
Evan Vucci/AP via CP
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

30 Somethings Are Earning More Than Their Parents Did: StatsCan

Working Woman
gradyreese via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

2 Shot To Death In Calgary Superstore Parking Lot

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

Toronto Police Invited To March In Uniform At NYC Pride

Toronto Police Pride
Roberto Machado Noa via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

Canadian Accused Of Opening Plane Door Had Taken Cocaine: Lawyer

Air Canada Plane
NurPhoto via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

Largest Privacy Breach In CRA History Was The Work Of An Employee

Canada Revenue Agency
CP
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

Several Killed After Explosion At Ariana Grande Concert In U.K.

Manchester Arena Explosion
Dave Thompson via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

The World Is Looking For Canada To Tout Tolerance: Freeland

Chrystia Freeland
Todd Korol/CP
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

Foul Play Not Suspected In 8-Year-Old's Provincial Park Death

Ontario Provincial Police
Raymond Boyd via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

Donald Trump Softens Rhetoric On Islam

Diplomacy Horizontal
MANDEL NGAN via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

TPP To Go Ahead Without U.S.

Francoisphilippe Champagne
POOL via Getty Images

More Business
Best Jobs In Canada.. Gin Recall.. Rent Hike Revolt.. Sad State Of Canada's Minimum Wages

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

Mom Allegedly Beat Daughters For Only Getting A Card On Mother's Day

Handcuffs
Bill Oxford via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

Mentally Ill B.C. Dad Who Killed His 3 Kids Up For Review

Allan Schoenborn
BC RCMP

More British Columbia
U2 Kicks Off Tour In Vancouver.. Sea Lion Found Shot In The Face.. Oil Tanker Bill

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

Nunavut's Food Insecurity Alarmingly Worse Than Rest Of Canada: Report

Nunavut Food Prices
CP
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

Report: Big Cuts To Medicaid Coming In Trump Budget This Week

Athena Image
Kevin Lamarque / Reuters
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

McMaster Can't Remember If Trump Called Comey A ‘Nut Job’ In Meeting With Russians

Horizontal International Landmark James Brady Pres
NurPhoto via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

Healthy Pets Sometimes Euthanized For Owner's Convenience: Vets

Veterinarian
Jeng_Niamwhan via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

Twitter Co-Founder Sorry For Social Media's Role In Trump's Rise

Evan Williams Twitter
Bloomberg via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

Top Oversight Dem: ‘I Want Every Note’ White House Has On Trump’s Meeting With Russians

Russia Elijah Cummings Trump Comey
Anadolu Agency via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

UPDATED
Suspect Charged In Calgary Sexual Assault Of 5-Year-Old

Sexual Assault Suspect
Calgary Police Service
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

Chief's Walk Will Counter Canada's 150th

Derek Nepinak
John Woods/CP
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

Rape Victim Castrates Her Alleged Attacker In India

Thiruvananthapuram
Google Maps
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

Canadian Woman In Very Critical Condition After Times Square Attack

Times Square Attack
EDUARDO MUNOZ ALVAREZ via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +

Gas Prices Up, Food Prices Down In Canada

Supermarket
Dan Dalton via Getty Images

More Business
Best Jobs In Canada.. Gin Recall.. Rent Hike Revolt.. Sad State Of Canada's Minimum Wages

+ + + + + + + +
+ + + + + + + + + + + + + + + +
+
+
+ + + + +
+ + + +
+ +
+ + +
+ +
+
+

FOLLOW HUFFPOST

+
+
+ +
    + + +
  • +
      +
    1. + HuffPost + + +
    2. +
    +
  • + + + +
  • +
      + + +
    1. + HuffPost + +
    2. +
    +
  • + +
  • +
      +
    1. + + HuffPost +
    2. +
    3. + View all RSS feeds +
    4. +
    +
  • + +
  • +
      +
    1. + +
    2. +
    3. + +
    4. +
    5. + +
    6. +
    +
  • + +
+
+ +
+ + +
+ + + +
+ + + + +
+ + + +
+
+ + + + + + + + + + + + + + + + +
+ + +

Meghan Markle And Prince Harry Were Kept Apart At Pippa Middleton's Reception

Prince Harry Meghan Markle
Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ + +

James Corden Sends Emotional Message To Manchester Families

James Corden Manchester Attack
Andrew Kelly / Reuters
+ + + + + + + +
+ + + + + + + + + + + + + +
+
+ + + + + + + + + + +
+ + +

Queen Expresses 'Deepest Sympathy' To Victims Of Manchester Attack

Queen Elizabeth
Samir Hussein via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ + +

This Quality May Be More Important Than Confidence At Work

Intelligent Woman
David Lees via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ + +

Ariana Grande 'Broken' After Manchester Arena Attack

Ariana Grande
C Flanigan via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ + +

Celebrities Share Their Grief Over Manchester Attack

Ariana Grande
Taylor Hill via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ + +

Shawn Mendes, Drake And Other Young Canadian Stars Grieve For Manchester

Shawn Mendes Drake
Kevin Mazur/AMA2016 via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ + +

James Bond Star Roger Moore Dies Aged 89

Roger Moore
Sony
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ + +

#RoomForManchester Emerges On Twitter To Help Stranded Concert-Goers

Room For Manchester
Dave Thompson via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ + +

Watch Melania Trump Swat Donald Away When He Tries To Hold Her Hand

Athena Image
Jonathan Ernst / Reuters
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ + +

Trudeau Photobombs A Prom Picture, Which Makes Him King

Trudeau
Adam Scotti
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ + +

Drake Bows Down To Canada’s Honorary Queen, Céline Dion

Athena Image
Kevin Mazur/BBMA2017 via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ + +

The Obamas Are Living Their Best Lives As Tourists In Italy

Athena Image
Toscana Photos/BACKGRID USA
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ + +

The Billboard Awards' Red Carpet Was All Over The Place

Billboard Awards 2017 Red Carpet
Getty
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ + +

Hilarious Video Basically Nails What It's Like To Live With A Toddler

Bingham
This Is How We Bingham
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ + +

5 TV Shows For Kids You Didn't Know Were Canadian

Inspector Gadget
Nelvana
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ + +

It's Celine Dion's World, And We're All Just Living In It

Celine Dion Billboard Awards
John Shearer/BBMA2017 via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ + +

How To Be Kind (When Everyone Else Is A Jerk)

Smiling Woman
Todor Tsvetkov via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ + +

Sorry Beyoncé And Adele, Drake Is The New Billboard Awards Champion

Drake Billboard
Michael Tran via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ + +

Judge Confirms Prince's 6 Siblings Heirs To His Estate

Prince Singer
Chris Pizzello / Reuters
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ + +

A Relationship Counsellor Told Us How To Have A Better Breakup

Perfect Breakup
Wavebreakmedia Ltd via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ + +

Photos Galore Of Pippa Middleton's Wedding!

Pippa Middleton Wedding
JUSTIN TALLIS via Getty Images
+ + + + + + + +
+ + + + + + + + + + + + + + + +
+
+ +
+ + +
+
+ + + +
+ +
+ + +
+ +
+ + + + + +
+
+
+ +
+
+
+
+
+
+ +
+
+ + + +
Today's Videos
+ + + + + + +
+
+ +
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + diff --git a/test/testdata/49a7146ea16042067a505241b1f2e4f7f7f2f1ca.json b/test/testdata/49a7146ea16042067a505241b1f2e4f7f7f2f1ca.json new file mode 100644 index 00000000..e56d814f --- /dev/null +++ b/test/testdata/49a7146ea16042067a505241b1f2e4f7f7f2f1ca.json @@ -0,0 +1,18 @@ +{ + "encoding": "utf-8", + "headers": { + "Cache-Control": "max-age=30", + "Content-Encoding": "gzip", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:53:22 GMT", + "Expires": "Tue, 23 May 2017 17:53:52 GMT", + "P3P": "CP='NO P3P'", + "Server": "Apache", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding", + "X-EC-Lua": "19365-geo", + "X-Mobile-URL": "http://m.huffpost.com/ca" + }, + "status_code": 200, + "url": "http://www.huffingtonpost.ca/" +} \ No newline at end of file diff --git a/test/testdata/49b9fb17f48cad493027d03c29940368201af4ef.html b/test/testdata/49b9fb17f48cad493027d03c29940368201af4ef.html new file mode 100644 index 00000000..5d830081 --- /dev/null +++ b/test/testdata/49b9fb17f48cad493027d03c29940368201af4ef.html @@ -0,0 +1,88 @@ +'Star Wars': Disney+ switches up controversial Han Solo/Greedo scene

The infamous 'Han shot first' scene in 'Star Wars' has changed yet again on Disney+

Brian Truitt
USA TODAY

One of the most controversial scenes in the galaxy far, far away just got inexplicably more complicated.

A key early moment in George Lucas' 1977 original "Star Wars" movie features Han Solo gunning down bad guy Greedo in the Mos Eisley cantina before heading off into space with Luke, Leia and the gang to take on the evil Empire. Who shot first and when has been at the center of a slew of changes over four decades that have irked the hardcore "Star Wars" faithful, and an alteration in a new cut streaming on Disney+ muddies up the affair even more.

"Did Han shoot first?" has been debated for years. In the original cut, lovable rogue Han (Harrison Ford) guns down Greedo, but in a 1997 special edition, Lucas edited the movie to make it seem like Greedo was the one who fired first, making Solo look a little more heroic but irking fans in the process. In an interview with The Hollywood Reporter in 2012, Lucas said he wanted to "clean up the confusion. ... Obviously, it upset people because they wanted Solo to be a coldblooded killer, but he actually isn't."

Disney+ review: First 'Star Wars' live-action TV series 'The Mandalorian' doesn't rule the galaxy

More:Here are all the new Disney+ shows and movies, from 'Mandalorian' to 'High School Musical'

Did Han shoot first? An old debate surrounding Han Solo's rogue in the 1977 "Star Wars" gets new life after changes made for a new Disney+ edition.

In 2004, for an updated DVD release of Lucas' first "Star Wars" trilogy, the sequence was changed again to show Greedo firing just a hair before Han. 

Much of the scene is the same in the updated Disney+ version, except Greedo says, "Maclunkey" – we're still translating that one – and he and Han shoot simultaneously. (Luckily for the rest of the movies to follow, Han is still a much better shot.)

The scene still reflects the 2004 version on other streaming platforms, including iTunes.

Lucasfilm confirmed to USA TODAY that the new change in the Disney+ version was made by Lucas before Disney's $4 billion acquisition of his company in 2012.

Social media had a field day with the alteration, which generated just as much conversation as the ballyhooed new "Star Wars" TV series "The Mandalorian." 

New York Times culture reporter Dave Itzkoff used the kerfuffle to make a timely "OK Boomer" joke.

Writer Richard Littler compared it to changing other classic movies, like Michael Corleone playing a kazoo in "The Godfather II." 

And CNN media reporter Frank Pallotta joked that he's looking forward to the next big changes 20 years down the line, "where Han and Greedo hug."

\ No newline at end of file diff --git a/test/testdata/49b9fb17f48cad493027d03c29940368201af4ef.json b/test/testdata/49b9fb17f48cad493027d03c29940368201af4ef.json new file mode 100644 index 00000000..2a74bd90 --- /dev/null +++ b/test/testdata/49b9fb17f48cad493027d03c29940368201af4ef.json @@ -0,0 +1,36 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "1238", + "Cache-Control": "no-store", + "Connection": "keep-alive", + "Content-Encoding": "br", + "Content-Length": "42011", + "Content-Security-Policy": "upgrade-insecure-requests;frame-ancestors 'none';object-src 'none'", + "Content-Security-Policy-Report-Only": "script-src https: blob: 'unsafe-inline' 'unsafe-eval' 'self';base-uri 'self';report-uri https://reporting-api.gannettinnovation.com;report-to default", + "Content-Type": "text/html; charset=utf-8", + "Cross-Origin-Opener-Policy": "same-origin", + "Cross-Origin-Resource-Policy": "same-origin", + "Date": "Tue, 15 Mar 2022 14:57:21 GMT", + "Feature-Policy": "camera 'none';display-capture 'none';geolocation 'none';microphone 'none';payment 'none';usb 'none';xr-spatial-tracking 'none'", + "Gannett-Cam-Experience-Id": "control:5", + "NEL": "{\"report_to\":\"default\",\"max_age\":31557600,\"include_subdomains\":true,\"success_fraction\":0.005}", + "Origin-Agent-Cluster": "?1", + "Permissions-Policy": "camera=(),display-capture=(),geolocation=(),microphone=(),payment=(),usb=(),xr-spatial-tracking=()", + "Referrer-Policy": "strict-origin-when-cross-origin", + "Report-to": "{\"max_age\":31557600,\"include_subdomains\":true,\"endpoints\":[{\"url\":\"https://reporting-api.gannettinnovation.com\"}]}", + "Set-Cookie": "gup_anonid=b64f7501-829e-41ff-81b8-ba60d480814c; Domain=.usatoday.com; Max-Age=31536000; Path=/; SameSite=Lax; Secure, gup_clientid=5a59a0a0-1ec6-4b3e-852f-f774f6cf92d3; Domain=.usatoday.com; Max-Age=31536000; Path=/; SameSite=Lax; Secure, gnt_ub=23; domain=.usatoday.com; path=/; secure; samesite=lax; max-age=31536000;, gnt_sb=5; domain=.usatoday.com; path=/; secure; samesite=lax; max-age=31536000;, gnt_eid=control:5; domain=.usatoday.com; path=/; secure; samesite=lax; max-age=5184000;, gnt_d=%7B%22w%22%3A%7B%22t%22%3A%2258%22%2C%22f%22%3A%221-q1a2z32cb0f2f2%22%2C%22c%22%3A%22Sunny%22%7D%2C%22z%22%3A%2222102%22%2C%22c%22%3A%22McLean%22%2C%22s%22%3A%22VA%22%7D; domain=.usatoday.com; path=/; samesite=lax; secure; priority=high;", + "Strict-Transport-Security": "max-age=63072000", + "Vary": "X-AbVariant,X-AbVCfg,X-AltUrl,Accept-Encoding,User-Agent", + "X-Cache": "MISS, HIT", + "X-Content-Type-Options": "nosniff", + "X-Frame-Options": "deny", + "X-Timer": "S1647356241.162671,VS0,VE2", + "X-XSS-Protection": "1; mode=block", + "etag": "W/\"29401-987fg33/YeSOUqwIV3yG/+bU/5g\"", + "link": ";rel=preload;as=image;nopush" + }, + "status_code": 200, + "url": "https://www.usatoday.com/story/entertainment/movies/2019/11/12/star-wars-disney-plus-changes-controversial-han-solo-greedo-scene/2576097001/" +} \ No newline at end of file diff --git a/test/testdata/4af5128487af9dc53d43a38b74a65ac4bd6eac0a.html b/test/testdata/4af5128487af9dc53d43a38b74a65ac4bd6eac0a.html new file mode 100644 index 00000000..e2e09de3 --- /dev/null +++ b/test/testdata/4af5128487af9dc53d43a38b74a65ac4bd6eac0a.html @@ -0,0 +1,1237 @@ + + + + + + + + + + + + + + + + + “Calamidades Meteorológicas no Brasil Meridional, em Agôsto de 1965” on JSTOR + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + +
+ +
+ + + + + + + + + + + +
+
+ + + + + + Have library access? + + Log in through your library + + + + + + + +
+ + +
+ + + + + + + +
+
+ + + + +
+ + + +
+
+ +
+ +
+ + + + + + + + Revista Geográfica + + + + + + +
+ +
+
+ + journal article + +
+ + + “Calamidades Meteorológicas no Brasil Meridional, em Agôsto de 1965” + + +
+ + Carlos Augusto de Figueiredo Monteiro + +
+ + +
+
+
Revista Geográfica
+ +
T. 35, No. 63 (2.o SEMESTRE 1965), pp. 173-178 (8 pages)
+ +
Published By: Pan American Institute of Geography and History
+
+
+ +
+ + + + + + + + Revista Geográfica + + + + + + +
+ +
+
+
+ + + + +
+ https://www.jstor.org/stable/40991855 +
+ +
+ + +
+ + + + + + Cite this Item + + + + + + + + + + + + +
+ + + + + +
+ + + + + + + +
+ +
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + +
+
+ +
+ + +
+
+
+ Preview +
+
+ Preview +
+
+
+ +
+
+
+ + +
+ +
+
+ +
+
+ +
+
+
+ + + + +
+
+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + + + + diff --git a/test/testdata/4af5128487af9dc53d43a38b74a65ac4bd6eac0a.json b/test/testdata/4af5128487af9dc53d43a38b74a65ac4bd6eac0a.json new file mode 100644 index 00000000..1b08960a --- /dev/null +++ b/test/testdata/4af5128487af9dc53d43a38b74a65ac4bd6eac0a.json @@ -0,0 +1,22 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Type": "text/html; charset=utf-8", + "Date": "Fri, 18 Jun 2021 07:00:03 GMT", + "Server": "Apache/2.4.29 (Ubuntu)", + "Set-Cookie": "AccessSession=H4sIAAAAAAAAAK1Sy27bMBC85ysEnUOD5JIUtzfFqJsix-RWFMVqSbYqlMSwpABNkH-vHnQMt82tR87MDoc7fLkoirINZfGhKL0lEzApCLoxjIyVIYfWNckgRR3Ky1nMWd0SCyKSaoV_ZrjW1Ud0NdS425odIuoKUKv6CsBor-2qPmQ1JQkKfONT8MZPtzlUCqyxlWanMF85jlmeApGCBEICs5jCekFGsZBWRzbeeVnpdYTG4cc8kqjr44I8Ube6KKeNVNJIAKcWqt0v7qA2ymyUrTZHk3DfPpy79P3jOTDw-ZmYh36GvhQv0_m0XSWlxMV3wnKU0hyBmAFE0wCgFQnJCoNaCu8pCo0heeU5QaDjzPBrH5ehT4fHcf9m_QZfUd_yGTc9p71vn-Ouo--zZDiMcWJeL_-RFf7Mqv_KGqd9h8giQdUIw5UVxF4LycpP6bmRJP9_1uLrWsSQu4Tpj7lTlx29R4zvEDQMh6WxvIPtdv3It8ck27sFaNoH6sbw7dTi7Up8vruub-pyDnfx-hvJXPziUQMAAA; Path=/; SameSite=Lax; Secure, AccessSessionSignature=0038919d19f718984d1a0332097cdc6f27bbe1cc7e3f51d7092ae2d570fd024c; Path=/; SameSite=Lax; Secure, AccessSessionTimedSignature=18d21b50ab389d17c819444086ee56d4059f9b63f2cf3840fff6735bc65528b5; Path=/; SameSite=Lax; Secure, UUID=fdaa13f3-03cc-4d98-a41c-052ec4868072; expires=Mon, 17 Jun 2024 07:00:03 GMT; Max-Age=94608000; Path=/; SameSite=None; Secure, csrftoken=4WAqVn3Y2fEnlzkb9Ux3akMvNiPn2clvSbmPc1z2xcqr4ckDtUE7WBJrDvhtpGEV; expires=Fri, 17 Jun 2022 07:00:03 GMT; Max-Age=31449600; Path=/; SameSite=Lax; Secure, ReferringRequestId=excelsior:76b2e94015eec5b5b884a16b2d726b40; Path=/; SameSite=Lax; Secure, _pxhd=DpK7p-9NvNIusm2g6WLVg7eHusJqej8ViIjXo0ZMJtvNiKoqmi2hCqvRhlrSBxtXp/xa/kqBc5bPXgv8uaB6Vg==:4sVaYTj-PAd/0nIrD6SnoeZSYdsyHXVHEmHAFjUWgIO-8BP3Nb0jN9RPahpLNupRy-M7DzjEwrB-J40mp3S1iMKyksFTDe1dXeSmV7afOfU=; Expires=Fri, 01 Jan 2021 00:00:00 GMT; path=/;", + "Vary": "Cookie,Accept-Encoding,Fastly-SSL,Origin,X-Requested-Host", + "Via": "1.1 varnish", + "X-Cache": "MISS", + "X-Cache-Hits": "0", + "X-Frame-Options": "SAMEORIGIN", + "X-JSTOR-Restarts": "2", + "X-Served-By": "cache-fra19171-FRA", + "transfer-encoding": "chunked" + }, + "status_code": 200, + "url": "https://www.jstor.org/stable/40991855" +} \ No newline at end of file diff --git a/test/testdata/4bf2a1aafc49525227308aa50e362f6d2cbc8cd3.html b/test/testdata/4bf2a1aafc49525227308aa50e362f6d2cbc8cd3.html new file mode 100644 index 00000000..f72b92bd --- /dev/null +++ b/test/testdata/4bf2a1aafc49525227308aa50e362f6d2cbc8cd3.html @@ -0,0 +1,699 @@ + + + + + + روانشناسی-سلامت-به-ضمیمه-نگرشی-بر-منابع-اسلامی | سمت | خانه کتاب و ادبیات ایران + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + +
+
+
+
+ + + + + + + +
+ + + + + +
+ + + ورود + + ثبت نام + + + + + + +
+
+
+
+ + +
+ + + + +
+
+
+
+
+ روانشناسی سلامت به ضمیمه نگرشی بر منابع اسلامی | خانه کتاب و ادبیات ایران +
+
+
+ صفحات اولیه کتاب +

+ روانشناسی سلامت به ضمیمه نگرشی بر منابع اسلامی

+

+ + + بیماران - روان‌شناسی + + + رفتار بهداشتی + + + پزشکی - خدمات + + + نگرشهای بهداشتی + + +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
پدیدآور + + نويسنده : + + دیماتیو ، ام.رابین + - + + + مترجم : + + کاویانی ، محمد + - + + + زيرنظر : + + هاشمیان ، کیانوش + - + + + ويراستار : + + جباری ، کریم + + +
ناشر + + + + سمت + + (برای تماس با ناشر و خرید کتاب کلیک کنید) + + + + +
شابک978-964-459-398-7
تاریخ نشر + +13790523 +
قیمت +10,000
کد دیویی155.916
زبان کتابفارسی
محل نشرتهران - تهران
توضیحات + جلد 1 - + 422 صفحه - + ترجمه - + چاپ 1 +
+
+
+
+
+
+
+
+
+
معرفی مختصر کتاب
+

+ در این کتاب صور گوناگون پیوند جسم و روان و تاثیر متقابل آنها در سلامت جسم و بیماری بررسی شده است .بخش دیگری از کتاب نیز پیش آگهی و درمان بیماری‌ها مربوط می‌شود . +

+
+
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/testdata/4bf2a1aafc49525227308aa50e362f6d2cbc8cd3.json b/test/testdata/4bf2a1aafc49525227308aa50e362f6d2cbc8cd3.json new file mode 100644 index 00000000..084fb936 --- /dev/null +++ b/test/testdata/4bf2a1aafc49525227308aa50e362f6d2cbc8cd3.json @@ -0,0 +1,22 @@ +{ + "encoding": "utf-8", + "headers": { + "AR-ATIME": "0.107", + "AR-CACHE": "BYPASS", + "AR-PoweredBy": "Arvan Cloud (arvancloud.com)", + "AR-Request-ID": "1be9a14cb99bb763b04e032c1bf60d48", + "AR-SID": "2012", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Security-Policy": "upgrade-insecure-requests", + "Content-Type": "text/html; charset=utf-8", + "Date": "Sat, 27 Aug 2022 23:10:11 GMT", + "Keep-Alive": "timeout=65", + "Server": "ArvanCloud", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding, Accept-Encoding", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://ketab.ir/book/4cc231f9-35c2-4b60-a714-a0a11135e932" +} \ No newline at end of file diff --git a/test/testdata/4c39fe2a62971ec3c359b4969344f4ea223ab299.html b/test/testdata/4c39fe2a62971ec3c359b4969344f4ea223ab299.html new file mode 100644 index 00000000..a6a385eb --- /dev/null +++ b/test/testdata/4c39fe2a62971ec3c359b4969344f4ea223ab299.html @@ -0,0 +1,1136 @@ + + + + + + + 19th-century harpoon gives clue on whales - The New York Times + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + +
+ + +
+
+ + + + +
+ +
+ + + + +
+
+

Biologists, long stumped at figuring out how old whales are, lucked out when a 50-ton bowhead caught off Alaska came with a telltale clue: fragments of a harpoon lodged in a shoulder bone.

The weapon was used more than a century ago by whalers from New Bedford, Massachusetts, enabling researchers to estimate that the whale was at least 115 years old and providing more evidence for their long-held belief that the bowhead whale is one of the longest-living mammals on earth, surviving for up to 200 years.

"It's pretty rare that you get the chance to date the age of a whale," said John Bockstoce, the whaling historian at the New Bedford Whaling Museum who analyzed the fragments.

"We're all finding it very interesting," he said Tuesday.

A biologist in Alaska spotted the pieces of the projectile as they were being pulled from the whale's blubber by Eskimos who had killed the animal last month.

Continue reading the main story +
+
+
+
+
+ +
+
+
+

He sent them to Bockstoce, who identified them as parts of an exploding lance made in New Bedford in the late 1800s, when the city was the world's whaling capital. Hunters would spear the animal with the weapon, which would detonate once inside.

+

Hunters used a similar modern device to kill the whale.

Anthropologists have analyzed hunting devices found in whales before, said Scott Kraus, vice president for research at the New England Aquarium in Boston. It was often difficult, however, to narrow down when the weapon was fired.

+
+

"What you don't know is if some Yankee whaler had a harpoon made in 1830, traded it to an Inuit, and the Inuit or his offspring used it 40 years later," Kraus said.

But because the bomb lance was patented and stocks were used up quickly, Bockstoce and his colleagues identified a narrow window in which they believe the whale was shot, sometime between 1885 and 1895.

Biologists in Alaska will now try to verify the estimate by examining the lens of the whale's eyes. Whales' eyes generally become cloudy as they age.

Found only in Arctic waters, the bowhead was in danger of being hunted to extinction at the turn of the century but bounced back after demand for whalebone corsets plummeted, Bockstoce said. Today, only Alaska's indigenous tribes hunt bowheads.

+

After it is analyzed, the fragment will be displayed at the Inupiat Heritage Center in Barrow, Alaska.

+ Continue reading the main story +
+
+
+
+ + + + + + +
+ + + + + +
+
+
+
+

Go to Home Page »

+

+ Site Index + + The New York Times + +

+ +
+ + + +
+ + +
+
+ + + + + + + + + + + + + + + + diff --git a/test/testdata/4c39fe2a62971ec3c359b4969344f4ea223ab299.json b/test/testdata/4c39fe2a62971ec3c359b4969344f4ea223ab299.json new file mode 100644 index 00000000..5ab7f2f2 --- /dev/null +++ b/test/testdata/4c39fe2a62971ec3c359b4969344f4ea223ab299.json @@ -0,0 +1,27 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes, bytes", + "Cache-Control": "no-cache", + "Connection": "close", + "Content-Encoding": "gzip", + "Content-Type": "text/html; charset=utf-8", + "Cteonnt-Length": "69696", + "Date": "Tue, 23 May 2017 17:53:40 GMT", + "Server": "Apache", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding, Fastly-SSL", + "X-API-Version": "F-5-5", + "X-Age": "0", + "X-Cache": "MISS", + "X-Cache-Hits": "0", + "X-ESI": "1", + "X-Frame-Options": "DENY", + "X-Origin-Time": "2017-05-23 13:53:40 EDT", + "X-PageType": "article", + "X-Served-By": "cache-iad2123-IAD", + "X-Timer": "S1495562021.686309,VS0,VE181" + }, + "status_code": 200, + "url": "http://www.nytimes.com/2007/06/13/world/americas/13iht-whale.1.6123654.html" +} \ No newline at end of file diff --git a/test/testdata/4de792250b41b8acbcb9ad05623e0a70873bbae5.html b/test/testdata/4de792250b41b8acbcb9ad05623e0a70873bbae5.html new file mode 100644 index 00000000..75ad33da --- /dev/null +++ b/test/testdata/4de792250b41b8acbcb9ad05623e0a70873bbae5.html @@ -0,0 +1,1150 @@ + + + + + + + Adding Weight to Suspicion, Sonar Is Linked to Whale Deaths - The New York Times + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + +
+ + +
+
+ + + + +
+ +
+ + + + +
+
+

Scientists have long suspected a link between mass whale strandings and the Navy's use of powerful sonar systems, but the evidence -- dying whales washing ashore when sonar exercises occur -- has been mostly anecdotal.

Now, international researchers have identified a disorder similar to decompression sickness, or the bends, as the cause of at least some whale beachings, and they say military sonar is most likely to blame.

The new findings, being reported today in Nature, are based primarily on necropsies of 10 whales that stranded themselves in the Canary Islands during a 2002 international naval exercise there that included one American ship.

The incident drew worldwide attention, and this year environmentalists in California sued to stop the Navy from developing a newer, more far-reaching sonar system.

Continue reading the main story +
+
+
+
+
+ +
+
+
+

All of the Canary whales examined had widespread bubble formation in tissue and blood vessels, the study says. The same thing occurs in scuba divers who surface too quickly after a deep dive.

+

''The bubbles forming in these animals may not be immediately fatal,'' said Dr. Paul Jepson, a lead author of the study and a researcher at the Zoological Society of London. ''But it does make them distressed or causes impairment, and it's quite logical to conclude that this is what leads them to strand.''

The study challenges the conventional notion that marine mammals cannot suffer from decompression sickness. But more important, says Jean-Michel Cousteau, director of the Ocean Futures Society in California, it demonstrates the toll that underwater noise pollution can have on marine life.

+
+

''A lot of people know the oceans have become a dumping ground for sewage and pollution, but they aren't aware that sonar is also a major issue affecting the quality of marine life,'' Mr. Cousteau said.

The United States Navy, emphasizing that it uses highly trained lookouts and other methods to protect whales, is reluctant to accept the study's conclusions.

+

''Previous studies have not, to date, revealed evidence of decompression sickness as suggested by the article,'' Lt. Cmdr. Cappy Surette, a Navy spokesman, said in an e-mail message. ''The National Oceanic and Atmospheric Administration and the U.S. Navy were not invited to participate in the studies conducted of these beaked whales, and as a result, we are unable to determine the actual cause of the strandings.''

+

It is widely known that breathing compressed air from a scuba tank, even at the depths that recreational divers frequent, causes gases like nitrogen to dissolve and build up in the blood and tissues. If divers have been too deep, or down too long and ascend too quickly, the accumulated nitrogen can turn to bubbles as the pressure decreases.

The bubbles block blood flow and cause tissue damage, which, when severe enough, can be fatal.

How the bends would occur in whales and other marine mammals is not completely understood, Dr. Jepson said. What is known is that the beaked whale and the dolphin species that strand themselves most often when sonar is used nearby tend to have enormous levels of nitrogen in their tissues.

+

One theory holds that high-decibel military sonar can lead to bubble formation by startling the animals into shooting too rapidly from deep to shallow waters. Another suggests that the acoustic signals may somehow directly set off bubble eruptions in the nitrogen-saturated tissues.

Continue reading the main story +
+
+
+
+ + + + + + +
+ + + + + +
+
+
+
+

Go to Home Page »

+

+ Site Index + + The New York Times + +

+ +
+ + + +
+ + +
+
+ + + + + + + + + + + + + + + + diff --git a/test/testdata/4de792250b41b8acbcb9ad05623e0a70873bbae5.json b/test/testdata/4de792250b41b8acbcb9ad05623e0a70873bbae5.json new file mode 100644 index 00000000..6071d5ed --- /dev/null +++ b/test/testdata/4de792250b41b8acbcb9ad05623e0a70873bbae5.json @@ -0,0 +1,27 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes, bytes", + "Cache-Control": "no-cache", + "Connection": "close", + "Content-Encoding": "gzip", + "Content-Type": "text/html; charset=utf-8", + "Cteonnt-Length": "74622", + "Date": "Tue, 23 May 2017 17:53:42 GMT", + "Server": "Apache", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding, Fastly-SSL", + "X-API-Version": "F-5-5", + "X-Age": "0", + "X-Cache": "MISS", + "X-Cache-Hits": "0", + "X-ESI": "1", + "X-Frame-Options": "DENY", + "X-Origin-Time": "2017-05-23 13:53:42 EDT", + "X-PageType": "article", + "X-Served-By": "cache-iad2635-IAD", + "X-Timer": "S1495562023.615402,VS0,VE225" + }, + "status_code": 200, + "url": "http://www.nytimes.com/2003/10/09/us/adding-weight-to-suspicion-sonar-is-linked-to-whale-deaths.html" +} \ No newline at end of file diff --git a/test/testdata/4e6536b6e57bdfc287fba4feffae16b552d4f4d2.html b/test/testdata/4e6536b6e57bdfc287fba4feffae16b552d4f4d2.html new file mode 100644 index 00000000..bb844c1a --- /dev/null +++ b/test/testdata/4e6536b6e57bdfc287fba4feffae16b552d4f4d2.html @@ -0,0 +1,18 @@ +TY - BOOK +T1 - InterACT with Web Standards: A holistic approach to web design +A1 - Anderson, E. +A1 - DeBolt, V. +A1 - Featherstone, D. +A1 - Gunther, L. +A1 - Jacobs, D.R. +A1 - Mills, C. +A1 - Schmitt, C. +A1 - Sims, G. +A1 - Walter, A. +A1 - Jensen-Inman, L. +SN - 9780132704908 +T3 - Voices That Matter +UR - https://books.google.com/books?id=U46IzqYLZvAC +Y1 - 2010 +PB - Pearson Education +ER - diff --git a/test/testdata/4e6536b6e57bdfc287fba4feffae16b552d4f4d2.json b/test/testdata/4e6536b6e57bdfc287fba4feffae16b552d4f4d2.json new file mode 100644 index 00000000..cd3a8e14 --- /dev/null +++ b/test/testdata/4e6536b6e57bdfc287fba4feffae16b552d4f4d2.json @@ -0,0 +1,21 @@ +{ + "encoding": null, + "headers": { + "Alt-Svc": "h3-29=\":443\"; ma=2592000,h3-27=\":443\"; ma=2592000,h3-25=\":443\"; ma=2592000,h3-T050=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"", + "Cache-Control": "private, max-age=0", + "Content-Disposition": "attachment; filename=InterACT_with_Web_Standards.ris", + "Content-Length": "423", + "Content-Type": "application/x-research-info-systems", + "Date": "Sat, 11 Jul 2020 09:16:22 GMT", + "Expires": "Sat, 11 Jul 2020 09:16:22 GMT", + "P3P": "CP=\"This is not a P3P policy! See g.co/p3phelp for more info.\"", + "Server": "OFE/0.1", + "Set-Cookie": "NID=204=JFJcDxX9bEhvmrlahiwvTMYFHDZj1JXAvq3Euiwv95dIK5LbiGs-0UZEJB2_4ORZs-MOmRv20iFbozI4CIH6ZX2n8tLbT2Wt6izFnYlGVWFfAwjQ5iLVRs1IFQB8T352oN-Jme3cqxnrhqZfXpqQtbu7nZanw-bD5XG0wRZ3150; expires=Sun, 10-Jan-2021 09:16:22 GMT; path=/; domain=.google.com; Secure; HttpOnly; SameSite=none", + "Strict-Transport-Security": "max-age=604800", + "X-Content-Type-Options": "nosniff", + "X-Frame-Options": "SAMEORIGIN", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://books.google.com/books/download/?id=U46IzqYLZvAC&output=ris" +} \ No newline at end of file diff --git a/test/testdata/4ef7bafc6f9fb14670ab3808cd3ad9ed49da67c8.html b/test/testdata/4ef7bafc6f9fb14670ab3808cd3ad9ed49da67c8.html new file mode 100644 index 00000000..4af89b44 --- /dev/null +++ b/test/testdata/4ef7bafc6f9fb14670ab3808cd3ad9ed49da67c8.html @@ -0,0 +1,664 @@ + + + + + + + + + + + + + +TG Daily + + + + + + + + + + + + + + + + +
+ +
+
+
+
+ + + + + +
+

What Is Kratom and Is It Dangerous?

+ +

What Is Kratom and Is It Dangerous?

+ + + + +
+
+
+ +
+ +
+
+
+ + + + + +
+

What You Shоuld Know About a Binary Option Robot

+ + + + + +
+
+
+ + + + + +
+

How a Startup Can Use SEO to Raise Brand Awareness

+ + + + + +
+
+
+ + + + + +
+

Company formation agents or DIY- which is better

+ + + + + +
+
+
+
+ +
+ +
+

Picked for you

+
+ + + + + +
+

Should You Expand Your Small Business

+ + + + + +
+
+ +
+ + + + + +
+

3 Important Tips About Motorcycle Safety

+ + + + + +
+
+ +
+ + + + + +
+

4 New Technologies Making Traffic Management Smoother & Safer

+ + + + + +
+
+ +
+ + + + + +
+

Top 5 Beautiful +Places In Sri Lanka

+ + + + + +
+
+
+ +
+ +
+
+ + + + + +
+

Benefits of Online PDF Converter

+ + + + + +
+
+ +
+ + + + + +
+

All you need to know +about Unlocking iPhone 7

+ + + + + +
+
+ +
+ + + + + +
+

7 Essential Things Startups Must Do Before Launching

+ + + + + +
+
+ +
+
+
+ + + + +
+
+
+
+
+ +
+ + + + + + + diff --git a/test/testdata/4ef7bafc6f9fb14670ab3808cd3ad9ed49da67c8.json b/test/testdata/4ef7bafc6f9fb14670ab3808cd3ad9ed49da67c8.json new file mode 100644 index 00000000..77fcd170 --- /dev/null +++ b/test/testdata/4ef7bafc6f9fb14670ab3808cd3ad9ed49da67c8.json @@ -0,0 +1,22 @@ +{ + "encoding": "UTF-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "70", + "Cache-Control": "max-age=60, public", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "8232", + "Content-Type": "text/html; charset=UTF-8", + "Date": "Tue, 23 May 2017 17:55:10 GMT", + "Vary": "Accept-Encoding", + "Via": "1.1 varnish", + "X-Cache": "HIT", + "X-Cache-Hits": "1", + "X-Powered-By": "PHP/5.5.9-1ubuntu4.20", + "X-Served-By": "cache-iad2625-IAD", + "X-Timer": "S1495562110.050546,VS0,VE1" + }, + "status_code": 200, + "url": "http://www.tgdaily.com/" +} \ No newline at end of file diff --git a/test/testdata/54da19032ceea110e3cb60c74b47abeba00e954e.html b/test/testdata/54da19032ceea110e3cb60c74b47abeba00e954e.html new file mode 100644 index 00000000..6adfc8bb --- /dev/null +++ b/test/testdata/54da19032ceea110e3cb60c74b47abeba00e954e.html @@ -0,0 +1,7535 @@ + + +Kim Kardashian's meltdown at nude magazine cover three years before full frontal photoshoot | Daily Mail Online + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
 
+ + + + + + +
+ + + +
+
+ +
+ + + + + +
+
+
+

'You can see nipple!' Kim Kardashian's meltdown at nude magazine cover... three years before full frontal photoshoot

+ +
  • Paper magazine says it was 'Kim's idea to show more than her butt'
+ +

+

Her fully frontal nude photoshoot for Paper magazine has got everyone talking, much to her delight.

But three years ago, Kim Kardashian was a little less eager to bare all.

During a January 2011 episode of Kourtney & Kim Take New York, Kim had a meltdown at the prospect of being seen nude on the cover of W magazine.

Scroll down for video

+ Meltdown: Kim broke down on a 2011 episode of Kourtney & Kim Take New York after seeing  revealing nude photographs of herself in W magazine +
+ + +

Meltdown: Kim broke down on a 2011 episode of Kourtney & Kim Take New York after seeing revealing nude photographs of herself in W magazine

+ Changed your mind Kim? The reality star was so upset by her nude W magazine cover, left, that she had a televised meltdown back in 2011, but happily posed for this Paper magazine cover back in September +
+ + +
+ Changed your mind Kim? The reality star was so upset by her nude W magazine cover, left, that she had a televised meltdown back in 2011, but happily posed for this Paper magazine cover back in September +
+ + +

Changed your mind Kim? The reality star was so upset by her nude W magazine cover, left, that she had a televised meltdown back in 2011, but happily posed for this Paper magazine cover back in September

'Oh my God, I'm more naked than I was in Playboy!' she wept. 'I'm so mad right now. She promised I would be covered with artwork. You can see nipple. The whole concept was sold to me that nothing would be seen.

'This really pisses me off… this is serious porn!' she added at the cover story, which was entitled Kim Kardashian: The Art of Reality.

The cover photo shows Kim nude, but her modesty was protected by bars. Inside shots featured the star nearly completely covered in metallic body paint.  

+
+ +
+ +
+
+ I'm never going naked again: The reality show star  +
+ + +

I'm never going naked again: The reality show star 

+ +
+ + +

Please don't judge me: 'I don’t want people to be like, "All she’s good for is, you know, being naked," she added

Kim told sister Kourtney that the magazine's artist would put 'images of architecture and buildings and stuff on top of me so you will see my body shape and the outline but not actually my boobs or anything.' 

So when showed the rather revealing snapshots, Kim freaked out, and was nearly hysterical when she called her mother Kris Jenner. 

'I feel so taken advantage of,' sobbed Kim, who found fame thanks to a leaked sex tape.

 'I've definitely learned my lesson… I'm never taking my clothes off again, even if it’s for Vogue.

'I don’t want people to be like, "All she’s good for is, you know, being naked," she added.

+ 'I feel so taken advantage of': Kim was eager to move away from the sex tape leaked in 2007  +
+ + +

'I feel so taken advantage of': Kim was eager to move away from the sex tape leaked in 2007 

+ Reassurance: Kris Jenner tried to comfort Kim by telling her the pictures were gorgeous  +
+ + +

Reassurance: Kris Jenner tried to comfort Kim by telling her the pictures were gorgeous 

Attempting to carve out a career as a businesswoman and TV star, Kim was perhaps, at that time, eager to move on from her sex tape which catapulted her to fame in 2007.

Now happily married to rapper Kanye West and having lost more than 50 lbs in baby weight, Kim was clearly keen to bare all, even telling her sisters during a 2013 episode of Keeping Up With The Kardashians: 'As soon as I pop this [baby] out - as soon as I get in shape - the first thing I want to do is Playboy or some nude shoot.

'I just wanna walk down the street fully naked. I'm gonna be the sexy hot mom.'

And it's clear Kim was eager to perform her about-turn in style, appearing full-frontal in the inside pages of Paper magazine, along with the striking front cover which shows her grinning at the camera while baring her ample bare booty covered in baby oil.

Another picture, in the shoot by famed photographer Jean-Paul Goude showed Kim wearing a cocktail dress and recreating Goude's well-known Champagne Incident shot, with bubbly shooting into the air and landing into a glass,perfectly placed on her posterior.

+ Holding nothing back: Kim Kardashian went fully nude in a newly released shot from her shoot for Paper magazine with Jean-Paul Goude (censored by MailOnline) +
+ + +

Holding nothing back: Kim Kardashian went fully nude in a newly released shot from her shoot for Paper magazine with Jean-Paul Goude (censored by MailOnline)

And if anyone wondered if Kim had to be talked into posing for the naked shot, they can think again.

In an interview published on Yahoo Style, Paper's editorial director Mickey Boardman explained that it was all Kim's idea.

'Kim’s attitude was "if we’re gonna do it, let’s really go there,"' he said. 

'And it was her idea to take off her clothes and show more than her butt. But we [Paper] didn’t say "let’s do a cover with your butt hanging out."

'She said she was willing to take her clothes off and one thing lead to another.' 

+ Trending: Kim certainly lit up the internet with comical discussion of her new magazine cover +
+ + +

Trending: Kim certainly lit up the internet with comical discussion of her new magazine cover, available online at Paper.com

 


+
+ + + + + + + +
+
+ + + + + + +
+
+ + +
+ + + + + + + + + + +
+ +
+ + + + + + + + + + + + + + + +
+ +
+ + + + + + + + +
+ +
+ + + +

The comments below have not been moderated.

+ + +
+ +
+ +
+ +

+ The views expressed in the contents above are those of our users and do not necessarily reflect the views of MailOnline. +

+ +
+ + + + + + +

We are no longer accepting comments on this article.

+ + +
+
+
+ + + +
+
+

More top stories

+ +
+ + + +
+ +
+
+ +
+
+
+ Bing +
+ + + + + + + + +
+ +
+ + + + +
+
+ +
+ +
+ +
+ + +
+ + + +
+
+ +   +   +

Femail Today

+ + +
+ +
+ +
+ +
+
+ +   +   +

DON'T MISS

+ + +
+ +
+ +
+
+ +
+ +
+
+ +
+ + + +
+
+ +
+
+ +
+ +
+ +
+ +
+ + +
+ + + +
+ +
+ +
+ +
+ + +
+
+ + + + + + + +
+ +
+
+ +
+
+ + + + + + + +
+ +
+
+
+ + + + + + + + +
+ + + + + + + + + + +
+
+
+
+
+
+ + +
+
+
+
+ + +
+
+
+
+ + + + + + + + + + + + +
+ + + + + + + + + + + + + +
+ + + + + + + + +
 
+ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/testdata/54da19032ceea110e3cb60c74b47abeba00e954e.json b/test/testdata/54da19032ceea110e3cb60c74b47abeba00e954e.json new file mode 100644 index 00000000..ba6c8c36 --- /dev/null +++ b/test/testdata/54da19032ceea110e3cb60c74b47abeba00e954e.json @@ -0,0 +1,21 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "max-age=0, no-cache", + "Connection": "close", + "Content-Encoding": "gzip", + "Content-Type": "text/html;charset=UTF-8", + "Date": "Tue, 23 May 2017 17:53:13 GMT", + "Expires": "Tue, 23 May 2017 17:53:13 GMT", + "Pragma": "no-cache", + "Vary": "User-Agent, Accept-Encoding", + "X-MOL-GEORESP": "us", + "X-rs-ops": "10.250.203.249:6081", + "x-rs-ben": "cljfe-a6:8181", + "x-rs-ctime": "1800", + "x-rs-time": "Tue, 23 May 2017 17-53-13 GMT", + "x-storage": "dmoldarticles" + }, + "status_code": 200, + "url": "http://www.dailymail.co.uk/tvshowbiz/article-2834145/I-m-never-taking-clothes-s-Vogue-Throwback-2011-video-shows-Kim-Kardashian-s-meltdown-nude-magazine-cover.html" +} \ No newline at end of file diff --git a/test/testdata/55117c9cdc8bd6bc9b9557725f3a3cbeee54fd20.html b/test/testdata/55117c9cdc8bd6bc9b9557725f3a3cbeee54fd20.html new file mode 100644 index 00000000..1dfe6f53 --- /dev/null +++ b/test/testdata/55117c9cdc8bd6bc9b9557725f3a3cbeee54fd20.html @@ -0,0 +1 @@ +{"type":"https://mediawiki.org/wiki/HyperSwitch/errors/not_found","title":"Not found.","method":"get","uri":"/en.wikipedia.org/v1/data/citation/mediawiki/964-92962-6-3"} \ No newline at end of file diff --git a/test/testdata/55117c9cdc8bd6bc9b9557725f3a3cbeee54fd20.json b/test/testdata/55117c9cdc8bd6bc9b9557725f3a3cbeee54fd20.json new file mode 100644 index 00000000..520fec41 --- /dev/null +++ b/test/testdata/55117c9cdc8bd6bc9b9557725f3a3cbeee54fd20.json @@ -0,0 +1,36 @@ +{ + "encoding": null, + "headers": { + "Age": "0", + "Connection": "keep-alive", + "NEL": "{ \"report_to\": \"wm_nel\", \"max_age\": 86400, \"failure_fraction\": 0.05, \"success_fraction\": 0.0}", + "Permissions-Policy": "interest-cohort=()", + "Report-To": "{ \"group\": \"wm_nel\", \"max_age\": 86400, \"endpoints\": [{ \"url\": \"https://intake-logging.wikimedia.org/v1/events?stream=w3c.reportingapi.network_error&schema_uri=/w3c/reportingapi/network_error/1.0.0\" }] }", + "Server-Timing": "cache;desc=\"pass\", host;desc=\"cp3060\"", + "Set-Cookie": "WMF-Last-Access=08-Jan-2022;Path=/;HttpOnly;secure;Expires=Wed, 09 Feb 2022 12:00:00 GMT, WMF-Last-Access-Global=08-Jan-2022;Path=/;Domain=.wikipedia.org;HttpOnly;secure;Expires=Wed, 09 Feb 2022 12:00:00 GMT, GeoIP=IR:09:Mashhad:36.30:59.59:v4; Path=/; secure; Domain=.wikipedia.org", + "Strict-Transport-Security": "max-age=106384710; includeSubDomains; preload", + "X-Cache": "cp3064 miss, cp3060 pass", + "X-Cache-Status": "pass", + "X-Client-IP": "31.14.145.3", + "access-control-allow-headers": "accept, content-type, content-length, cache-control, accept-language, api-user-agent, if-match, if-modified-since, if-none-match, dnt, accept-encoding", + "access-control-allow-methods": "GET,HEAD", + "access-control-allow-origin": "*", + "access-control-expose-headers": "etag", + "cache-control": "private, max-age=0, s-maxage=0, must-revalidate", + "content-length": "169", + "content-location": "https://en.wikipedia.org/api/rest_v1/data/citation/mediawiki/964-92962-6-3", + "content-security-policy": "default-src 'none'; frame-ancestors 'none'", + "content-type": "application/problem+json", + "date": "Sat, 08 Jan 2022 14:34:18 GMT", + "referrer-policy": "origin-when-cross-origin", + "server": "restbase1016", + "vary": "Accept-Encoding", + "x-content-security-policy": "default-src 'none'; frame-ancestors 'none'", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-webkit-csp": "default-src 'none'; frame-ancestors 'none'", + "x-xss-protection": "1; mode=block" + }, + "status_code": 404, + "url": "https://en.wikipedia.org/api/rest_v1/data/citation/mediawiki/964-92962-6-3" +} \ No newline at end of file diff --git a/test/testdata/558d0d51925af51732f2e4ae3c921675663ee5c7.html b/test/testdata/558d0d51925af51732f2e4ae3c921675663ee5c7.html new file mode 100644 index 00000000..8edd42ed --- /dev/null +++ b/test/testdata/558d0d51925af51732f2e4ae3c921675663ee5c7.html @@ -0,0 +1 @@ +{"institution":[{"name":"University of Twente","place":["Enschede, The Netherlands"]}],"indexed":{"date-parts":[[2022,3,29]],"date-time":"2022-03-29T23:22:15Z","timestamp":1648596135890},"reference-count":0,"publisher":"University Library\/University of Twente","isbn-type":[{"value":"9789036526326","type":"print"}],"content-domain":{"domain":[],"crossmark-restriction":false},"DOI":"10.3990\/1.9789036526326","type":"dissertation","created":{"date-parts":[[2008,9,23]],"date-time":"2008-09-23T08:54:12Z","timestamp":1222160052000},"approved":{"date-parts":[[2007,2,8]]},"source":"Crossref","is-referenced-by-count":0,"title":"Forecasting water waves and currents : a space-time approach","prefix":"10.3990","author":[{"given":"V.R.","family":"Ambati","sequence":"first","affiliation":[]}],"member":"2372","container-title":[],"original-title":[],"deposited":{"date-parts":[[2008,9,23]],"date-time":"2008-09-23T08:40:49Z","timestamp":1222159249000},"score":1,"degree":["PhD"],"resource":{"primary":{"URL":"http:\/\/purl.org\/utwente\/doi\/10.3990\/1.9789036526326"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[null]]},"ISBN":["9789036526326"],"references-count":0,"URL":"http:\/\/dx.doi.org\/10.3990\/1.9789036526326","relation":{}} \ No newline at end of file diff --git a/test/testdata/558d0d51925af51732f2e4ae3c921675663ee5c7.json b/test/testdata/558d0d51925af51732f2e4ae3c921675663ee5c7.json new file mode 100644 index 00000000..7b22ab66 --- /dev/null +++ b/test/testdata/558d0d51925af51732f2e4ae3c921675663ee5c7.json @@ -0,0 +1,24 @@ +{ + "encoding": null, + "headers": { + "access-control-allow-headers": "X-Requested-With, Accept, Accept-Encoding, Accept-Charset, Accept-Language, Accept-Ranges, Cache-Control", + "access-control-allow-origin": "*", + "access-control-expose-headers": "Link", + "connection": "close", + "content-encoding": "gzip", + "content-length": "640", + "content-type": "application/vnd.citationstyles.csl+json", + "date": "Thu, 09 Jun 2022 10:35:27 GMT", + "link": "; rel=\"canonical\"", + "permissions-policy": "interest-cohort=()", + "server": "Jetty(9.4.40.v20210413)", + "vary": "Accept, Accept-Encoding", + "x-api-pool": "public", + "x-rate-limit-interval": "1s", + "x-rate-limit-limit": "50", + "x-ratelimit-interval": "1s", + "x-ratelimit-limit": "50" + }, + "status_code": 200, + "url": "https://api.crossref.org/v1/works/10.3990%2F1.9789036526326/transform" +} \ No newline at end of file diff --git a/test/testdata/55ed9498dca175707fd51a42ea883564c70701f9.html b/test/testdata/55ed9498dca175707fd51a42ea883564c70701f9.html new file mode 100644 index 00000000..6f1a279d --- /dev/null +++ b/test/testdata/55ed9498dca175707fd51a42ea883564c70701f9.html @@ -0,0 +1,2810 @@ + + + + + + + The Independent | News | UK and Worldwide News | Newspaper + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + +
    + +
  1. + + +
  2. + +
  3. + +
  4. +
+ + +
+
+
+
+
+
+
+ +
+
+

+ Top stories + +

+
+
+
+
+
+

+ Talking points

+
+
+

+ Finance

+
+
+
+
+
+
+
+
+

+ IndyBest

+
+
+ +
+
+
+
+
+
+
    +
  • + food + drink +
  • +
+
+

+ 10 best BBQ beers +

+
+ +
+
+
+
+
+
+
+
    +
  • + home + garden +
  • +
+
+

+ 10 best BBQ food +

+
+ +
+ +
+
+ +
+
+
+ + + +
+ +
+
+
+ + +
+ + + +
+ + + + + + + + + +
+ + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/testdata/55ed9498dca175707fd51a42ea883564c70701f9.json b/test/testdata/55ed9498dca175707fd51a42ea883564c70701f9.json new file mode 100644 index 00000000..f35a6cf6 --- /dev/null +++ b/test/testdata/55ed9498dca175707fd51a42ea883564c70701f9.json @@ -0,0 +1,33 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "1396", + "Cache-control": "no-cache, no-store, max-age=0, must-revalidate", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Language": "en", + "Content-Length": "32159", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:54:06 GMT", + "Etag": "\"1495560646-1\"", + "Expires": "Sun, 19 Nov 1978 05:00:00 GMT", + "Last-modified": "Fri, 03 Jun 2016 15:18:58 GMT", + "Link": "; rel=\"canonical\",; rel=\"shortcut icon\",; rel=\"apple-touch-icon\",; rel=\"apple-touch-icon_72x72\",; rel=\"apple-touch-icon_76x76\",; rel=\"apple-touch-icon_114x114\",; rel=\"apple-touch-icon_120x120\",; rel=\"apple-touch-icon_144x144\",; rel=\"apple-touch-icon_152x152\",; rel=\"apple-touch-icon_180x180\",; rel=\"android-icon_36x36\",; rel=\"android-icon_48x48\",; rel=\"android-icon_72x72\",; rel=\"android-icon_96x96\",; rel=\"android-icon_114x114\",; rel=\"android-icon_192x192\",; rel=\"ms-icon_70x70\",; rel=\"ms-icon_144x144\",; rel=\"ms-icon_150x150\",; rel=\"ms-icon_310x310\"", + "Server": "nginx", + "Vary": "Accept-Encoding, Locale, ines_tg", + "Via": "1.1 varnish-v4, 1.1 varnish", + "X-AH-Environment": "prod", + "X-Cache": "MISS, HIT", + "X-Cache-Hits": "15", + "X-Content-Type-Options": "nosniff", + "X-Drupal-Cache": "MISS", + "X-Frame-Options": "SAMEORIGIN", + "X-Generator": "Drupal 7 (http://drupal.org)", + "X-Request-ID": "v-8e7067e8-3fdd-11e7-85fb-22000b0a13de", + "X-Served-By": "cache-iad2125-IAD", + "X-Timer": "S1495562046.322771,VS0,VE0" + }, + "status_code": 200, + "url": "http://www.independent.co.uk/us" +} \ No newline at end of file diff --git a/test/testdata/590fdd60f2d6265bfc174c6e74f1882d8acca148.html b/test/testdata/590fdd60f2d6265bfc174c6e74f1882d8acca148.html new file mode 100644 index 00000000..0023464e --- /dev/null +++ b/test/testdata/590fdd60f2d6265bfc174c6e74f1882d8acca148.html @@ -0,0 +1,456 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Investor, Bettor, Golfer: Insider Trading Inquiry Includes Mickelson, Icahn and William T. Walters - The New York Times + + + + + + + + + + + + + + + + + + +
+ + + + + + + +
+
+
+
+
+
+ +
+
+
+
+
+ + +
+
+
+ +
+
+
+
+
+
+
+

Investor, Bettor, Golfer: Insider Trading Inquiry Includes Mickelson, Icahn and William T. Walters

+ +
+
+
Photo
The investigation is focusing on trading in two different stocks by Phil Mickelson, above, and the gambler William Walters.Credit Sam Greenwood/Getty Images
+
+
+ +

The divergent lives of a championship golfer, a high-rolling gambler and a billionaire investor have collided in a federal insider trading investigation.

+

Federal authorities are examining a series of well-timed trades made by the golfer Phil Mickelson and the gambler William T. Walters, people briefed on the investigation said, focusing on trading in two different stocks. The authorities are also questioning what role, if any, the investor Carl C. Icahn may have had in sharing information about one of the stocks: the consumer products company Clorox.

+

Mr. Walters, an owner of golf courses who is often considered the most successful sports bettor in the country, placed his Clorox trade in 2011, the people briefed on the investigation said. Mr. Icahn, a 78-year-old billionaire and one of the best-known investors in the world, was mounting a takeover bid for Clorox around the time of that trade.

+

The F.B.I. and Securities and Exchange Commission, which are leading the inquiry along with federal prosecutors in Manhattan, are examining whether Mr. Icahn leaked details of his Clorox bid to Mr. Walters, the people briefed on the investigation said. One initial theory, the people said, is that Mr. Walters might have passed that information to Mr. Mickelson. But Mr. Mickelson, a three-time winner of the Masters golf tournament and one of the country’s highest-earning athletes, did not bet on Clorox, though he did trade in the stock of another company tied to Mr. Walters, the people added.

+

Aound the time of the Clorox trading, the S.E.C. sent Mr. Icahn a routine request for information about his dealings in the company, the people briefed on the matter said. Federal authorities, whose investigation has dragged on for more than two years without yielding definitive evidence of insider trading, are also examining phone records to see whether Mr. Walters spoke to Mr. Icahn shortly before the trades.

+

Mr. Icahn’s bid for Clorox ultimately failed. Mr. Mickelson, Mr. Walters and Mr. Icahn have not been accused of any wrongdoing. Mr. Icahn, even if he did leak secret information about his firm’s intentions with Clorox, may have done so legally. It would be illegal if he breached a duty of confidentiality to his own investors.

+

In a separate strand of the investigation, federal authorities are looking into trading in Dean Foods that has no apparent connection to Mr. Icahn, the people briefed on the matter said. Mr. Walters and Mr. Mickelson placed the trades around August 2012, according to the people, just before the food and beverage company announced its quarterly earnings and a public offering of stock for one of its subsidiaries. The authorities are investigating whether Mr. Walters had a source inside the company itself — and whether others who know Mr. Walters may have traded on the information as well.

+

Mr. Walters, reached on Friday evening, said, “While I don’t have any comment, pal, I’ll talk to you later.”

+

In an interview, Mr. Icahn said “I don’t give out inside information,” adding that “for 50 years I have had an unblemished record.” Mr. Icahn, who acknowledged knowing Mr. Walters but said he never met or spoke to Mr. Mickelson, argued that any suggestion he did anything wrong is “irresponsible.”

+

Representatives for Mr. Mickelson did not respond to a request for comment. Federal authorities declined to comment.

+

For two years, authorities had little to go on besides trading records and a hunch. Then last year, F.B.I. agents approached Mr. Mickelson at Teterboro Airport in New Jersey, one of the people briefed on the matter said, asking the celebrity golfer to discuss his trading.

+

It is unclear whether Mr. Mickelson knows Mr. Icahn or provided any evidence implicating him or Mr. Walters in the trading. It is possible that the investigations will not produce any charges.

+

But if the investigation proceeds, it could undermine the reputation of one of America’s most popular athletes in Mr. Mickelson, who has won five major championships over a two-decade career. And for Mr. Icahn, the investigation might complicate one of the longest running and most successful careers on Wall Street.

+

Mr. Icahn made his foray into finance as a stockbroker in the 1960s. He later became a professional agitator, haranguing the country’s biggest companies to give him a board seat.

+

Long before activist investing was in vogue, Mr. Icahn was waging war with executives at companies like Motorola, RJR Nabisco and United States Steel, pushing for corporate changes to increase shareholder value. In the 1980s, Mr. Icahn became synonymous with an era of corporate raiding, leading a hostile takeover of Trans World Airlines.

+

In recent years, his strategy has mellowed some. Mr. Icahn has become something of an elder statesman on Wall Street, often appearing on the CNBC business channel, at investing conferences and even on Twitter, though he continues to pursue headline-grabbing takeover bids for companies like Clorox.

+

Mr. Icahn laid the groundwork for a Clorox takeover in early 2011, when he disclosed in a regulatory filing that his various investment firms began amassing shares in the consumer goods manufacturer, thinking the stock was undervalued. Shares of Clorox rose about 6 percent in February 2011, after Mr. Icahn disclosed his stake.

+

The shares jumped again a few months later after he announced an unsolicited takeover bid for the company. In a letter to Clorox, Mr. Icahn proposed buying the company for $76.50 a share and noted that his firms were Clorox’s largest investor.

+

In the days leading up to Mr. Icahn’s bid, there was unusual trading activity in shares of Clorox and options to buy the stock, according to published reports at the time. Successful options trading that comes ahead of corporate deals can be a red flag for regulators.

+

The investigation into the Clorox trading began at the Financial Industry Regulatory Authority, or Finra, Wall Street’s self-regulatory group that monitors suspicious trades. In 2011, the people briefed on the matter said, Finra traced a series of well-timed Clorox trades to Mr. Walters and other investors, just as Mr. Icahn was aiming to gain a foothold on the company’s board.

+

Ultimately, Clorox rebuffed Mr. Icahn’s overture. By September 2011, Mr. Icahn withdrew the bid.

+

Because the bid failed, it is unclear what inside information, if any, Mr. Icahn may have been privy to other than the trading strategy of his own firm, Icahn Enterprises. If Mr. Icahn provided Mr. Walters a heads-up about his activities in connection with the takeover bid, it is not necessarily a violation. Under the laws that govern insider trading, it is not illegal to leak secret information about a future trade.

+

For such a leak to be illegal, Mr. Icahn most likely would have had to breach a duty to keep the information confidential. Since Mr. Icahn never joined the Clorox board, he probably owed no duty to the company or its shareholders.

+

Yet if any potential bidder for Clorox breached a duty of confidentiality to his or her own investors, then that could present a legal problem. And in certain cases, even if there is no duty of confidentiality, a little-known securities rule might prevent someone who is mounting a takeover bid to leak “material, nonpublic information” about the offer.

+

It is unclear how well acquainted Mr. Icahn and Mr. Walters are, but they have crossed paths in Las Vegas. Mr. Icahn is no stranger to the city. Over the years, Mr. Icahn has invested in Las Vegas real estate and is chairman of Tropicana Entertainment, a casino company based in the city. Regulatory filings also show that a small Nevada company controlled by Mr. Walters and his business partner was an early investor in the mobile data provider Voltari, whose largest shareholders include Mr. Icahn’s investment firms.

+

Mr. Mickelson and Mr. Walters have participated in the Pebble Beach Pro Am golf tournament during which professional golfers partner with amateurs and celebrities. Mr. Walters won the tournament in 2008.

+

Mr. Walters, better known as Billy, has drawn federal scrutiny off and on for years. In 1992, he was acquitted of illegal gambling charges. Years later, the Nevada attorney general charged Mr. Walters with money laundering stemming from his gambling operation. The case resulted in three indictments; courts dismissed each one.

+

Despite the scrutiny, Mr. Walters firmly belongs to the Las Vegas elite, a generous philanthropist who epitomizes the city’s unconventional brand of capitalism. He bought up golf courses — his Bali Hai Golf Club has hosted what it calls the “sexiest golf tournament” in the world, featuring female caddies — and auto dealerships. He also helped finance political campaigns.

+

In 2011, “60 Minutes” captured Mr. Walters’s high-roller status. The anchor, Lara Logan, remarked that “It’s hard to find anyone better at winning than Billy Walters.”

+

But during the segment, Mr. Walters complained that his stock picks had not fared as well as his sports bets. He discussed how the worst “crooks” he met were on Wall Street, not in the casinos or betting parlors, mentioning that the most money he lost was on stocks like Enron.

+

Alexandra Stevenson, Azam Ahmed and Peter J. Henning contributed reporting.

+
Correction: June 12, 2014
An article on May 31 about an insider trading investigation, using information from people briefed on the inquiry, overstated the scope of an investigation involving the golfer Phil Mickelson. While investigators are looking at his trading in some stocks, Clorox is not among them. The error was repeated in articles on June 1 and June 2 and in the Common Sense column on Saturday. An article about the latest developments in the investigation appears today on Page B1. + + +
+ +
+
+ +
+ +
+
+
+
+
+ + + + + +
+
+

Advertisement

+
+ + + + +
+
+
+
+ + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/testdata/590fdd60f2d6265bfc174c6e74f1882d8acca148.json b/test/testdata/590fdd60f2d6265bfc174c6e74f1882d8acca148.json new file mode 100644 index 00000000..70276d27 --- /dev/null +++ b/test/testdata/590fdd60f2d6265bfc174c6e74f1882d8acca148.json @@ -0,0 +1,30 @@ +{ + "encoding": "UTF-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "12", + "Cache-Control": "no-cache", + "Connection": "close", + "Content-Encoding": "gzip", + "Content-Length": "14892", + "Content-Security-Policy": "default-src data: 'unsafe-inline' 'unsafe-eval' https:; script-src data: 'unsafe-inline' 'unsafe-eval' https: blob:; style-src data: 'unsafe-inline' https:; img-src data: https: blob:; font-src data: https:; connect-src https: wss:; media-src https: blob:; object-src https:; child-src https: data: blob:; form-action https:; block-all-mixed-content;", + "Content-Type": "text/html; charset=UTF-8", + "Cteonnt-Length": "48549", + "Date": "Wed, 24 May 2017 07:35:06 GMT", + "Link": "; rel=shortlink, ; rel=\"https://github.com/WP-API/WP-API\"", + "Server": "Apache", + "Vary": "Accept-Encoding, Fastly-SSL", + "X-API-Version": "F-5-5b", + "X-Age": "0", + "X-Cache": "HIT", + "X-Cache-Hits": "1", + "X-ESI": "1", + "X-Frame-Options": "DENY", + "X-Origin-Time": "2017-05-24 03:34:54 EDT", + "X-PageType": "blog", + "X-Served-By": "cache-sjc3625-SJC", + "X-Timer": "S1495611307.502434,VS0,VE1" + }, + "status_code": 200, + "url": "https://dealbook.nytimes.com/2014/05/30/insider-trading-inquiry-includes-mickelson-and-icahn/?_r=0" +} \ No newline at end of file diff --git a/test/testdata/5a9b17b3337c39c286575cf3547a345a2486310a.html b/test/testdata/5a9b17b3337c39c286575cf3547a345a2486310a.html new file mode 100644 index 00000000..1f827a08 --- /dev/null +++ b/test/testdata/5a9b17b3337c39c286575cf3547a345a2486310a.html @@ -0,0 +1,1191 @@ + + + + + + Home - BBC News + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+
+
+ + + + + + + +
+ + +
+
+ +
+
+

BBC News Home

+ + +

Top Stories

+ + +
+
+ + +
+ +

+ Sport + +

+ + + + +
+ +
+

+ + Weather +

+
+
+ + + +
+ +
+
+ +
+
+ + + + + +
+ +

+ + Follow Us +

+ + +
+ + + +
+ +

+ + Elsewhere on the BBC +

+
+ +
+ + +
+ + +
+
+ +
+
+ + + + +
+
+ + + + + + + + +
+
+ + + + + +
+ + + + +
+ +
+ + + +
+ + + + + + + + + + + + + + + + + + diff --git a/test/testdata/5a9b17b3337c39c286575cf3547a345a2486310a.json b/test/testdata/5a9b17b3337c39c286575cf3547a345a2486310a.json new file mode 100644 index 00000000..db6294c3 --- /dev/null +++ b/test/testdata/5a9b17b3337c39c286575cf3547a345a2486310a.json @@ -0,0 +1,24 @@ +{ + "encoding": "utf-8", + "headers": { + "Cache-Control": "max-age=30, stale-while-revalidate", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Language": "en-GB", + "Content-Length": "46532", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:52:58 GMT", + "Server": "Apache", + "Vary": "X-CDN,X-BBC-Edge-Cache,Accept-Encoding", + "Warning": "110 varnish 'Response is stale'", + "X-Cache-Action": "HIT", + "X-Cache-Age": "40", + "X-Cache-Hits": "89", + "X-LB-NoCache": "true", + "X-News-Cache-Id": "78741", + "X-News-Data-Centre": "telhc", + "X-PAL-Host": "pal1104.back.live.telhc.local:80" + }, + "status_code": 200, + "url": "http://www.bbc.co.uk/news" +} \ No newline at end of file diff --git a/test/testdata/5be66044e5f0c8076e8e80861f533521161b7b34.html b/test/testdata/5be66044e5f0c8076e8e80861f533521161b7b34.html new file mode 100644 index 00000000..2f2caabd --- /dev/null +++ b/test/testdata/5be66044e5f0c8076e8e80861f533521161b7b34.html @@ -0,0 +1,8 @@ +TY - BOOK +T1 - Microsoft Visual C# 2008 Comprehensive: An Introduction to Object-Oriented Programming +A1 - Farrell, J. +SN - 9781111786199 +UR - https://books.google.com/books?id=icMEAAAAQBAJ +Y1 - 2009 +PB - Cengage Learning +ER - diff --git a/test/testdata/5be66044e5f0c8076e8e80861f533521161b7b34.json b/test/testdata/5be66044e5f0c8076e8e80861f533521161b7b34.json new file mode 100644 index 00000000..59c0af98 --- /dev/null +++ b/test/testdata/5be66044e5f0c8076e8e80861f533521161b7b34.json @@ -0,0 +1,21 @@ +{ + "encoding": null, + "headers": { + "Alt-Svc": "h3-29=\":443\"; ma=2592000,h3-27=\":443\"; ma=2592000,h3-25=\":443\"; ma=2592000,h3-T050=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"", + "Cache-Control": "private, max-age=0", + "Content-Disposition": "attachment; filename=Microsoft_Visual_C_2008_Comprehensive_An.ris", + "Content-Length": "244", + "Content-Type": "application/x-research-info-systems", + "Date": "Sat, 11 Jul 2020 09:16:23 GMT", + "Expires": "Sat, 11 Jul 2020 09:16:23 GMT", + "P3P": "CP=\"This is not a P3P policy! See g.co/p3phelp for more info.\"", + "Server": "OFE/0.1", + "Set-Cookie": "NID=204=WJ0VgfbjPrRR64N4eplf3XvDEnHKRJjuYdlLkK6sKA5g4RiI3fkmJcbLqkj7PPh9kTUgesZqywMEhUfckAYeBBhPURmEjEmtvHMyvjpRuMNARyzhaW5io-8i44zbubKOXmgiKtrLHbP_yxhKOLJeJ7C_HWMG8E2KeTgf1o2pSRY; expires=Sun, 10-Jan-2021 09:16:23 GMT; path=/; domain=.google.com; Secure; HttpOnly; SameSite=none", + "Strict-Transport-Security": "max-age=604800", + "X-Content-Type-Options": "nosniff", + "X-Frame-Options": "SAMEORIGIN", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://books.google.com/books/download/?id=icMEAAAAQBAJ&output=ris" +} \ No newline at end of file diff --git a/test/testdata/5e1a99141326ce15f45af85e9705f5912b635e20.html b/test/testdata/5e1a99141326ce15f45af85e9705f5912b635e20.html new file mode 100644 index 00000000..7f847dba --- /dev/null +++ b/test/testdata/5e1a99141326ce15f45af85e9705f5912b635e20.html @@ -0,0 +1 @@ +{"result":{"total":1118288,"groups":{"author":{"total":64,"items":[{"entity_type":"Author","author_title":"اداره فرهنگی اجتماعی منطقه 6 مشهد","id":"Author-468740","url":"2bb6b242-92e6-40c0-9956-32013e2d8b91"}]},"printableBook":{"total":1118209,"items":[{"book_subject":null,"book_parent_subject":["ادبیات"],"image":"https://pic.ketab.ir/DataBase/BookImages/85/85904239.jpg","book_print_version":1,"book_cover_price":15000,"book_author":["حافظ ، شمس‌الدین‌محمد"],"book_page_count":494,"url":"bb12c0da-6ecc-4e84-90b8-3477998ba644","entity_type":"PrintableBook","book_title":"دیوان کامل حافظ همراه با فالنامه","book_issue_year":1385,"id":"Book-1276454","book_cover_type":"شومیز","book_publisher":"دیوان","book_volume_number":0}]},"publisher":{"total":15,"items":[{"image":"https://pic.ketab.ir/DataBase/Publishers/Arms/328375.jpg","entity_type":"Publisher","publisher_manager_fullname":" ","publisher_title":"سه سه تار","id":"Publisher-328375","url":"61cfafeb-cd3c-4fcd-8bb9-b5d1d17a8de2"}]}},"from":0},"facets":{"book_issue_year":{"1396":54944,"1385":51930,"1395":52050,"1384":51228,"1394":50535,"1383":40189,"1393":49066,"1382":35602,"1392":47181,"1381":32308,"1391":47592,"1390":53460,"1400":54295,"1389":53929,"1388":53664,"1399":49209,"1398":57453,"1387":53429,"1386":55099,"1397":54169},"book_parent_subject":{"آموزشی":100976,"فلسفه":45619,"ادبیات":177113,"کودک":164003,"کمک درسی":105693,"علوم طبیعی و ریاضیات":17616,"هنر":33829,"کمک درسی کودک":22892,"علوم اجتماعی":92946,"دانشگاهی":1,"دین":170222,"تاریخ و جغرافیا":48237,"علوم عملی":116939,"کلیات":1,"زبان":22122},"book_print_version_type":{"چاپ مجدد":552874,"چاپ اول":565335},"book_publisher":{"سمت":11413,"نشر نی":5560,"دانشگاه پیام نور":10682,"بین المللی گاج":12978,"موسسه فرهنگی مدرسه برهان":11068,"نشر چشمه":5220,"آیندگان هزار نکته وابسته به موسسه فرهنگی آیندگان هزار نکته":7238,"خیلی سبز":6902,"موسسه بوستان کتاب":6575,"قدیانی":11978,"پیدایش":4730,"مبتکران":14654,"شرکت انتشارات کانون فرهنگی آموزش":13774,"به نشر وابسته به آستان قدس رضوی":5664,"مدرسان شریف":12735,"شرکت انتشارات سوره مهر":5060,"امیرکبیر":6541,"موسسه چاپ و انتشارات دانشگاه تهران":5743,"نشر مرکز":5685,"افق":7574},"book_author":{"احمدی‌جزی ، کامران":897,"تریسی ، برایان":1462,"صادقی ، داریوش":866,"بازرگانی ، بهمن":1808,"محدثی ، جواد":973,"هیات ‌مولفان":1854,"شعبانی ، اسدالله":998,"فتاحی ، حسین":1532,"مطهری ، مرتضی":3479,"طباطبایی ، سیدمحمدحسین":1093,"موسوی ، سیدعلی":940,"قاسم‌نیا ، شکوه":1916,"سبحانی‌تبریزی ، جعفر":948,"مولوی ، جلال‌الدین‌محمدبن‌محمد":1470,"کوییلو ، پایولو":933,"انصاری ، حسین":839,"دپارتمان ا‌یندگان":2407,"عمیق ، مجید":984,"رحماندوست ، مصطفی":882,"فلاح‌زاده ، محمدحسین":926,"موحدی ، محمود":1694,"کشاورز ، ناصر":2042,"مکارم‌شیرازی ، ناصر":2020,"ال‌احمد ، جلال":1219,"حافظ ، شمس‌الدین‌محمد":2307,"حیدری‌ابهری ، غلامرضا":950,"حامی ، فرهاد":942,"قمی ، عباس":5236,"قرایتی ، محسن":1496,"کیانی ، مصطفی":910,"مجلسی ، محمدباقربن‌محمدتقی":1371,"قراچه‌داغی ، مهدی":2061,"فاضلی ، بنفشه":924,"جوادی‌املی ، عبدالله":1276,"وحیدی‌صدر ، مهدی":992,"اخلاصمندمنفرد ، علیرضا":1695,"صفایی‌دیبا ، علی‌اکبر":894,"نامی ، حسین":1300,"الهی‌قمشه‌ای ، مهدی":6875,"گروه مولفان":1390,"نیکوکار ، مسعود":940,"نجف‌خانی ، محبوبه":955,"گراس ، تونی":1239,"استاین ، ار.ال.":1096,"فردوسی ، ابوالقاسم":1038,"سعدی ، مصلح‌بن‌عبدالله":1458,"طالب‌تبار ، حمید":1050,"اناری ، شهاب":1350,"اعضای هیات علمی سنجش تکمیلی":1332,"محمدی‌ری‌شهری ، محمد":1254},"listModel":{"book_issue_year":[{"label":"1396","value":54944},{"label":"1385","value":51930},{"label":"1395","value":52050},{"label":"1384","value":51228},{"label":"1394","value":50535},{"label":"1383","value":40189},{"label":"1393","value":49066},{"label":"1382","value":35602},{"label":"1392","value":47181},{"label":"1381","value":32308},{"label":"1391","value":47592},{"label":"1390","value":53460},{"label":"1400","value":54295},{"label":"1389","value":53929},{"label":"1388","value":53664},{"label":"1399","value":49209},{"label":"1398","value":57453},{"label":"1387","value":53429},{"label":"1386","value":55099},{"label":"1397","value":54169}],"book_print_version_type":[{"label":"چاپ مجدد","value":552874},{"label":"چاپ اول","value":565335}],"book_parent_subject":[{"label":"آموزشی","value":100976},{"label":"فلسفه","value":45619},{"label":"ادبیات","value":177113},{"label":"کودک","value":164003},{"label":"کمک درسی","value":105693},{"label":"علوم طبیعی و ریاضیات","value":17616},{"label":"هنر","value":33829},{"label":"کمک درسی کودک","value":22892},{"label":"علوم اجتماعی","value":92946},{"label":"دانشگاهی","value":1},{"label":"دین","value":170222},{"label":"تاریخ و جغرافیا","value":48237},{"label":"علوم عملی","value":116939},{"label":"کلیات","value":1},{"label":"زبان","value":22122}],"book_publisher":[{"label":"سمت","value":11413},{"label":"نشر نی","value":5560},{"label":"دانشگاه پیام نور","value":10682},{"label":"بین المللی گاج","value":12978},{"label":"موسسه فرهنگی مدرسه برهان","value":11068},{"label":"نشر چشمه","value":5220},{"label":"آیندگان هزار نکته وابسته به موسسه فرهنگی آیندگان هزار نکته","value":7238},{"label":"خیلی سبز","value":6902},{"label":"موسسه بوستان کتاب","value":6575},{"label":"قدیانی","value":11978},{"label":"پیدایش","value":4730},{"label":"مبتکران","value":14654},{"label":"شرکت انتشارات کانون فرهنگی آموزش","value":13774},{"label":"به نشر وابسته به آستان قدس رضوی","value":5664},{"label":"مدرسان شریف","value":12735},{"label":"شرکت انتشارات سوره مهر","value":5060},{"label":"امیرکبیر","value":6541},{"label":"موسسه چاپ و انتشارات دانشگاه تهران","value":5743},{"label":"نشر مرکز","value":5685},{"label":"افق","value":7574}],"book_author":[{"label":"احمدی‌جزی ، کامران","value":897},{"label":"تریسی ، برایان","value":1462},{"label":"صادقی ، داریوش","value":866},{"label":"بازرگانی ، بهمن","value":1808},{"label":"محدثی ، جواد","value":973},{"label":"هیات ‌مولفان","value":1854},{"label":"شعبانی ، اسدالله","value":998},{"label":"فتاحی ، حسین","value":1532},{"label":"مطهری ، مرتضی","value":3479},{"label":"طباطبایی ، سیدمحمدحسین","value":1093},{"label":"موسوی ، سیدعلی","value":940},{"label":"قاسم‌نیا ، شکوه","value":1916},{"label":"سبحانی‌تبریزی ، جعفر","value":948},{"label":"مولوی ، جلال‌الدین‌محمدبن‌محمد","value":1470},{"label":"کوییلو ، پایولو","value":933},{"label":"انصاری ، حسین","value":839},{"label":"دپارتمان ا‌یندگان","value":2407},{"label":"عمیق ، مجید","value":984},{"label":"رحماندوست ، مصطفی","value":882},{"label":"فلاح‌زاده ، محمدحسین","value":926},{"label":"موحدی ، محمود","value":1694},{"label":"کشاورز ، ناصر","value":2042},{"label":"مکارم‌شیرازی ، ناصر","value":2020},{"label":"ال‌احمد ، جلال","value":1219},{"label":"حافظ ، شمس‌الدین‌محمد","value":2307},{"label":"حیدری‌ابهری ، غلامرضا","value":950},{"label":"حامی ، فرهاد","value":942},{"label":"قمی ، عباس","value":5236},{"label":"قرایتی ، محسن","value":1496},{"label":"کیانی ، مصطفی","value":910},{"label":"مجلسی ، محمدباقربن‌محمدتقی","value":1371},{"label":"قراچه‌داغی ، مهدی","value":2061},{"label":"فاضلی ، بنفشه","value":924},{"label":"جوادی‌املی ، عبدالله","value":1276},{"label":"وحیدی‌صدر ، مهدی","value":992},{"label":"اخلاصمندمنفرد ، علیرضا","value":1695},{"label":"صفایی‌دیبا ، علی‌اکبر","value":894},{"label":"نامی ، حسین","value":1300},{"label":"الهی‌قمشه‌ای ، مهدی","value":6875},{"label":"گروه مولفان","value":1390},{"label":"نیکوکار ، مسعود","value":940},{"label":"نجف‌خانی ، محبوبه","value":955},{"label":"گراس ، تونی","value":1239},{"label":"استاین ، ار.ال.","value":1096},{"label":"فردوسی ، ابوالقاسم","value":1038},{"label":"سعدی ، مصلح‌بن‌عبدالله","value":1458},{"label":"طالب‌تبار ، حمید","value":1050},{"label":"اناری ، شهاب","value":1350},{"label":"اعضای هیات علمی سنجش تکمیلی","value":1332},{"label":"محمدی‌ری‌شهری ، محمد","value":1254}]}},"spelling":null} \ No newline at end of file diff --git a/test/testdata/5e1a99141326ce15f45af85e9705f5912b635e20.json b/test/testdata/5e1a99141326ce15f45af85e9705f5912b635e20.json new file mode 100644 index 00000000..62d8814f --- /dev/null +++ b/test/testdata/5e1a99141326ce15f45af85e9705f5912b635e20.json @@ -0,0 +1,21 @@ +{ + "encoding": "utf-8", + "headers": { + "AR-ATIME": "1.756", + "AR-CACHE": "BYPASS", + "AR-PoweredBy": "Arvan Cloud (arvancloud.com)", + "AR-Request-ID": "6653382e9e1937d22c0c6d485c16d2e0", + "AR-SID": "2001", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Type": "application/json; charset=utf-8", + "Date": "Sat, 27 Aug 2022 23:28:57 GMT", + "Keep-Alive": "timeout=65", + "Server": "ArvanCloud", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding, Accept-Encoding", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://msapi.ketab.ir/search/?query=964-92962-6-3&limit=1" +} \ No newline at end of file diff --git a/test/testdata/5e97044bbcaf29d85c6645eeaea72af08749e93c.html b/test/testdata/5e97044bbcaf29d85c6645eeaea72af08749e93c.html new file mode 100644 index 00000000..74617cd9 --- /dev/null +++ b/test/testdata/5e97044bbcaf29d85c6645eeaea72af08749e93c.html @@ -0,0 +1,35 @@ +{ + "type": "article-journal", + "id": "https://doi.org/10.48550/arxiv.1811.06526", + "categories": [ + "Popular Physics (physics.pop-ph)", + "Space Physics (physics.space-ph)", + "FOS: Physical sciences", + "FOS: Physical sciences" + ], + "author": [ + { + "family": "Hein", + "given": "Andreas M." + }, + { + "family": "Baxter", + "given": "Stephen" + } + ], + "issued": { + "date-parts": [ + [ + 2018 + ] + ] + }, + "abstract": "The large distances involved in interstellar travel require a high degree of spacecraft autonomy, realized by artificial intelligence. The breadth of tasks artificial intelligence could perform on such spacecraft involves maintenance, data collection, designing and constructing an infrastructure using in-situ resources. Despite its importance, existing publications on artificial intelligence and interstellar travel are limited to cursory descriptions where little detail is given about the nature of the artificial intelligence. This article explores the role of artificial intelligence for interstellar travel by compiling use cases, exploring capabilities, and proposing typologies, system and mission architectures. Estimations for the required intelligence level for specific types of interstellar probes are given, along with potential system and mission architectures, covering those proposed in the literature but also presenting novel ones. Finally, a generic design for interstellar probes with an AI payload is proposed. Given current levels of increase in computational power, a spacecraft with a similar computational power as the human brain would have a mass from dozens to hundreds of tons in a 2050-2060 timeframe. Given that the advent of the first interstellar missions and artificial general intelligence are estimated to be by the mid-21st century, a more in-depth exploration of the relationship between the two should be attempted, focusing on neglected areas such as protecting the artificial intelligence payload from radiation in interstellar space and the role of artificial intelligence in self-replication.", + "container-title": "arXiv", + "DOI": "10.48550/ARXIV.1811.06526", + "publisher": "arXiv", + "title": "Artificial Intelligence for Interstellar Travel", + "URL": "https://arxiv.org/abs/1811.06526", + "copyright": "arXiv.org perpetual, non-exclusive license", + "version": "3" +} \ No newline at end of file diff --git a/test/testdata/5e97044bbcaf29d85c6645eeaea72af08749e93c.json b/test/testdata/5e97044bbcaf29d85c6645eeaea72af08749e93c.json new file mode 100644 index 00000000..501364a6 --- /dev/null +++ b/test/testdata/5e97044bbcaf29d85c6645eeaea72af08749e93c.json @@ -0,0 +1,20 @@ +{ + "encoding": "utf-8", + "headers": { + "Cache-Control": "max-age=0, private, must-revalidate", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Type": "application/vnd.citationstyles.csl+json; charset=utf-8", + "Date": "Thu, 09 Jun 2022 12:13:29 GMT", + "ETag": "W/\"3648c0fb656ded2bc53995302f803ae7\"", + "Server": "nginx/1.18.0 + Phusion Passenger(R) 6.0.13", + "Status": "200 OK", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding, Origin", + "X-Powered-By": "Phusion Passenger(R) 6.0.13", + "X-Request-Id": "bb634e5f-efe8-4908-9efd-11951478b10f", + "X-Runtime": "0.038989" + }, + "status_code": 200, + "url": "https://data.crosscite.org/10.48550%2FarXiv.1811.06526" +} \ No newline at end of file diff --git a/test/testdata/5ef38b44db6a0e5401efd48e99217eb6af8f1a86.html b/test/testdata/5ef38b44db6a0e5401efd48e99217eb6af8f1a86.html new file mode 100644 index 00000000..5fccf30f --- /dev/null +++ b/test/testdata/5ef38b44db6a0e5401efd48e99217eb6af8f1a86.html @@ -0,0 +1,1815 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Boston.com - Local breaking news, sports, and culture + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+ +
+
+
+
+
+
+
+ + +
+
+
+
+
+
+
+ + +
+
+
+ +
+
+ + +
Most Popular
+ +
+
+
+ +
+
+
+
+
+
+
+
+ + + + + +
+
+
+ +
+
+ + + + +
+
+
+
+
+ + +
+
+
+
+
+ + + + +
+
+
+
+
+ + +
+
+
+
+
+ + + + +
+
+
+
+
+ + + + +
+
+
+
+
+ +
+
+
+
+
+
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+ +
+ + + +
+ + + + + + + + + + + + + + + + + + +
+
+
+ +
+
+
+
+
+ +
+
+ + + diff --git a/test/testdata/5ef38b44db6a0e5401efd48e99217eb6af8f1a86.json b/test/testdata/5ef38b44db6a0e5401efd48e99217eb6af8f1a86.json new file mode 100644 index 00000000..a6c26e5f --- /dev/null +++ b/test/testdata/5ef38b44db6a0e5401efd48e99217eb6af8f1a86.json @@ -0,0 +1,26 @@ +{ + "encoding": "UTF-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "4156", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "61275", + "Content-Security-Policy": "upgrade-insecure-requests", + "Content-Type": "text/html; charset=UTF-8", + "Date": "Tue, 23 May 2017 17:53:02 GMT", + "Fastly-Debug-Digest": "583312a3eb81a4b4e4557aabbc3bf20c65443f6ee8de175c8d7e8f17bce912a4", + "Fastly-SSL": "1", + "Link": "; rel=\"https://api.w.org/\", ; rel=shortlink", + "Server": "Apache", + "Vary": "Accept-Encoding, Origin,Fastly-SSL,Fastly-SSL", + "Via": "1.1 varnish, 1.1 varnish", + "X-Cache": "HIT, HIT", + "X-Cache-Hits": "1166, 40", + "X-Served-By": "cache-jfk8129-JFK, cache-iad2121-IAD", + "X-TTL": "default", + "X-Timer": "S1495561983.579645,VS0,VE0" + }, + "status_code": 200, + "url": "https://www.boston.com/" +} \ No newline at end of file diff --git a/test/testdata/60894204cf153bd64d77c4ed222bb0d11ac3f4a8.html b/test/testdata/60894204cf153bd64d77c4ed222bb0d11ac3f4a8.html new file mode 100644 index 00000000..6b67e204 --- /dev/null +++ b/test/testdata/60894204cf153bd64d77c4ed222bb0d11ac3f4a8.html @@ -0,0 +1,675 @@ + + + + + + فراهنجاری-در-مثنوی‌سرایی:-بررسی-قالب-غزل---مثنوی-در-ادب-فارسی | هنر-رسانه-اردیبهشت | خانه کتاب و ادبیات ایران + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + +
+
+
+
+ + + + + + + +
+ + + + + +
+ + + ورود + + ثبت نام + + + + + + +
+
+
+
+ + +
+ + + + +
+
+
+
+
+ فراهنجاری در مثنوی‌سرایی: بررسی قالب غزل - مثنوی در ادب فارسی | خانه کتاب و ادبیات ایران +
+
+
+ صفحات اولیه کتاب +

+ فراهنجاری در مثنوی‌سرایی: بررسی قالب غزل - مثنوی در ادب فارسی

+

+ + + غزل + + + مثنوی + + +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
پدیدآور + + نويسنده : + + یوسف‌نژاد ، یوسف‌علی + + +
ناشر + + + + هنر رسانه اردیبهشت + + (برای تماس با ناشر و خرید کتاب کلیک کنید) + + + + +
شابک978-964-2656-34-9
تاریخ نشر + +13880220 +
قیمت +30,000
کد دیویی8fa1.0092
زبان کتابفارسی
محل نشرتهران - تهران
توضیحات + جلد - + 200 صفحه - + تالیف - + چاپ 1 +
+
+
+
+
+
+
+
+
+
معرفی مختصر کتاب
+

+ «غزل ـ مثنوی»، به منزلة یکی از فراهنجاری‌های قالبی، عبارت است از آوردن غزل در متن مثنوی که امروزه کاربرد فراوانی یافته است. سرچشمة این فراهنجاری در ادبیات کهن فارسی دیده می‌شود. در این کتاب سعی شده تا سیری مختصر از این رویکرد و تحول قالب غزل ـ مثنوی از آغاز تاکنون آورده شود. کتاب در دو بخش «طبقه‌بندی انواع ادبی» و «فراهنجاری در قالب مثنوی» این موضوعات را شامل می‌شود. انواع ادبی؛ سیر غزل و مثنوی در شعر فارسی؛ غزل مثنوی در شعر گذشته و معاصر؛ و ده نامه‌سرایی در شعر فارسی. +

+
+
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/testdata/60894204cf153bd64d77c4ed222bb0d11ac3f4a8.json b/test/testdata/60894204cf153bd64d77c4ed222bb0d11ac3f4a8.json new file mode 100644 index 00000000..ba56a5a5 --- /dev/null +++ b/test/testdata/60894204cf153bd64d77c4ed222bb0d11ac3f4a8.json @@ -0,0 +1,22 @@ +{ + "encoding": "utf-8", + "headers": { + "AR-ATIME": "0.116", + "AR-CACHE": "BYPASS", + "AR-PoweredBy": "Arvan Cloud (arvancloud.com)", + "AR-Request-ID": "f7041532a20be71bce048f9ee9cb5270", + "AR-SID": "2012", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Security-Policy": "upgrade-insecure-requests", + "Content-Type": "text/html; charset=utf-8", + "Date": "Sat, 27 Aug 2022 23:07:08 GMT", + "Keep-Alive": "timeout=65", + "Server": "ArvanCloud", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding, Accept-Encoding", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://ketab.ir/book/a5958832-5c43-460d-bf42-65acb6077e52" +} \ No newline at end of file diff --git a/test/testdata/60aa087f20252dedec62fcb65275a164648fe344.html b/test/testdata/60aa087f20252dedec62fcb65275a164648fe344.html new file mode 100644 index 00000000..d3e5a12d --- /dev/null +++ b/test/testdata/60aa087f20252dedec62fcb65275a164648fe344.html @@ -0,0 +1 @@ +The Times & The Sunday Times
Subscription Notification
We have noticed that there is an issue with your subscription billing details. Please update your billing details here
Please update your billing information
The subscription details associated with this account need to be updated. Please update your billing details here to continue enjoying your subscription.
Your subscription will end shortly
Please update your billing details here to continue enjoying your access to the most informative and considered journalism in the UK.

Welcome to The Times and The Sunday Times

The Times & The Sunday Times Homepage

Subscribe and get a complimentary Nespresso machine

Get a Nespresso connected coffee machine when you subscribe to The Digital Pack

Subscribe now

Get 3 months for the price of 1

Save up to £123.89 when you subscribe

Subscribe Now

Access the stories behind the headlines

Subscriptions starting from just £1

Subscribe now

Sir Roger Moore

The debonaire actor who brought a suave wit to the role of James Bond

Sir Roger Moore

Kimono mania

Sling on a silken wrap to follow one of fashion’s easiest fads

Kimono mania

Red Box podcast

Matt Chorley reflects on the shocking event that took place in Manchester

Red Box podcast

Police name bomber as arrests made across Manchester

new

Anti-terrorism raids were continuing across south Manchester today as police hunted accomplices of Salman Abedi, the suicide bomber who killed 22 people and injured 59 at a concert in the city last night. In a fast-moving operation, a 23-year-old man was arrested in the Chorlton area by armed...Anti-terrorism raids were continuing across south Manchester today as police hunted accomplices of Salman Abedi, the suicide bomber who killed 22 people and injured 59 at a concert in the city last night. In a fast-moving operation, a 23-year-old man was arrested in the Chorlton area by armed...Anti-terrorism raids were continuing across south Manchester today as police hunted accomplices of Salman Abedi, the suicide...

Read the full story

Isis supporters gloat about attack on social media

new

Supporters of Islamic State took to social media to celebrate the Manchester Arena attack that left 22 dead last night, with hashtags such as #Just_Terror. Gloating posts on Facebook, Twitter and the Telegram app said the bombing would give the British “crusaders” a taste of the explosives dropped on Raqqa by coalition forces. Isis claimed...Supporters of Islamic State took to social media to celebrate the Manchester Arena attack that left 22 dead last night, with hashtags such as #Just_Terror. Gloating posts on Facebook, Twitter and the Telegram app said the bombing would give the British “crusaders” a taste of the explosives dropped on Raqqa by coalition forces. Isis claimed...Supporters of Islamic State took to social media to celebrate the Manchester Arena attack that left 22 dead last night, with...

Read the full story

Armed police arrest ‘would-be suicide attackers’ in Spain

new

Two Moroccans who were allegedly poised to commit a suicide attack have been arrested in Madrid. The unnamed men, aged 43 and 22, were held by armed police in early morning raids today, just hours after the suicide attack in Manchester that left 22 people dead. The Spanish interior ministry said that both suspects...Two Moroccans who were allegedly poised to commit a suicide attack have been arrested in Madrid.Two Moroccans who were allegedly poised to commit a suicide attack have been arrested in Madrid.

Read the full story

City rallies together to care for wounded

new

In the hours after the explosion, taxi drivers gave free lifts to stranded teenagers, residents offered accommodation to strangers and queues formed at local blood banks as Mancunians rallied together. The hashtag #RoomForManchester spread across...In the hours after the explosion, taxi drivers gave free lifts to stranded teenagers, residents...In the hours after the explosion, taxi drivers gave free lifts to stranded teenagers, residents...

Read the full story

May’s flaws are now exposed for all to see

The U-turn on social care is symptomatic of a leadership style that ranks the views of her inner circle above the facts

Make a deal, Trump tells Israelis and Palestinians

new

President Trump said today that he was “personally committed” to helping Israelis and Palestinians reach a peace deal, but gave no specifics on how he might pursue that goal. Speaking at the Israel Museum in Jerusalem, Mr Trump condemned the suicide bombing in Manchester and denounced terrorism...President Trump said today that he was “personally committed” to helping Israelis and Palestinians reach a peace deal, but gave no specifics on how he might pursue that goal. Speaking at the Israel Museum in Jerusalem, Mr Trump condemned the suicide bombing in Manchester and denounced terrorism...President Trump said today that he was “personally committed” to helping Israelis and Palestinians reach a peace deal, but gave...

Read the full story

President steams in where predecessors feared to tread

Standard operating procedure for dealing with the Middle East, for American presidents, rule one: no nasty surprises. President Obama, looking over his predecessor’s record, put it more crudely. “Don’t do stupid s***.” On the thorny issue of the “Is-Pal issue”, where policy is embedded in contested history, powerful and argumentative...Standard operating procedure for dealing with the Middle East, for American presidents, rule one: no nasty surprises. President Obama, looking over his predecessor’s record, put it more crudely. “Don’t do stupid s***.” On the thorny issue of the “Is-Pal issue”, where policy is embedded in contested history, powerful and argumentative...Standard operating procedure for dealing with the Middle East, for American presidents, rule one: no nasty surprises. President...

Read the full story

Mob tries to lynch Atatürk statue vandal

new

A Turkish mob has tried to lynch a man who vandalised a statue of Kemal Atatürk, the republic’s founding father. The man poured petrol over the statue before hacking at it with an axe in Sakarya, western Turkey, yesterday morning. Witnesses say he was speaking in Arabic before he carried out the attack. A crowd of...A Turkish mob has tried to lynch a man who vandalised a statue of Kemal Atatürk, the republic’s...A Turkish mob has tried to lynch a man who vandalised a statue of Kemal Atatürk, the republic’s...

Read the full story

Inflation hits VAT receipts as families curb spending

Rising inflation is hitting the public finances as cash-strapped shoppers rein in their spending, hitting VAT receipts, according to the latest official figures. Borrowing rose in April to £10.4 billion — £1.2 billion more than in April 2016 and £1.6 billion more than economists’ forecasts. The...Rising inflation is hitting the public finances as cash-strapped shoppers rein in their spending, hitting VAT receipts, according to the latest official figures. Borrowing rose in April to £10.4 billion — £1.2 billion more than in April 2016 and £1.6 billion more than economists’ forecasts. The...Rising inflation is hitting the public finances as cash-strapped shoppers rein in their spending, hitting VAT receipts...

Read the full story

German boardrooms in a champagne mood

German business confidence has soared to its highest level since the reunification of the country more than a quarter of a century ago, boosted by the election of Emmanuel Macron in France. The monthly business climate index from the Institute for Economic Research (Ifo) in Munich, based on a monthly survey of 7,000 firms, beat expectations...German business confidence has soared to its highest level since the reunification of the country more than a quarter of a century ago, boosted by the election of Emmanuel Macron in France. The monthly business climate index from the Institute for Economic Research (Ifo) in Munich, based on a monthly survey of 7,000 firms, beat expectations...German business confidence has soared to its highest level since the reunification of the country more than a quarter of a...

Read the full story

Carney caught out by same email prankster who conned Barclays boss

new

Tight as the Bank of England’s cyber-security protocols for an institution with £100 billion of gold in its vaults must be, they proved no match for a prankster with a point to prove. Having claimed the scalp of Jes Staley, chief executive of Barclays, this month, an anonymous hoaxer aimed one higher and succeeding...Tight as the Bank of England’s cyber-security protocols for an institution with £100 billion of...Tight as the Bank of England’s cyber-security protocols for an institution with £100 billion of...

Read the full story

Allardyce resigns as Crystal Palace manager

new

Sam Allardyce stunned Crystal Palace today by resigning as manager barely a week after leading them to safety in the Premier League. The former England manager informed Steve Parish, the Palace chairman, of his decision this afternoon and the club are expected to confirm his departure in a...Sam Allardyce stunned Crystal Palace today by resigning as manager barely a week after leading them to safety in the Premier League. The former England manager informed Steve Parish, the Palace chairman, of his decision this afternoon and the club are expected to confirm his departure in a...Sam Allardyce stunned Crystal Palace today by resigning as manager barely a week after leading them to safety in the Premier...

Read the full story

Anderson a doubt for South Africa Test with groin tear

new

James Anderson is a doubt for England’s first Test of the summer against South Africa in July after it was confirmed today that he has torn his right groin. Anderson, 34, broke down in the middle of his seventh over during day one of Lancashire’s County Championship match against Yorkshire at Old Trafford on Friday. He did not bowl again in...James Anderson is a doubt for England’s first Test of the summer against South Africa in July after it was confirmed today that he has torn his right groin. Anderson, 34, broke down in the middle of his seventh over during day one of Lancashire’s County Championship match against Yorkshire at Old Trafford on Friday. He did not bowl again in...James Anderson is a doubt for England’s first Test of the summer against South Africa in July after it was confirmed today that...

Read the full story

Moyes had no plan, no control and no hope at Sunderland

It was supposed to be a return to his roots, taking comfort in what he was good at. After ten harrowing months at Manchester United, after 12 at Real Sociedad, David Moyes arrived at Sunderland last summer determined to reconstruct a failing club and rebuild his own reputation, to focus on the long term. But like...It was supposed to be a return to his roots, taking comfort in what he was good at. After ten...It was supposed to be a return to his roots, taking comfort in what he was good at. After ten...

Read the full story

Toe-curling exit belongs in wrestling, not football

Sport is unscripted drama. Its beauty emerges from the clash of competing wills, two teams or individuals going at...Sport is unscripted drama. Its beauty emerges from the clash of competing wills, two teams or...Sport is unscripted drama. Its beauty emerges from the clash of competing wills, two teams or...

Read the full story

Kvitova to play at Wimbledon six months after stabbing

Petra Kvitova is set to play at Wimbledon this summer six months after she was stabbed by an intruder in her home.Petra Kvitova is set to play at Wimbledon this summer six months after she was stabbed by an...Petra Kvitova is set to play at Wimbledon this summer six months after she was stabbed by an...

Read the full story

Sir Roger Moore

Debonair actor with a nice line in raised eyebrows who made millions as 007 in James Bond films and Simon Templar on TV

Sir Roger Moore may not have been the best Bond, indeed by his own estimation he was the fourth best, but off screen he was undoubtedly the most endearing of the actors who played 007. This likeability had much to do with his unwillingness, perhaps inability, to take himself too seriously. When he was cast in the 007 role, for example, he was asked what he thought he...Sir Roger Moore may not have been the best Bond, indeed by his own estimation he was the fourth...Sir Roger Moore may not have been the best Bond, indeed by his own estimation he was the fourth...

Read the full story

Roy Ackerman

Roy Ackerman’s dinner guest had been talking vividly to him for several minutes when the waiter arrived with two...Roy Ackerman’s dinner guest had been talking vividly to him for several minutes when the waiter arrived with two plates of food, but they were for someone else. “I told him that they were for the couple sitting at the corner table who, I advised him, were also waiting for their wine,” he said. “It was at this point...Roy Ackerman’s dinner guest had been talking vividly to him for several minutes when the waiter arrived with two plates of food...

Read the full story

Vinod Khanna

Alexina McWhinnie

When Alexina McWhinnie began researching adoption as her PhD subject in the 1950s, the conventional wisdom went as...When Alexina McWhinnie began researching adoption as her PhD subject in the 1950s, the...When Alexina McWhinnie began researching adoption as her PhD subject in the 1950s, the...

Read the full story

May 22

Buckingham Palace 22nd May, 2017 The Queen, Patron, Royal Horticultural Society, and The Duke of Edinburgh this...Buckingham Palace 22nd May, 2017 The Queen, Patron, Royal Horticultural Society, and The Duke of...Buckingham Palace 22nd May, 2017 The Queen, Patron, Royal Horticultural Society, and The Duke of...

Read the full story

The dark side of an elite Oxbridge education

A cocaine-fuelled attack by a student at Oxford has raised difficult questions

I’m crowdfunding to adopt an orphan I met by chance

Emilie Larter went to Africa in search of adventures — and ended up looking after a baby

Hot, young and ferociously talented: the stars of tomorrow

Our critics introduce their nominees for the Times Breakthrough award

Dr Mark Porter: We all need to know how to use a defibrillator

A battery scare has put them in the headlines, but their lack of use poses the bigger threat

Smear campaign against nurse forces Sturgeon to step in

Nicola Sturgeon has been forced to defend a party candidate accused of launching a “smear” campaign against a nurse who challenged the first minister over problems in the health service. Ms Sturgeon looked uncomfortable when she was questioned by Claire Austin on live television on Sunday evening...Nicola Sturgeon has been forced to defend a party candidate accused of launching a “smear” campaign against a nurse who challenged the first minister over problems in the health service. Ms Sturgeon looked uncomfortable when she was questioned by Claire Austin on live television on Sunday evening...Nicola Sturgeon has been forced to defend a party candidate accused of launching a “smear” campaign against a nurse who...

Read the full story

Dugdale stays cheerful in the face of adversity

It remains mind-popping to think that, in terms of the number of seats won, Labour might finish this Scottish campaign in fourth place. As mighty falls go, this takes some beating. It is difficult to attend a Labour Party event these days without thinking of Shelley’s Ozymandias: Look on Labour’s works and despair for “Nothing beside...It remains mind-popping to think that, in terms of the number of seats won, Labour might finish this Scottish campaign in fourth place. As mighty falls go, this takes some beating. It is difficult to attend a Labour Party event these days without thinking of Shelley’s Ozymandias: Look on Labour’s works and despair for “Nothing beside...It remains mind-popping to think that, in terms of the number of seats won, Labour might finish this Scottish campaign in...

Read the full story

Fire engulfs former hospital

Firefighters were tackling a large blaze last night in a former hospital in Edinburgh. Crews were called to the site of the former Royal Victoria Hospital at about 4.15pm, when thick smoke started billowing from the building. For hours last night a thick plume of smoke drifted across the north of the city as crews...Firefighters were tackling a large blaze last night in a former hospital in Edinburgh. Crews were...Firefighters were tackling a large blaze last night in a former hospital in Edinburgh. Crews were...

Read the full story

Ireland will need to tighten belt, EU warns

Ireland will have limited scope for tax cuts and spending increases in the October budget, the European Commission has warned. In a move that could create a showdown with the government, the commission said that Ireland was still highly indebted and the focus must be on reducing the national debt...Ireland will have limited scope for tax cuts and spending increases in the October budget, the European Commission has warned. In a move that could create a showdown with the government, the commission said that Ireland was still highly indebted and the focus must be on reducing the national debt...Ireland will have limited scope for tax cuts and spending increases in the October budget, the European Commission has warned.

Read the full story

Pay rises depend on reform, union talks told

The government has outlined the productivity reforms it is demanding in return for pay increases and warned that resources were limited for next year. Talks opened at the Workplace Relations Commission between management and unions yesterday with both sides aiming to agree an extension to the Lansdowne Road agreement within the next two weeks.The government has outlined the productivity reforms it is demanding in return for pay increases and warned that resources were limited for next year. Talks opened at the Workplace Relations Commission between management and unions yesterday with both sides aiming to agree an extension to the Lansdowne Road agreement within the next two weeks.The government has outlined the productivity reforms it is demanding in return for pay increases and warned that resources were...

Read the full story

Varadkar promises something for everyone

Tax cuts, a pensions boost and substantial increase in capital spending are on the agenda if Leo Varadkar becomes Fine Gael leader and taoiseach. The social protection minister and frontrunner in the bid to replace Enda Kenny unveiled his policy initiatives yesterday, which also include water refunds by the end of...Tax cuts, a pensions boost and substantial increase in capital spending are on the agenda if Leo...Tax cuts, a pensions boost and substantial increase in capital spending are on the agenda if Leo...

Read the full story
\ No newline at end of file diff --git a/test/testdata/60aa087f20252dedec62fcb65275a164648fe344.json b/test/testdata/60aa087f20252dedec62fcb65275a164648fe344.json new file mode 100644 index 00000000..73d8d256 --- /dev/null +++ b/test/testdata/60aa087f20252dedec62fcb65275a164648fe344.json @@ -0,0 +1,21 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Cache-Control": "max-age=0,private", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "61030", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:53:53 GMT", + "ETag": "W/\"74ca8-ayznDhyBSd6BIkGfacl29w\"", + "Expires": "Tue, 23 May 2017 17:53:53 GMT", + "Pragma": "no-cache", + "Set-Cookie": "ak_bmsc=E5A85E39B477E2CF8661D6AB4B0441DDB833657DCA340000317724594BF20204~plMNXsXa2LfQpwa98FmX1lU0TALNOcXs29cWZCnGaUSJicXuLLmJEHpIyflXOQa9qxT3BzP7m9vo/nSSyTnrlq+9yn42C2cQ/szabxifIeTwGOCyon4rrseORVAtScS3AU6M3VuaMfa0Lf6YH2rl6r5LGfUEqhXhZJ9Y2abRYog//yZ9t71ZwNWgA/U6zWzccyklDkVA8v/1Kicz8XgrIQk257WIDhGm0q8e+PTDWxR5g=; expires=Tue, 23 May 2017 19:53:53 GMT; max-age=7200; path=/; domain=.thetimes.co.uk; HttpOnly", + "Vary": "Accept-Encoding", + "X-NU-AKA-ACS-Version": "2.0, 2.0", + "X-Varnish": "1054357" + }, + "status_code": 200, + "url": "https://www.thetimes.co.uk/" +} \ No newline at end of file diff --git a/test/testdata/6217e94d808fcf0c594dca614a5847975789f7d5.html b/test/testdata/6217e94d808fcf0c594dca614a5847975789f7d5.html new file mode 100644 index 00000000..b6bf55c3 --- /dev/null +++ b/test/testdata/6217e94d808fcf0c594dca614a5847975789f7d5.html @@ -0,0 +1,730 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + تحلیل منافع بهره وری ناشی از اصلاحات صنعت برق استرالیا: چارچوب های روش شناختی - پایگاه مجلات تخصصی نور + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Skip to main content + + + + + + +
+ + + + +
+ + + +
+
+ + + + + + + +
+
+ + + + + +
+
+ فهرست مقالات +
+ +

+ تحلیل منافع بهره وری ناشی از اصلاحات صنعت برق استرالیا: چارچوب های روش شناختی +

+
+
+ + + + + +
+
+ +
+

+ نویسنده: + +

+
+

+ + + (1 صفحه - از 55 تا 55) +

+
+ +
+ + + +
+
+ + + + + + +
+
+
+
+ +
+
+
+
+
+
+

+ کلیدواژه ها : + بهره‌وری + ،اصلاحات + ،برق +

+

+ کلید واژه های ماشینی : + چارچوب روش‌شناختی تحلیل منافع بهره‌وری + ،اصلاحات صنعت برق استرالیا + ،بهبود چارچوب روش‌شناختی تحلیل منافع + ،اصلاحاتی بهره‌وری صنعت برق + ،خصوصی‌سازی بخش قابل‌ملاحظه‌ای از صنعت + ،چارچوب جدید مقرراتی و خصوصی‌سازی + ،صنعت برق استرالیا متمرکز + ،تحلیل منافع بهره‌وری ناشی + ،بخش قابل‌ملاحظه‌ای از صنعت برق + ،منافع بهره‌وری ناشی از سیاست‌گذاری + ،اصلاحات + ،صنایع + ،مقرراتی و خصوصی‌سازی بخش + ،ری ناشی از اصلاحات صنعت + ،روش‌شناختی + ،اصلی + ،دهه از اصلاحات صنعت + ،ماورای + ،تحلیل حاضر بر صنعت برق + ،تحلیل منافع بهره و ری + ،منافع بهره و ری ناشی + ،مطالعات به ارایه توصیه‌هایی + ،کشورها قابل تعمیم + ،تجدید ساختار بازارهای برق + ،کاربردهای روش‌شناختی + ،توسعه + ،سطح کلان کمک + ،جنبه‌های اصلی + ،رشد اقتصادی در سطح + ،بحث اصلی +

+
+
+
+
+
+
+
+
+
+
+

+ اکنون بیش از یک دهه از اصلاحات صنعت برق استرالیا می‌گذارد.جنبه‌های اصلی این +اصلاحات عبارتنداز:تجدید ساختار بازارهای برق، توسعه یک چارچوب جدید مقرراتی و +خصوصی‌سازی بخش قابل‌ملاحظه‌ای از صنعت برق.بحث اصلی ماورای این اصلاحات-همانند +سایر نقاط دنیا-اساسا بر این استدلال استوار بوده است که چنین اصلاحاتی بهره‌وری +صنعت برق را بهبود بخشیده و از این طریق به رشد اقتصادی در سطح کلان کمک خواهد +کرد.مطالعات متعددی در خلال سال‌های اخیر انجام شده است تا منافع بهره‌وری ناشی از +این اصلاحات را مورد ارزیابی قرار دهد.این مقاله ضمن مرور این مطالعات به ارایه +توصیه‌هایی برای بهبود چارچوب روش شناختی تحلیل منافع بهره‌وری ناشی از سیاست‌گذاری +می‌پردازد.اگرچه تحلیل حاضر بر صنعت برق استرالیا متمرکز است، اما کاربردهای روش +شناختی آن می‌تواند برای سایر صنایع و همچنین سایر کشورها قابل تعمیم باشد. +

+ خلاصه ماشینی: +

+ "جنبه‌های اصلی این اصلاحات عبارتنداز:تجدید ساختار بازارهای برق، توسعه یک چارچوب جدید مقرراتی و خصوصی‌سازی بخش قابل‌ملاحظه‌ای از صنعت برق. +بحث اصلی ماورای این اصلاحات-همانند سایر نقاط دنیا-اساسا بر این استدلال استوار بوده است که چنین اصلاحاتی بهره‌وری صنعت برق را بهبود بخشیده و از این طریق به رشد اقتصادی در سطح کلان کمک خواهد کرد. +این مقاله ضمن مرور این مطالعات به ارایه توصیه‌هایی برای بهبود چارچوب روش شناختی تحلیل منافع بهره‌وری ناشی از سیاست‌گذاری می‌پردازد." + +

+
+
+
+
+
+ +
+
+
+ +
+
+
+
+
+
+ +
+
+
+
+
+
+ +
+
+
+
+
+

+ +

+
+
+
+
+ + + + +
+ +
+ +
+
+
    +
  • +
    + دانلود HTML +
    + +
  • +
  • +
    + دانلود PDF +
    + +
  • +
+
+
+ +
+ +
+ +

+ برای مشاهده محتوای مقاله لازم است وارد پایگاه شوید. در صورتی که عضو نیستید از قسمت عضویت اقدام فرمایید. +

+
+
+
+ + + + cloob + + + + + + + +
+
+ + +
+
+
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + +
+ + +
+ + + +
+ + +
+ + diff --git a/test/testdata/6217e94d808fcf0c594dca614a5847975789f7d5.json b/test/testdata/6217e94d808fcf0c594dca614a5847975789f7d5.json new file mode 100644 index 00000000..b5df062b --- /dev/null +++ b/test/testdata/6217e94d808fcf0c594dca614a5847975789f7d5.json @@ -0,0 +1,15 @@ +{ + "encoding": "utf-8", + "headers": { + "Cache-Control": "private", + "Content-Encoding": "gzip", + "Content-Length": "10550", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:51:54 GMT", + "Set-Cookie": "CRCIS_SessionId=x3qnfv44zfmmar1fjmetmay4; path=/, __RequestVerificationToken=1b_dxVRwwqSEoUMoIXevFeWxJWejquWWEL_NkbjvRnDTYnqsAjVnIES4fnFDhIZH4RTGcmAZYMOE3QCjVc_Nc5imaDUwFmfbv89XHOkZ1Nk1; path=/; HttpOnly, .ASPXBrowserOverride=Mozilla%2f4.0+(compatible%3b+MSIE+6.0%3b+Windows+CE%3b+IEMobile+8.12%3b+MSIEMobile+6.0); expires=Tue, 30-May-2017 17:51:55 GMT; path=/", + "X-Download-Options": "noopen", + "X-Frame-Options": "SAMEORIGIN" + }, + "status_code": 200, + "url": "http://www.noormags.ir/view/fa/articlepage/105489/%d8%aa%d8%ad%d9%84%db%8c%d9%84-%d9%85%d9%86%d8%a7%d9%81%d8%b9-%d8%a8%d9%87%d8%b1%d9%87-%d9%88%d8%b1%db%8c-%d9%86%d8%a7%d8%b4%db%8c-%d8%a7%d8%b2-%d8%a7%d8%b5%d9%84%d8%a7%d8%ad%d8%a7%d8%aa-%d8%b5%d9%86%d8%b9%d8%aa-%d8%a8%d8%b1%d9%82-%d8%a7%d8%b3%d8%aa%d8%b1%d8%a7%d9%84%db%8c%d8%a7--%da%86%d8%a7%d8%b1%da%86%d9%88%d8%a8-%d9%87%d8%a7%db%8c-%d8%b1%d9%88%d8%b4-%d8%b4%d9%86%d8%a7%d8%ae%d8%aa%db%8c?q=%D8%A8%D8%B1%D9%82&score=21.639421&rownumber=1" +} \ No newline at end of file diff --git a/test/testdata/62d5905f4e5d6ebd3d4de2537d1bd64747195dae.html b/test/testdata/62d5905f4e5d6ebd3d4de2537d1bd64747195dae.html new file mode 100644 index 00000000..d6057328 --- /dev/null +++ b/test/testdata/62d5905f4e5d6ebd3d4de2537d1bd64747195dae.html @@ -0,0 +1,591 @@ +'Star Wars': Disney+ switches up controversial Han Solo/Greedo sceneSkip to main content +

The infamous 'Han shot first' scene in 'Star Wars' has changed yet again on Disney+


play
Show Caption + +

One of the most controversial scenes in the galaxy far, far away just got inexplicably more complicated.

A key early moment in George Lucas' 1977 original "Star Wars" movie features Han Solo gunning down bad guy Greedo in the Mos Eisley cantina before heading off into space with Luke, Leia and the gang to take on the evil Empire. Who shot first and when has been at the center of a slew of changes over four decades that have irked the hardcore "Star Wars" faithful, and an alteration in a new cut streaming on Disney+ muddies up the affair even more.

"Did Han shoot first?" has been debated for years. In the original cut, lovable rogue Han (Harrison Ford) guns down Greedo, but in a 1997 special edition, Lucas edited the movie to make it seem like Greedo was the one who fired first, making Solo look a little more heroic but irking fans in the process. In an interview with The Hollywood Reporter in 2012, Lucas said he wanted to "clean up the confusion. ... Obviously, it upset people because they wanted Solo to be a coldblooded killer, but he actually isn't."

Disney+ review: First 'Star Wars' live-action TV series 'The Mandalorian' doesn't rule the galaxy

More: Here are all the new Disney+ shows and movies, from 'Mandalorian' to 'High School Musical'

In 2004, for an updated DVD release of Lucas' first "Star Wars" trilogy, the sequence was changed again to show Greedo firing just a hair before Han. 

Much of the scene is the same in the updated Disney+ version, except Greedo says, "Maclunkey" – we're still translating that one – and he and Han shoot simultaneously. (Luckily for the rest of the movies to follow, Han is still a much better shot.)

The scene still reflects the 2004 version on other streaming platforms, including iTunes.

Lucasfilm confirmed to USA TODAY that the new change in the Disney+ version was made by Lucas before Disney's $4 billion acquisition of his company in 2012.

Social media had a field day with the alteration, which generated just as much conversation as the ballyhooed new "Star Wars" TV series "The Mandalorian." 

New York Times culture reporter Dave Itzkoff used the kerfuffle to make a timely "OK Boomer" joke.

Writer Richard Littler compared it to changing other classic movies, like Michael Corleone playing a kazoo in "The Godfather II." 

And CNN media reporter Frank Pallotta joked that he's looking forward to the next big changes 20 years down the line, "where Han and Greedo hug."

\ No newline at end of file diff --git a/test/testdata/62d5905f4e5d6ebd3d4de2537d1bd64747195dae.json b/test/testdata/62d5905f4e5d6ebd3d4de2537d1bd64747195dae.json new file mode 100644 index 00000000..9d98d820 --- /dev/null +++ b/test/testdata/62d5905f4e5d6ebd3d4de2537d1bd64747195dae.json @@ -0,0 +1,36 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "0", + "Cache-Control": "no-store", + "Connection": "keep-alive", + "Content-Encoding": "br", + "Content-Length": "46973", + "Content-Security-Policy": "upgrade-insecure-requests;frame-ancestors 'none';object-src 'none'", + "Content-Security-Policy-Report-Only": "script-src https: blob: 'unsafe-inline' 'unsafe-eval' 'self';base-uri 'self';report-uri https://reporting-api.gannettinnovation.com;report-to default", + "Content-Type": "text/html; charset=utf-8", + "Cross-Origin-Opener-Policy": "same-origin", + "Cross-Origin-Resource-Policy": "same-origin", + "Date": "Mon, 18 Apr 2022 16:33:17 GMT", + "Feature-Policy": "camera 'none';display-capture 'none';geolocation 'none';microphone 'none';payment 'none';usb 'none';xr-spatial-tracking 'none'", + "Gannett-Cam-Experience-Id": "control:8", + "NEL": "{\"report_to\":\"default\",\"max_age\":31557600,\"include_subdomains\":true,\"success_fraction\":0.005}", + "Origin-Agent-Cluster": "?1", + "Permissions-Policy": "camera=(),display-capture=(),geolocation=(),microphone=(),payment=(),usb=(),xr-spatial-tracking=()", + "Referrer-Policy": "strict-origin-when-cross-origin", + "Report-to": "{\"max_age\":31557600,\"include_subdomains\":true,\"endpoints\":[{\"url\":\"https://reporting-api.gannettinnovation.com\"}]}", + "Set-Cookie": "gup_anonid=171eb396-00ca-4ffe-a643-7ab082d0ff43; Domain=.usatoday.com; Max-Age=31536000; Path=/; SameSite=Lax; Secure; Priority=High, gup_clientid=f29de158-b5d6-43b6-a5eb-c9fd95244050; Domain=.usatoday.com; Max-Age=31536000; Path=/; SameSite=Lax; Secure; Priority=High, gnt_eid=control:8; domain=.usatoday.com; path=/; secure; samesite=lax; max-age=5184000;", + "Strict-Transport-Security": "max-age=63072000", + "Vary": "X-AbVariant,X-AbVCfg,X-AltUrl,Accept-Encoding,User-Agent", + "X-Cache": "MISS, MISS", + "X-Content-Type-Options": "nosniff", + "X-Frame-Options": "deny", + "X-Timer": "S1650299597.492195,VS0,VE258", + "X-XSS-Protection": "1; mode=block", + "etag": "W/\"2e2c1-8v9YG/+xHBQMFLx2KoWPzkJvvvk\"", + "link": ";rel=preload;as=image;nopush" + }, + "status_code": 200, + "url": "https://www.usatoday.com/story/entertainment/movies/2019/11/12/star-wars-disney-plus-changes-controversial-han-solo-greedo-scene/2576097001/" +} \ No newline at end of file diff --git a/test/testdata/6609857609979b65e8866a0de30e7389fa701ee5.html b/test/testdata/6609857609979b65e8866a0de30e7389fa701ee5.html new file mode 100644 index 00000000..c56bdb57 --- /dev/null +++ b/test/testdata/6609857609979b65e8866a0de30e7389fa701ee5.html @@ -0,0 +1,693 @@ + + + + + + 101-راه-برای-اینکه-پدر-بهتری-باشید | پیک-ادبیات | خانه کتاب و ادبیات ایران + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + +
+
+
+
+ + + + + + + +
+ + + + + +
+ + + ورود + + ثبت نام + + + + + + +
+
+
+
+ + +
+ + + + +
+
+
+
+
+ 101 راه برای اینکه پدر بهتری باشید | خانه کتاب و ادبیات ایران +
+
+
+ صفحات اولیه کتاب +

+ 101 راه برای اینکه پدر بهتری باشید

+

+ + + پدر و کودک - مسائل متفرقه + + + پدری - مسائل متفرقه + + +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
پدیدآور + + نويسنده : + + لانسکی ، ویکی + - + + + مترجم : + + دالکی ، فیروزه + - + + + مترجم : + + امیرفروغی ، مژگان + - + + + تصويرگر : + + وایت ، کی + + +
ناشر + + + + پیک ادبیات + + (برای تماس با ناشر و خرید کتاب کلیک کنید) + + + + +
شابک978-964-8165-81-4
تاریخ نشر + +13860605 +
قیمت +
کد دیویی306.8742
زبان کتابفارسی
محل نشرتهران - تهران
توضیحات + جلد - + 112 صفحه - + ترجمه - + چاپ 1 +
+
+
+
+
+
+
+
+
+
معرفی مختصر کتاب
+

+ تاثیر تربیتی پدران با تاثیر تربیتی مادران در تربیت فرزندان کاملا متفاوت است. اما برخی اوقات پدران نقش تربیتی خویش را به دلیل مشغولیت‌های روزانه‌ی زندگی، فراموش می‌کنند. نگارنده در کتاب حاضر 101 راه‌کار را به پدران معرفی می‌کند که با استفاده از آن می‌توان کودکان و پدران را به یک‌دیگر نزدیک کرد. این ایده‌ها برگرفته از تجربیات پدرانی است که از آن‌ها برای ایجاد رابطه‌ی مناسب با کودکان خویش استفاده کرده‌اند. برخی از این راه‌کارها عبارت‌اند از: زمان‌هایی را برای با خانواده بودن، در نظر بگیرید؛ گاهی کار را کنار گذاشته و زمان را به کودکان اختصاص دهید؛ از جملاتی چون می‌فهمم چه می‌گویی، دوستت دارم، اشتباه کردم و متاسفم استفاده کنید؛ و به مادر کودکتان احترام گذاشته و به او عشق بورزید. +

+
+
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/testdata/6609857609979b65e8866a0de30e7389fa701ee5.json b/test/testdata/6609857609979b65e8866a0de30e7389fa701ee5.json new file mode 100644 index 00000000..6241e27a --- /dev/null +++ b/test/testdata/6609857609979b65e8866a0de30e7389fa701ee5.json @@ -0,0 +1,22 @@ +{ + "encoding": "utf-8", + "headers": { + "AR-ATIME": "0.108", + "AR-CACHE": "BYPASS", + "AR-PoweredBy": "Arvan Cloud (arvancloud.com)", + "AR-Request-ID": "ae4c864722ff77c0dd1722b726924635", + "AR-SID": "2000", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Security-Policy": "upgrade-insecure-requests", + "Content-Type": "text/html; charset=utf-8", + "Date": "Sat, 27 Aug 2022 22:31:55 GMT", + "Keep-Alive": "timeout=65", + "Server": "ArvanCloud", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding, Accept-Encoding", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://ketab.ir/book/27b3444f-1175-4db0-8411-b1719a5d7ed1" +} \ No newline at end of file diff --git a/test/testdata/6643d1625f0703c8f7a2830963be66760069c90e.html b/test/testdata/6643d1625f0703c8f7a2830963be66760069c90e.html new file mode 100644 index 00000000..5694014d --- /dev/null +++ b/test/testdata/6643d1625f0703c8f7a2830963be66760069c90e.html @@ -0,0 +1,1032 @@ + + + + + + Walsh meets with college leaders on off-campus housing - The Boston Globe + + + + + +Walsh meets with college leaders on off-campus housing - The Boston Globe + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + +
+
+ + +
+ + +
+ Menu + +
+
+ + +
+ + +
Metro
+ +
+ + + + + + +
+ + + + +
+
+ +
+
+ + + + +
+
+

Walsh meets with college leaders on off-campus housing

+
+ +
+
+ + + + + + +
+ Mayor Walsh met with the representatives of BU, BC, Northeastern, Suffolk and several other city colleges at the Parkman House, the city-owned mansion on Beacon Hill. +
+ +

John Tlumacki/Globe Staff

+

Mayor Walsh met with the representatives of BU, BC, Northeastern, Suffolk and several other city colleges at the Parkman House, the city-owned mansion on Beacon Hill.

+
+
+
+ + + + +

Mayor Martin J. Walsh said today the leaders of Boston’s largest universities have largely agreed to disclose the addresses of their off-campus students — a long-resisted step that he called critical to combat chronic overcrowded conditions that he said imperils their safety.

“They don’t see a problem with it,’’ Walsh said after emerging from an hour-long meeting with more than two dozen college officials. “Not one college pushed back.’’

+ Advertisement +

+ +
+ +

The mayor met with the representatives of Boston University, Boston College, Northeastern University, Suffolk University and several other city colleges at the Parkman House, the city-owned mansion on Beacon Hill.

Walsh had said that if he met with resistance from university leaders he would explore legislative action to force disclosure of off-campus addresses through a city ordinance, a measure he said needed to protect the health and safety of tens of thousands of university students.

+
+
+
+ Get Fast Forward in your inbox: +
+
+ Forget yesterday's news. Get what you need today in this early-morning email.
+
+
+
+ + +
+ +
+
+
+
+ Thank you for signing up! + Sign up for more newsletters here +
+
+ +

But the mayor said he heard no objections from any college official, several of whom pledged speedy compliance.

+ + +

+ +

“It’s fine. It’s not a problem,’’ said James McCarthy, Suffolk University’s president. “We’ll move immediately to get the addresses that they’re looking for. ... We appreciate the attention being given to this. This is important to our students.’’

Walsh’s session with college officials comes one month after the Globe Spotlight Team uncovered widespread problems in Boston’s college neighborhoods, where students confront living hazards ranging from rats and bed bugs, to smoke detectors dangling uselessly from ceilings, to bedrooms stuffed illegally into basements or firetrap attics.

+

+ Advertisement + +

+ +
+ +

The hazardous overcrowded conditions are especially acute in sections of Brighton near Boston College, where a Spotlight Team survey found 80 percent of the students questioned said they had more than four undergraduates in their apartments.

A city zoning provision prohibits more than four, full-time undergraduates from sharing a house or apartment — a provision that the Globe investigation found is widely ignored by students and winked at by landlords and their property managers.

After a fire last year that killed 22-year-old Binland Lee, a Boston University student, community activists called on city universities to release the addresses of their off-campus students to enable the city to build a database to detect dangerous, overcrowded living conditions.

Only Boston University complied.

Boston College said federal student privacy laws do not allow the school to disclose where their students live off campus, even though federal regulators say schools that designate addresses as directory information are generally permitted to do so.

Before today’s session with Walsh, Northeastern University said its legal counsel was studying the issue.

But Walsh said those objections were not voiced today when college officials said they would comply with his request.

“We’ll do that,’’ said Sister Janet Eisner, president of Emmanuel College. “I think that’s the sense of everyone there.’’

The mayor was joined at today’s session by Police Commissioner William B. Evans and William Christopher, the newly installed commissioner of the city’s Inspectional Services Department.

Walsh has said he wants Boston’s colleges and universities to live up to promises they have made to move students back to their campuses and out of neighborhoods. Rowdy behavior and exorbitant rents are seen as disruptive forces for long-time neighborhood residents. They say the flood of college tenants into overcrowded units is rendering housing unaffordable to them while endangering young, student renters.

The number of undergraduate and graduate students living off campus in the city has soared 36 percent to more than 45,000 from 2006 to 2013, according to a Spotlight Team analysis of reports private colleges submit to Boston’s city clerk and data provided to the Globe by three public schools.

Globe reporters and correspondents canvassed block after block of student rental apartments and found overcrowding rampant. In a Spotlight survey of 266 students living off campus in Boston, nearly a third of respondents said at least five undergraduates were living together, in apparent violation of the 2008 zoning rule.

In addition to demanding the off-campus addresses from the colleges, Walsh has vowed to increase the number of city inspectors to check the city’s 154,000 rental units for possible code infractions. And he has said he will begin to levy $300 daily fines on scofflaw landlords who repeatedly violate city and state regulations but escape harsh penalties because of toothless enforcement practices.

McCarthy, the Suffolk president, said he expects a series of meetings to be convened in short order to determine how best to get the address data to the city. He said another meeting will be held before students return to the city for the fall semester.

+ + + +
+

+ +
+ + +
+ +
+
+
+ Loading comments... +
+
+
+
+
+
+ + +
+
+ + + + + + + + + + + + + +
+ + + + + + +
+
+
+ +
+
+ We hope you've enjoyed your free articles. +
+
+ Continue reading by subscribing to Globe.com for just 99¢. +
+
+  Already a member? Log in Home +
+
+
+
+
+ + + + + + + + + + +
+
+ +
+ Subscriber Log In +

We hope you've enjoyed your 5 free articles'

+ + + +
+
+
+

Stay informed with unlimited access to Boston’s trusted news source.

+
    +
  • High-quality journalism from the region’s largest newsroom
  • +
  • Convenient access across all of your devices
  • +
  • Today’s Headlines daily newsletter
  • +
  • Subscriber-only access to exclusive offers, events, contests, eBooks, and more
  • +
  • Less than 25¢ a week
  • +
+
+ +
+
+ Marketing image of BostonGlobe.com +
+
+
+ +
+ +
+
+
+ + + + + +
+
+ +
+
+ +
+
+ Marketing image of BostonGlobe.com +
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + diff --git a/test/testdata/6643d1625f0703c8f7a2830963be66760069c90e.json b/test/testdata/6643d1625f0703c8f7a2830963be66760069c90e.json new file mode 100644 index 00000000..e13250df --- /dev/null +++ b/test/testdata/6643d1625f0703c8f7a2830963be66760069c90e.json @@ -0,0 +1,26 @@ +{ + "encoding": "UTF-8", + "headers": { + "Accept-Ranges": "bytes", + "Cache-Control": "no-cache, must-revalidate, max-age=0, no-store", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "19150", + "Content-Type": "text/html;charset=UTF-8", + "Date": "Tue, 23 May 2017 17:53:04 GMT", + "Eomportal-Instance": "216", + "Expires": "Thu, 01 Jan 1970 00:00:00 GMT", + "Pragma": "no-cache", + "Server": "BostonGlobe.com Frontend", + "Set-Cookie": "JSESSIONID=E0C04C3E5C92D80A5F37F57EA16BE8EE; Path=/, pathUrl=/metro/2014/06/03/walsh-meets-with-college-leaders-off-campus-housing/lsxtLSGJMD86Gbkjay3D6J/story.html; Domain=.bostonglobe.com; Path=/, FM=20170523:1:3.2.4143921833; Domain=.bostonglobe.com; Expires=Fri, 07-Jul-2017 17:53:05 GMT; Path=/", + "Vary": "Origin, Accept-Encoding", + "Via": "1.1 varnish", + "X-Cache": "MISS", + "X-Cache-Hits": "0", + "X-Served-By": "cache-iad2620-IAD", + "X-TTL": "pass", + "X-Timer": "S1495561984.291190,VS0,VE521" + }, + "status_code": 200, + "url": "http://www.bostonglobe.com/metro/2014/06/03/walsh-meets-with-college-leaders-off-campus-housing/lsxtLSGJMD86Gbkjay3D6J/story.html" +} \ No newline at end of file diff --git a/test/testdata/6673dab6274696f175f517b9708d1ff953cbcf18.html b/test/testdata/6673dab6274696f175f517b9708d1ff953cbcf18.html new file mode 100644 index 00000000..be9fdd50 --- /dev/null +++ b/test/testdata/6673dab6274696f175f517b9708d1ff953cbcf18.html @@ -0,0 +1,19 @@ + +TY - JOUR +T1 - بررسی فضایل قرآنی در دعای ابوحمزه ثمالی +T2 - بینات (موسسه معارف اسلامی امام رضا علیه السلام) +JF - بینات (موسسه معارف اسلامی امام رضا علیه السلام) +Y1 - 1389/// + +LA - fa +UR - http://www.noormags.ir/view/fa/articlepage/692447 +SP - 103 +EP - 124 +SN - +VL - 68 +IS - 17 + +ID - 692447 +AU - سلیمانی‌میمند,‌مریم + +ER - diff --git a/test/testdata/6673dab6274696f175f517b9708d1ff953cbcf18.json b/test/testdata/6673dab6274696f175f517b9708d1ff953cbcf18.json new file mode 100644 index 00000000..85fabcbc --- /dev/null +++ b/test/testdata/6673dab6274696f175f517b9708d1ff953cbcf18.json @@ -0,0 +1,15 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "private", + "Content-Disposition": "attachment; filename=\"noormags-692447.ris\"", + "Content-Length": "487", + "Content-Type": "application/x-Research-Info-Systems; charset=UTF-8", + "Date": "Tue, 23 May 2017 17:51:57 GMT", + "Set-Cookie": "CRCIS_SessionId=xraopgplfryhupptyrwiidp2; path=/, .ASPXBrowserOverride=Mozilla%2f4.0+(compatible%3b+MSIE+6.0%3b+Windows+CE%3b+IEMobile+8.12%3b+MSIEMobile+6.0); expires=Tue, 30-May-2017 17:51:57 GMT; path=/", + "X-Download-Options": "noopen", + "X-Frame-Options": "SAMEORIGIN" + }, + "status_code": 200, + "url": "http://www.noormags.ir/view/fa/citation/ris/692447" +} \ No newline at end of file diff --git a/test/testdata/6763912a77ee3861422300a8895252a6d997023a.html b/test/testdata/6763912a77ee3861422300a8895252a6d997023a.html new file mode 100644 index 00000000..66ad8dc7 --- /dev/null +++ b/test/testdata/6763912a77ee3861422300a8895252a6d997023a.html @@ -0,0 +1,1394 @@ + + + + + + + + + + + + + + + + Boston Magazine + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ + +
+
+ +
+ + Advertisement + +
+ + + +
+ +
+ +
+
+ + +
+ +
+ +
+ +
+ + + + +
+ + + + +
+ + +
+ +
+ +
+ + + + + + +
+ + +
 
+ +
+ + +
+ + +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/testdata/6763912a77ee3861422300a8895252a6d997023a.json b/test/testdata/6763912a77ee3861422300a8895252a6d997023a.json new file mode 100644 index 00000000..a33a0496 --- /dev/null +++ b/test/testdata/6763912a77ee3861422300a8895252a6d997023a.json @@ -0,0 +1,22 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "max-age=281, must-revalidate", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Type": "text/html; charset=UTF-8", + "Date": "Tue, 23 May 2017 17:53:05 GMT", + "Last-Modified": "Tue, 23 May 2017 17:39:27 GMT", + "Link": "; rel=\"https://api.w.org/\"", + "Server": "nginx", + "Transfer-Encoding": "chunked", + "Vary": "Cookie", + "X-Cache": "HIT", + "X-Powered-By": "PHP/5.6.30", + "X-batcache": "True", + "X-device": "_desktop_", + "x-cache-key": "http://_desktop_www.bostonmagazine.comGET/" + }, + "status_code": 200, + "url": "http://www.bostonmagazine.com/" +} \ No newline at end of file diff --git a/test/testdata/6c5928e29d75168fe7b080b704e0d12a1436dbff.html b/test/testdata/6c5928e29d75168fe7b080b704e0d12a1436dbff.html new file mode 100644 index 00000000..76813183 --- /dev/null +++ b/test/testdata/6c5928e29d75168fe7b080b704e0d12a1436dbff.html @@ -0,0 +1,260 @@ +News: US News, Top News in India, US election news, Business news, Sports & International News | Times of India
\ No newline at end of file diff --git a/test/testdata/6c5928e29d75168fe7b080b704e0d12a1436dbff.json b/test/testdata/6c5928e29d75168fe7b080b704e0d12a1436dbff.json new file mode 100644 index 00000000..f2570bc4 --- /dev/null +++ b/test/testdata/6c5928e29d75168fe7b080b704e0d12a1436dbff.json @@ -0,0 +1,27 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Access-Control-Allow-Credentials": "false", + "Access-Control-Allow-Headers": "Origin,X-Requested-With,Content-Type,Accept", + "Access-Control-Allow-Methods": "GET,POST", + "Access-Control-Max-Age": "86400", + "Cache-Control": "max-age=0, no-cache, no-store", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "54066", + "Content-Type": "text/html; charset=utf-8", + "Date": "Wed, 21 Jul 2021 03:23:42 GMT", + "ETag": "\"02d11062438b692d7b6d2f91fa5f3a85\"", + "Expires": "Wed, 21 Jul 2021 03:23:42 GMT", + "Last-Modified": "Wed, 21 Jul 2021 03:22:52 GMT", + "Pragma": "no-cache", + "Server": "AmazonS3", + "Strict-Transport-Security": "max-age=86400", + "Vary": "Accept-Encoding", + "x-amz-id-2": "MScAgV9gZt4vk6teZMU9noSZ1m5n41+2n+bfbiwVcFucuqmaJyXygtHqEzawL7RWslrbmFUPdys=", + "x-amz-request-id": "NQJM8S3TJPV5EW5W" + }, + "status_code": 200, + "url": "https://timesofindia.indiatimes.com/us" +} \ No newline at end of file diff --git a/test/testdata/6d81ee8f462fe9be3e5e2a75f3531c4db5bf0e85.html b/test/testdata/6d81ee8f462fe9be3e5e2a75f3531c4db5bf0e85.html new file mode 100644 index 00000000..760f25ef --- /dev/null +++ b/test/testdata/6d81ee8f462fe9be3e5e2a75f3531c4db5bf0e85.html @@ -0,0 +1,9 @@ +@techreport{10.2307/resrep26363.7, + URL = {http://www.jstor.org/stable/resrep26363.7}, + author = {Spandana Singh and Margerite Blase}, + institution = {New America}, + pages = {22--26}, + title = {Protecting the Vote: How Internet Platforms Are Addressing Election and Voter Suppression-Related Misinformation and Disinformation}, + year = {2020} +} + diff --git a/test/testdata/6d81ee8f462fe9be3e5e2a75f3531c4db5bf0e85.json b/test/testdata/6d81ee8f462fe9be3e5e2a75f3531c4db5bf0e85.json new file mode 100644 index 00000000..bb016513 --- /dev/null +++ b/test/testdata/6d81ee8f462fe9be3e5e2a75f3531c4db5bf0e85.json @@ -0,0 +1,24 @@ +{ + "encoding": "ISO-8859-1", + "headers": { + "Accept-Ranges": "bytes", + "Connection": "keep-alive", + "Content-Disposition": "attachment;filename=10.2307_resrep26363.7.txt;", + "Content-Encoding": "gzip", + "Content-Type": "text/plain", + "Date": "Fri, 28 May 2021 11:39:36 GMT", + "Server": "Apache/2.4.29 (Ubuntu)", + "Set-Cookie": "ReferringRequestId=citation-export:7a20139ad2b56952af901346b7aa88de; Path=/; SameSite=Lax; Secure", + "Vary": "Cookie,Accept-Encoding,Fastly-SSL,Origin,X-Requested-Host", + "Via": "1.1 varnish", + "X-Cache": "MISS", + "X-Cache-Hits": "0", + "X-Frame-Options": "SAMEORIGIN", + "X-JSTOR-Restarts": "0", + "X-Served-By": "cache-fra19176-FRA", + "X-Timer": "S1622201976.985909,VS0,VE712", + "transfer-encoding": "chunked" + }, + "status_code": 200, + "url": "https://www.jstor.org/citation/text/resrep26363.7" +} \ No newline at end of file diff --git a/test/testdata/710ef2de4a3f0d569273eeab117cf7841465c754.html b/test/testdata/710ef2de4a3f0d569273eeab117cf7841465c754.html new file mode 100644 index 00000000..431fe231 --- /dev/null +++ b/test/testdata/710ef2de4a3f0d569273eeab117cf7841465c754.html @@ -0,0 +1 @@ +Spill spews tons of coal ash into North Carolina's Dan River - CNN

Spill spews tons of coal ash into North Carolina river

Story highlights

  • A broken stormwater pipe sent millions of gallons of sludge into the Dan River
  • Duke Energy and regulators are coming up with a cleanup plan
  • Environmental activists have concerns about drinking water and wildlife
  • The river supplies water to communities in Virginia and North Carolina
The coal ash poured out of a broken pipe into the Dan River, turning water into dark muck.
It took nearly a week to stem the spill, which sent millions of gallons of sludge from a retired power plant into a river that supplies drinking water to communities in North Carolina and neighboring Virginia.
Workers stopped the spill by plugging the broken pipe with concrete this weekend. Now government scientists and the United States' largest electric utility face a daunting task: cleaning it up.
Tests since the spill have turned up higher levels of harmful chemicals such as arsenic in the river. But so far, officials say tap water is safe to drink.
Some environmental activists in the area say they aren't so sure. They fear the consequences for wildlife and say that the situation shows state regulators haven't done enough to crack down on Duke Energy.
The utility has apologized for the spill and vowed to clean up any damage.
"We're committed to the Dan River and the communities that it serves," Charlie Gates, the company's senior vice president of power generation operations, said in a statement Saturday. "We are accountable for what has happened and have plenty of work ahead of us."
Concerns over drinking water, wildlife
Duke Energy announced last week that it found the leak in a 48-inch stormwater pipe at the retired Dan River Steam Station in Eden, North Carolina, on February 2.
On Saturday, six days later, the company said it had plugged the broken pipe that was causing the spill and was working with officials on developing a cleanup plan.
Coal ash, the material that remains after burning coal for electricity, contains metals such as arsenic, selenium and cadmium.
Tests of the river last week revealed levels of copper, aluminum, iron and arsenic above state standards for surface water, state environmental officials said.
It's unclear what that could mean for wildlife in the area, said Jamie Kritzer, a spokesman for the North Carolina Department of Environment and Natural Resources.
"It's certainly cause for concern for the long-term impacts of this coal ash spill on the health of the Dan River," he said.
Environmental advocates warn that the damage could be significant, potentially harming fish in the river and impacting the food chain.
"You have a cleanup effort that is going to be difficult," said Sam Perkins, who works for the Catawba Riverkeeper Foundation, a nonprofit advocacy group dedicated to protecting waterways in the area. "This shows even a small spill has an impact on the ecosystem."
Some activists accuse authorities of deliberately playing down the danger of the situation and taking too long to notify the public.
"This is another shameless attempt by (environmental officials) to downplay the risks facing the communities along the Dan River downstream," Peter Harrison of the Waterkeeper Alliance advocacy group said in a written statement to CNN on Friday.
"Are we supposed to feel good that there are only four hazardous toxins, including a carcinogen, in our drinking water supply?"
Samples taken by the Waterkeeper Alliance last week contained "extremely high levels of arsenic, chromium, iron, lead and other toxic metals," the group said in a statement.
State environmental officials said Sunday that arsenic levels appeared to be decreasing, but recommended avoiding prolonged direct contact with the river in the area of the spill until further notice.
Unclear how long cleanup will take
Kritzer said authorities have been open about what they've found.
"We're not downplaying risks. We're doing our objective analysis of what we're seeing so far, and I think we are concerned," he said. "The Dan River is a gem, and people value it throughout the state for not only being a source of drinking water, but also for its aquatic life that it provides a home to and all the recreational uses. This is certainly something that concerns all of us."
Tiffany Haworth, executive director of the Dan River Basin Association, first learned about the spill from a mail carrier, who warned that the river's water had turned black. The situation is heartbreaking, she said.
"I stood on the bank a day or two after the spill, and I can say that I openly cried," she said. "I was thinking, 'How can this ever heal? How can this ever be cleaned up? And what is this going to do to what I would consider one of the most beautiful parts of our country?'"
Now, she said, cleanup is key.
"The longer it's allowed to sit there ...t he sediment that has not gone down the river will be constantly churned up as it goes downstream, and the longer that we wait, obviously the more damage can occur," she said.
The North Carolina spill comes weeks after a chemical spill in West Virginia left 300,000 people unable to use their water supply for days. Now, a federal grand jury is looking into that spill in what one official called a criminal investigation.
In North Carolina, authorities will investigate the coal ash spill to determine what violations occurred, Kritzer said.
State and federal agencies are working with Duke Energy to figure out the next steps for cleanup, he said. At this point, it's unclear how long that could take.
Authorities were still working to develop a cleanup plan Sunday, Duke Energy spokeswoman Lisa Parrish said.
"Simultaneous efforts have been under way to not only plug the pipe and cap the system, which we successfully achieved last night, but also to test water quality. We've been testing water quality since the leak occurred and will continue to do so," she said. "Water quality tests will inform our cleanup efforts and accelerate our planning for the best long-term solution at the site."
Even before last week's spill, coal ash contamination was a concern for North Carolina officials. The state filed lawsuits against Duke Energy last year, asking the court to order the utility to deal with groundwater and wastewater violations at 14 sites where the byproducts of coal power plants are stored, according to a statement from Gov. Pat McCrory's office.
Parrish said Sunday that the utility is in the midst of plans to close the sites where it stores coal ash in North Carolina.
"Ash basin closure planning is already well under way for the ash basins located in North Carolina, including the one at Dan River," Parrish said. "We look forward to moving ahead with that project."
Governor has close ties with company
Before he ran for governor, McCrory worked for Duke Energy for nearly three decades, and critics have claimed he's shied away from regulation during his time in public office due to his close ties with the utility.
In a statement last week, McCrory said his administration was the first in the state's history to take legal action against the utility over the ponds.
"We have been moving on this issue since the beginning of my term and will continue to do so," he said.
Environmental advocates say the spill is a reminder of a troubling problem that's widespread in the state: coal ash ponds storing large amounts of waste close to drinking water supplies.
The spill raises questions, the Catawba Riverkeeper Foundation advocacy group said in an online post.
Key among them -- what chemicals are scientists testing for, was drinking water contaminated and will the utility change its coal ash disposal approach as a result?
According to figures released by Duke Energy, last week's spill appears to be similar to, but smaller than, a 2008 coal ash spill at a power plant in Kingston, Tennessee, which sent 1.1 billion gallons of sludge into the adjacent Emory River.
State authorities slapped the Tennessee Valley Authority with $11.5 million fines after that spill, which authorities said violated state clean-water and solid waste disposal laws.
In statements announcing the North Carolina spill last week, Duke Energy said up to 82,000 tons of ash had been released and up to 27 million gallons of basin water had flooded into the river. That amount of ash, the company said, would fill up to 32 Olympic-size swimming pools.
        \ No newline at end of file diff --git a/test/testdata/710ef2de4a3f0d569273eeab117cf7841465c754.json b/test/testdata/710ef2de4a3f0d569273eeab117cf7841465c754.json new file mode 100644 index 00000000..c0703308 --- /dev/null +++ b/test/testdata/710ef2de4a3f0d569273eeab117cf7841465c754.json @@ -0,0 +1,28 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "261", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "36447", + "Content-Type": "text/html; charset=utf-8", + "Date": "Mon, 08 Jan 2018 15:09:45 GMT", + "Fastly-Debug-Digest": "554e4fad54bb5853b1037e3a815667d427fd15cefd389b688014e0ca50befa27", + "Set-Cookie": "countryCode=IR; Domain=.cnn.com; Path=/, geoData=tehran|07|0|IR|AS; Domain=.cnn.com; Path=/, tryThing00=0193; Domain=.cnn.com; Path=/; Expires=Sun Apr 01 2018 00:00:00 GMT", + "Vary": "Accept-Encoding, Fastly-SSL, Fastly-SSL", + "Via": "1.1 varnish, 1.1 varnish", + "X-Cache": "MISS, HIT", + "X-Cache-Hits": "0, 1", + "X-Served-By": "cache-iad2142-IAD, cache-hhn1537-HHN", + "X-Timer": "S1515424185.087869,VS0,VE1", + "access-control-allow-origin": "*", + "cache-control": "max-age=60", + "content-security-policy": "default-src 'self' blob: https://*.cnn.com:* http://*.cnn.com:* *.cnn.io:* *.cnn.net:* *.turner.com:* *.turner.io:* *.ugdturner.com:* courageousstudio.com *.vgtf.net:*; script-src 'unsafe-eval' 'unsafe-inline' 'self' *; style-src 'unsafe-inline' 'self' blob: *; child-src 'self' blob: *; frame-src 'self' *; object-src 'self' *; img-src 'self' data: blob: *; media-src 'self' data: blob: *; font-src 'self' data: *; connect-src 'self' *; frame-ancestors 'self' *.cnn.com:* *.turner.com:* courageousstudio.com;", + "x-content-type-options": "nosniff", + "x-servedByHost": "::ffff:172.17.46.18", + "x-xss-protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://edition.cnn.com/2014/02/09/us/north-carolina-coal-ash-spill/" +} \ No newline at end of file diff --git a/test/testdata/7145a60336517a4fcb4b34eb13c37c7b52667814.html b/test/testdata/7145a60336517a4fcb4b34eb13c37c7b52667814.html new file mode 100644 index 00000000..d9784cc0 --- /dev/null +++ b/test/testdata/7145a60336517a4fcb4b34eb13c37c7b52667814.html @@ -0,0 +1,1264 @@ + + + برجام شرایط بین‌المللی ایران را کاملا متحول کرد - ایسنا + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
        + +
        + + +
        +
        +
        +
        +
        +
        +
        +
        + +
          +
        • چهارشنبه / ۶ بهمن ۱۳۹۵ / ۰۹:۴۳
        • +
        • دسته‌بندی: + سیاست خارجی + +
        • +
        • کد خبر: 95110603890
        • +
        • خبرنگار : 71429
        • + +
        +
        +
        + +
        +
        +
        + +

        عراقچی:

        +

        برجام شرایط بین‌المللی ایران را کاملا متحول کرد

        +
        +
        +
        +
        + نشست خبری عراقچی +
        +

        معاون وزیر امور خارجه گفت: برجام شرایط بین‌المللی ایران را کاملا متحول کرد.

        + +

        به گزارش ایسنا، سیدعباس عراقچی در همایش تخصصی معاونان ستادی و رؤسای پلیس مبارزه با مواد مخدر گفت: ایران به کشوری سازنده و بازیگر فعال در عرصه بین‌الملل بعد از برجام تبدیل شد و می‌تواند نقش موثری در عرصه بین‌الملل ایفا کند.

        + +

        وی اضافه کرد: شورای امنیت نقش ایران در سوریه را به عنوان ضامن صلح و امنیت به رسمیت می‌شناسد و به آن احترام می‌گذارد، این درحالی است که قبل از برجام ایران را تهدید منطقه در قطعنامه هسته‌ای عنوان کرده بودند.

        + +

        معاون حقوقی و امور بین‌الملل وزارت امور خارجه تصریح کرد: فعالیت‌های هسته‌ای ایران به عنوان تهدید قلمداد شده بود و همان قطعنامه‌های شورای امنیت که ایران را تهدید و برنامه‌های هسته‌ای ایران را تهدید شناسایی می‌کرد و از همه کشورها می‌خواست با ایران همکاری نکنند و به دانشجویان ایرانی اجازه تحصیل در دیگر کشورها را نمی‌داد و از ایران می‌خواست فردو، اراک و غنی سازی را ببندد، همه با برجام یکجا لغو شد.

        + +

        وی افزود: اما شورای امنیت با قطعنامه جدید ٢٢٣١ صادر شده برنامه هسته‌ای ایران را به رسمیت شناخت.

        + +

        عراقچی با اشاره به نشست اخیر آستانه برای بررسی موضوع صلح در سوریه گفت: خود آمریکایی‌ها گفتند ننگی برای آمریکا بدتر از این نیست که ایران و روسیه کنار هم بنشینند و تصمیم بگیرند آیا آمریکا به نشست دعوت شود یا نه.

        + +

        وی افزود: مقوله امنیت، مقوله‌ای بسیار پیچیده است و هر روز با رشد تکنولوژی و تهدیدات جدید پیچیده‌تر هم می‌شود و هر روز مؤلفه‌های بیشتری به آن اضافه می‌شود. امروزه وظایف پلیس با وظایفی که در گذشته داشته است تفاوت دارد و امنیت در حال حاضر در جامعه به شدت به هم پیوسته شده است. امروزه شکل تهدیدات در جامعه عوض شده که به تبع آن مقابله با آن نیز چندبُعدی شده است.

        + +

        عراقچی با اشاره به حادثه پلاسکو، گفت: به عنوان مثال در حادثه پلاسکو بی‌احتیاطی یک نفر یک فاجعه ملی را رقم زد. بنابراین مقوله امنیت بسیار به هم پیوسته و پیچیده است. موضوعاتی نظیر فقر و کمبود منابع هر کدام می‌تواند به یک تهدید ملی منجر شود که این امر فعالیت پلیس و دیگر دستگاه‌ها را در مقابله با این تهدیدات زیاد می‌کند.

        + +

        وی با اشاره به مقوله مواد مخدر گفت: تاکنون در حوزه مقابله با مواد مخدر در عرصه جهانی، اقدامات خوبی انجام شده است، این در حالی است که این حوزه با محدودیت‌های بسیاری همراه بود، چرا که چهره ایران در عرصه جهانی به عنوان یک چهره خطرناک شناخته شده بود و به تبع آن نیز در زمینه مبارزه با مواد مخدر محدودیت‌هایی را داشتیم.

        + +

        وی یاداور شد: در عرصه بین‌المللی نیز تهدیدات در حال حاضر نسبت به گذشته فرق کرده است و امروزه مقابله با تهدیدها از عهده یک کشور خارج است و باید کشورها در زمینه مبارزه با تهدیدها با هم مشارکت کنند بنابراین همکاری بین‌المللی برای مقابله با تهدیدها امری انکارناپذیر است.

        + +

        وی افزود: امروزه منابع حاصل از قاچاق مواد مخدر از اصلی‌ترین منابع مالی تروریست‌هایی چون داعش و القاعده است.  بحث پولشویی و تروریست کاملاً به هم مرتبط هستند و برخورد با این دو معضل نیازمند همکاری‌های بین‌المللی است.

        + +

        عراقچی اضافه کرد: امروزه یکی از شاخص‌ترین تهدیدها موضوع مواد مخدر است که در همین راستا فعالیت قاچاقچیان به یک معضل جهانی تبدیل شده است و نیازمند همکاری کشورهای مختلف است. امروزه مواد مخدر در یک کشور دیگر تولید می‌شود اما آسیب‌های آن به کشور ما وارد می‌شود.

        + +

        معاون وزیر امور خارجه با اشاره به اقدامات نیروی انتظامی در زمینه مقابله با مواد مخدر گفت: نیروی انتظامی تاکنون اقدامات بسیار خوبی در زمینه مقابله با مواد مخدر انجام داده که با توجه به بررسی‌های ما و مقایسه نیروی انتظامی با کشورهای دیگر عملکرد نیروی انتظامی ایران بسیار مطلوب است. خوشبختانه فضای همکاری جمهوری اسلامی ایران با کشورهای دیگر در زمینه مبارزه با مواد مخدر از جایگاه بسیار خوبی برخوردار است.

        + +


        +انتهای پیام

        + +
        + + + +
        + +
        +
        +
        + +
        +
        +
        +
        + +
        • در زمینه انتشار نظرات مخاطبان رعایت چند مورد ضروری است:
        • -لطفا نظرات خود را با حروف فارسی تایپ کنید.
        • -«ایسنا» مجاز به ویرایش ادبی نظرات مخاطبان است.
        • - ایسنا از انتشار نظراتی که حاوی مطالب کذب، توهین یا بی‌احترامی به اشخاص، قومیت‌ها، عقاید دیگران، موارد مغایر با قوانین کشور و آموزه‌های دین مبین اسلام باشد معذور است.
        • - نظرات پس از تأیید مدیر بخش مربوطه منتشر می‌شود.
        +
        +
        +
        +
        +

        نظرات

        +
        +
        + +
        +
        +
        شما در حال پاسخ به نظر «» هستید. + +
        +
        +
        + + + + +
        +
        + +
        +
        +
        + + + +
        +
        +
        +
        +
        + +
        +
        +
        +
        +
        +
        +
        + + +
        +
        +
        +
        +
        +
        +
        + + + + + + + + + \ No newline at end of file diff --git a/test/testdata/7145a60336517a4fcb4b34eb13c37c7b52667814.json b/test/testdata/7145a60336517a4fcb4b34eb13c37c7b52667814.json new file mode 100644 index 00000000..b4c5a10f --- /dev/null +++ b/test/testdata/7145a60336517a4fcb4b34eb13c37c7b52667814.json @@ -0,0 +1,19 @@ +{ + "encoding": "UTF-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "0", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Type": "text/html;charset=UTF-8", + "Date": "Tue, 23 May 2017 17:54:10 GMT", + "Server": "Apache-Coyote/1.1", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding", + "Via": "1.1 varnish-v4", + "X-Varnish": "288057294", + "grace": "none" + }, + "status_code": 200, + "url": "http://www.isna.ir/news/95110603890/%D8%A8%D8%B1%D8%AC%D8%A7%D9%85-%D8%B4%D8%B1%D8%A7%DB%8C%D8%B7-%D8%A8%DB%8C%D9%86-%D8%A7%D9%84%D9%85%D9%84%D9%84%DB%8C-%D8%A7%DB%8C%D8%B1%D8%A7%D9%86-%D8%B1%D8%A7-%DA%A9%D8%A7%D9%85%D9%84%D8%A7-%D9%85%D8%AA%D8%AD%D9%88%D9%84-%DA%A9%D8%B1%D8%AF" +} \ No newline at end of file diff --git a/test/testdata/74537c7640b35258e9ac3596c95e9ee7b57a66a2.html b/test/testdata/74537c7640b35258e9ac3596c95e9ee7b57a66a2.html new file mode 100644 index 00000000..32f49e7b --- /dev/null +++ b/test/testdata/74537c7640b35258e9ac3596c95e9ee7b57a66a2.html @@ -0,0 +1,151 @@ +San Francisco families earning $117,000 qualify as ‘low income’

        Money

        In San Francisco, households earning $117,000 qualify as ‘low income’

        Share
        San Francisco, California
        RICOWde | Getty Images

        The increasingly steep cost of living in the Bay Area means that even earning six figures in San Francisco might not be enough to make ends meet.

        A new report from the Department of Housing and Urban Development says that a San Francisco metro area family of four bringing in $117,400 a year qualifies as “low income." Last year, the cut off was $105,350. An annual salary of $82,000 now puts single adults in the “low income” bracket as well.

        Other notoriously expensive cities aren’t nearly as extreme. In New York, the “low income” threshold for a family of four is $83,450 per year. In Los Angeles, it’s $77,500.

        Making ends meet for a family of four in San Francisco requires a household income of $92,139, according to MIT’s living wage calculator. The model takes into account factors such as the costs of child care and health insurance, in addition to food and other regular expenses, but doesn’t include conveniences such as restaurant meals, vacations and money left over for investments.

        VIDEO0:5400:54
        Here's how rent changed in 2017 in 5 major cities

        Silicon Valley’s income inequality hits certain groups particularly hard. Despite working at major tech companies, contract workers, such as janitors and cafeteria workers, . One contractor at Facebook, security guard Jiovanny Martinez, must also drive for Lyft and work as a park ranger to support his family, he tells The Guardian.

        Nicole and Victor, a married couple who are both contract workers in the cafeteria at Facebook's headquarters,  with their three children. They borrow money from friends and family to stay afloat, they tell The Guardian in a separate article, and occasionally resort to payday loans. They cannot afford the company's health care plan.

        Although Facebook implemented a minimum wage of $15 for all of its contractors in 2015, the paychecks don't go far around San Francisco, where the cost of living is 62 percent higher than the U.S. average.

        VIDEO1:2101:21
        The 10 most affordable places to live in the US

        Even some white-collar employees in the area are struggling. In 2017, one Twitter employee earning a $160,000 salary told The Guardian that he's .

        The employee's biggest expense is the $3,000 monthly rent he pays on a two-bedroom house where he lives with his wife and two kids, which he describes as "ultra cheap."

        "Families are priced out of the market," he says, explaining that it's hard to compete with the hordes of 20-somethings willing to pile into a shared house — and still pay $2,000 per person for a room.

        Don't miss: Here's how much you have to earn to live comfortably in the 15 largest US cities

        Like this story? Subscribe to CNBC Make It on YouTube!

        VIDEO1:0601:06
        The biggest mistake millennials are making is not buying a home, says financial expert
        \ No newline at end of file diff --git a/test/testdata/74537c7640b35258e9ac3596c95e9ee7b57a66a2.json b/test/testdata/74537c7640b35258e9ac3596c95e9ee7b57a66a2.json new file mode 100644 index 00000000..a64c59ad --- /dev/null +++ b/test/testdata/74537c7640b35258e9ac3596c95e9ee7b57a66a2.json @@ -0,0 +1,22 @@ +{ + "encoding": "utf-8", + "headers": { + "Access-Control-Allow-Origin": "*", + "Cache-Control": "max-age=0, no-cache", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "126204", + "Content-Security-Policy": "frame-ancestors 'self' *.cnbc.com *.acorns.com;", + "Content-Type": "text/html; charset=utf-8", + "Date": "Sun, 02 Jan 2022 14:59:10 GMT", + "Expires": "Sun, 02 Jan 2022 14:59:10 GMT", + "Link": ";rel=\"preconnect\",;rel=\"preconnect\"", + "Pragma": "no-cache", + "Set-Cookie": "region=WORLD; expires=Sat, 02-Apr-2022 14:59:10 GMT; path=/; domain=.cnbc.com, AKA_A2=A; expires=Sun, 02-Jan-2022 15:59:10 GMT; path=/; domain=cnbc.com; secure; HttpOnly, akaas_CNBC_Audience_Segmentation=1643727550~rv=16~id=4f27a49a9913465a00a5e4c501da67d3; path=/; Expires=Tue, 01 Feb 2022 14:59:10 GMT; Domain=.www.cnbc.com; Secure; SameSite=None", + "Vary": "Accept-Encoding, User-Agent", + "X-Aicache-OS": "xxx.x1.15.16:81, xx.xx.225.187:80", + "X-Request-Id": "7ee423c8-4171-4ccb-8dcd-6d836fd2f981" + }, + "status_code": 200, + "url": "https://www.cnbc.com/2018/06/28/families-earning-117000-qualify-as-low-income-in-san-francisco.html" +} \ No newline at end of file diff --git a/test/testdata/78ddabf2d3e27f8c1bee9d3afe34973da781ad3a.html b/test/testdata/78ddabf2d3e27f8c1bee9d3afe34973da781ad3a.html new file mode 100644 index 00000000..a5287f63 --- /dev/null +++ b/test/testdata/78ddabf2d3e27f8c1bee9d3afe34973da781ad3a.html @@ -0,0 +1,11 @@ + +@article{noormags105489, +title = { تحلیل منافع بهره وری ناشی از اصلاحات صنعت برق استرالیا: چارچوب های روش شناختی }, +journal = { مطالعات اقتصاد انرژی }, +number = { 3 }, +year = { 1383 }, +author = { +فتح‌الله‌زاده‌اقدم,‌رضا and }, +pages = { 55 -- 55 }, +url = { http://www.noormags.ir/view/fa/articlepage/105489 } +} \ No newline at end of file diff --git a/test/testdata/78ddabf2d3e27f8c1bee9d3afe34973da781ad3a.json b/test/testdata/78ddabf2d3e27f8c1bee9d3afe34973da781ad3a.json new file mode 100644 index 00000000..e592c8c9 --- /dev/null +++ b/test/testdata/78ddabf2d3e27f8c1bee9d3afe34973da781ad3a.json @@ -0,0 +1,15 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "private", + "Content-Disposition": "attachment; filename=\"noormags-105489.bib\"", + "Content-Length": "428", + "Content-Type": "application/x-bibtex; charset=UTF-8", + "Date": "Tue, 23 May 2017 17:51:56 GMT", + "Set-Cookie": "CRCIS_SessionId=spfagyprftg1fmawbwjapukk; path=/, .ASPXBrowserOverride=Mozilla%2f4.0+(compatible%3b+MSIE+6.0%3b+Windows+CE%3b+IEMobile+8.12%3b+MSIEMobile+6.0); expires=Tue, 30-May-2017 17:51:56 GMT; path=/", + "X-Download-Options": "noopen", + "X-Frame-Options": "SAMEORIGIN" + }, + "status_code": 200, + "url": "http://www.noormags.ir/view/fa/citation/bibtex/105489" +} \ No newline at end of file diff --git a/test/testdata/7a73b8c1ab361bbf3846c7931bd8c07dee93022d.html b/test/testdata/7a73b8c1ab361bbf3846c7931bd8c07dee93022d.html new file mode 100644 index 00000000..4170cca1 --- /dev/null +++ b/test/testdata/7a73b8c1ab361bbf3846c7931bd8c07dee93022d.html @@ -0,0 +1,21 @@ + پایگاه اطلاع رسانی شبکه خبر صدا و سیمای جمهوری اسلامی ایران

        irinn | وب سایت شبکه خبر

        نسخه آزمایشی       
        16:39 - چهارشنبه 03 خرداد 1396
        نظامیان صهیونیست یک جوان فلسطینی را در شمال کرانه غربی به ضرب گلوله به شهادت رساندند رژیم صهیونیستی روستای دیراستیا در کرانه غربی رود اردن را منطقه بسته نظامی اعلام کرد نظامیان صهیونیست با یورش به کرانه غربی و نوار غزه ده ها فلسطینی را زخمی کردند انفجار خودروی بمب گذاری شده در جنوب بنغازی لیبی ۶ کشته و ۱۱ زخمی بر جا گذاشت در حمله به یک پایگاه هوایی در جنوب لیبی بیش از ۱۴۰ تن کشته شدند رئیس جمهور ونزوئلا(خطاب به رئیس جمهور آمریکا): دخالت بس است از ونزوئلا بیرون برو آمریکا به بهانه حمایت از دموکراسی تحریم های جدیدی را ضد ونزوئلا وضع کرده است زمین لرزه ۶ ریشتری مرکز فیلیپین را لرزاند
        بغض یک ملت

        بغض یک ملت

        نارضایتی اکثریت ملت بحرین از بی کفایتی های یک خانواده سلطنتی با دخالت آل سعود و در پی فشار بر رهبران...
        بزرکترین قرارداد صنعت نفت در پسابرجام

        بزرکترین قرارداد صنعت نفت در پسابرجام

        قرارداد ساخت و انتقال دانش لوله های CRA به ارزش تقریبی 556 میلیون یورو میان شرکت ملی نفت ایران و کنسرسیومی...
        15 هزار سبد کالا در ماه رمضان توزیع می شود
        بین مددجویان استان تهران

        15 هزار سبد کالا در ماه رمضان توزیع می شود

        مديركل كميته امداد استان تهران گفت: امسال 15 هزار سبد كالا ميان مددجويان استان تهران كه امكان حضور در...
        تلاش محققان ایرانی برای کاهش سمیت نانو ذرات

        تلاش محققان ایرانی برای کاهش سمیت نانو ذرات

        تیمی از محققان دانشگاه آزاد واحد علوم دارویی تهران مطالعاتی در خصوص کاهش سمیت نانو ذرات بعنوان ناقل‌...
        دغدغه های ناتمام والدین از شهریه مهدها

        دغدغه های ناتمام والدین از شهریه مهدها

        شهریه های مهدکودک ها قرار است با درصدی افزایش همراه باشد، این درحالی است که همین حالا هم بسیاری از والدین...
        شکست تلخ فوتبال جوانان مقابل زامبیا

        شکست تلخ فوتبال جوانان مقابل زامبیا

        تیم فوتبال جوانان کشورمان در دومین بازی جام جهانی 2017، با نتیجه 4-2 نتیجه را به تیم ملی زامبیا واگذار...
        خبر20
        خبر 20، سه شنبه 2 خرداد 96
        خبر 20 , سه‌شنبه ۰۲ خرداد ۹۶
        منطقه الدراز بحرین آماج حملات زمینی و هوایی نیروهای رژیم آل خلیفه/ تجدید میثاق رئیس جمهور منتخب با آرمان های امام (ره) و شهداء/ حضور پرصلابت ملت در انتخابات پشتوانه محکم امنیت ملی/ نشست خبری سخنگوی نظارت بر انتخابات شوراها و چند خبر دیگر.
        خبر ورزشی
        خبر ورزشی 14:30، چهارشنبه 3 خرداد 96
        خبر ورزشی 14:30 , چهارشنبه ۰۳ خرداد ۹۶
        شکست تیم ملی جوانان مقابل زامبیا/ فینال لیگ اروپا، منچستر - آژاکس/ جلسه ی شورای فنی کشتی/ برگزاری مسابقات سوارکاری و چند خبر کوتاه را در این بخش ببینید.
        مشروح
        اخبار مشروح چهارشنبه، 3 خرداد 96
        مشروح 13 , چهارشنبه ۰۳ خرداد ۹۶
        500 ویژه برنامه در سراسر کشور به مناسبت سالروز آزادسازی خرمشهر/ قدردانی نمایندگان مجلس از عملکرد رسانه ملی در انتخابات 29 اردیبهشت/ گسترش اعتراض های بین المللی به اقدام آل خلیفه در کشتار مردم بحرین و چند خبر دیگر.
        پربازدید ها
        روز
        هفته
        ماه
        آب و هوا
        ۱۸°    ۳۴°
        گزارش خبرنگاران
        ایرانی کالای ایرانی بخر!

        ایرانی کالای ایرانی بخر!

        صنعت نساجی یزد با بیش از 600 واحد صنعتی رتبه دوم کشوری را دارد. این استان در تولید رومبلی با سهم 66 درصد رتبه اول را دارد که در صورت حمایت از کالای ایرانی ظرفیت تولید این محصول به بیش از دوبرابر خواهد رسید.
        چهارشنبه ۰۳ خرداد ۹۶ - ۰۹:۱۳
        شبکه در شبکه 2 خرداد 96 اجتماعی

        شبکه در شبکه 2 خرداد 96

        سه‌شنبه ۰۲ خرداد ۹۶ - ۱۹:۳۵
        هفتادمین نشست مجمع جهانی سلامت علمی

        هفتادمین نشست مجمع جهانی سلامت

        سه‌شنبه ۰۲ خرداد ۹۶ - ۱۵:۴۴
        بیستون، جاذبه بی نظیر گردشگری اجتماعی

        بیستون، جاذبه بی نظیر گردشگری

        سه‌شنبه ۰۲ خرداد ۹۶ - ۱۴:۱۰
        چشم فعالان اقتصادی به جاده جدید ابریشم سیاسی

        چشم فعالان اقتصادی به جاده جدید ابریشم

        سه‌شنبه ۰۲ خرداد ۹۶ - ۱۱:۴۲
        \ No newline at end of file diff --git a/test/testdata/7a73b8c1ab361bbf3846c7931bd8c07dee93022d.json b/test/testdata/7a73b8c1ab361bbf3846c7931bd8c07dee93022d.json new file mode 100644 index 00000000..c7036510 --- /dev/null +++ b/test/testdata/7a73b8c1ab361bbf3846c7931bd8c07dee93022d.json @@ -0,0 +1,18 @@ +{ + "encoding": "utf-8", + "headers": { + "Cache-Control": "post-check=0, pre-check=0", + "Connection": "close", + "Content-Encoding": "gzip", + "Content-Length": "18431", + "Content-Type": "text/html; charset=utf-8", + "Date": "Wed, 24 May 2017 12:09:37 GMT", + "Expires": "Sat, 26 Jul 1997 05:00:00 GMT", + "Pragma": "no-cache", + "Server": "sepehr-proxy-1.2-rc3-server4-tabnak", + "X-Cache": "MISS from google.com", + "X-Cache-Lookup": "MISS from google.com:86" + }, + "status_code": 200, + "url": "http://www.irinn.ir/" +} \ No newline at end of file diff --git a/test/testdata/7aef1520fa9406d0d20860a4413187f663f12c4e.html b/test/testdata/7aef1520fa9406d0d20860a4413187f663f12c4e.html new file mode 100644 index 00000000..5865d5ea --- /dev/null +++ b/test/testdata/7aef1520fa9406d0d20860a4413187f663f12c4e.html @@ -0,0 +1 @@ +{"indexed":{"date-parts":[[2022,4,11]],"date-time":"2022-04-11T08:26:40Z","timestamp":1649665600002},"publisher-location":"Berlin, Heidelberg","reference-count":16,"publisher":"Springer Berlin Heidelberg","isbn-type":[{"value":"9783540071556","type":"print"},{"value":"9783540374831","type":"electronic"}],"content-domain":{"domain":[],"crossmark-restriction":false},"published-print":{"date-parts":[[1975]]},"DOI":"10.1007\/bfb0064872","type":"book-chapter","created":{"date-parts":[[2006,11,10]],"date-time":"2006-11-10T18:23:02Z","timestamp":1163182982000},"page":"132-154","source":"Crossref","is-referenced-by-count":67,"title":"Weak monadic second order theory of succesor is not elementary-recursive","prefix":"10.1007","author":[{"given":"Albert R.","family":"Meyer","sequence":"first","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2006,8,25]]},"reference":[{"issue":"2","key":"4_CR1","doi-asserted-by":"publisher","first-page":"322","DOI":"10.1145\/321386.321395","volume":"14","author":"M. Blum","year":"1967","unstructured":"Blum, M. A machine-independent theory of the complexity of recursive functions, Jour. Assoc. Comp. Mach., 14, 2 (April, 1967), 322\u2013336.","journal-title":"Jour. Assoc. Comp. Mach."},{"issue":"2","key":"4_CR2","doi-asserted-by":"publisher","first-page":"290","DOI":"10.1145\/321637.321648","volume":"18","author":"M. Blum","year":"1971","unstructured":"Blum, M. On effective procedures for speeding up algorithms, Jour. Assoc. Comp. Mach., 18, 2 (April, 1971), 290\u2013305.","journal-title":"Jour. Assoc. Comp. Mach."},{"key":"4_CR3","first-page":"834","volume":"5","author":"J.R. B\u00fcchi","year":"1959","unstructured":"B\u00fcchi, J.R. and C.C. Elgot, Decision problems of weak second order arithmetics and finite automata, Part I, (abstract), AMS Notices, 5 (1959), 834.","journal-title":"AMS Notices"},{"key":"4_CR4","doi-asserted-by":"publisher","first-page":"66","DOI":"10.1002\/malq.19600060105","volume":"6","author":"J.R. B\u00fcchi","year":"1960","unstructured":"B\u00fcchi, J.R. Weak second order arithmetic and finite automata, Zeit. f. Math. Log. and Grund. der Math., 6 (1960), 66\u201392.","journal-title":"Zeit. f. Math. Log. and Grund. der Math."},{"key":"4_CR5","unstructured":"Cooper, D.C. Theorem-proving in arithmetic without multiplication, Computer and Logic Group Memo. No. 16, U.C. of Swansea, April, 1972, to appear in Machine Intelligence 7."},{"issue":"2","key":"4_CR6","doi-asserted-by":"publisher","first-page":"169","DOI":"10.2307\/2269808","volume":"31","author":"C.C. Elgot","year":"1966","unstructured":"Elgot, C.C. and M.O. Rabin, Decidability and undecidability of extensions of second (first) order theory of (generalized) successor, Jour. Symb. Logic, 31, 2 (June, 1966), 169\u2013181.","journal-title":"Jour. Symb. Logic"},{"key":"4_CR7","unstructured":"Ferrante, J. and C. Rackoff, A decision procedure for the first order theory of real addition with order, Project MAC Tech. Memo 33, Mass. Inst. of Technology (May, 1973), 16pp., to appear SIAM Jour. Comp."},{"key":"4_CR8","first-page":"1","volume":"4","author":"A. Grzegorczyk","year":"1953","unstructured":"Grzegorczyk, A. Some classes of recursive functions, Rozprawy Matematyczne, 4 (1953), Warsaw, 1\u201345.","journal-title":"Rozprawy Matematyczne"},{"key":"4_CR9","unstructured":"Meyer, A.R. Weak SIS cannot be decided (abstract 72T-E67), AMS Notices, 19, 5 (August, 1972), p. A-598."},{"key":"4_CR10","unstructured":"Meyer, A.R. and L.J. Stockmeyer, The equivalence problem for regular expressions with squaring requires exponential space, 13 th Switching and Automata Theory Symp. (Oct. 1972), IEEE, 125\u2013129."},{"key":"4_CR11","doi-asserted-by":"crossref","unstructured":"Oppen, D.C. Elementary bounds for Presburger arithmetic, 5 th ACM Symp. Theory of Computing (April, 1973), 34\u201337.","DOI":"10.1145\/800125.804033"},{"key":"4_CR12","first-page":"1","volume":"141","author":"M.O. Rabin","year":"1969","unstructured":"Rabin, M.O. Decidability of second-order theories and automata on infinite trees, Trans. AMS, 141 (July, 1969), 1\u201335.","journal-title":"Trans. AMS"},{"key":"4_CR13","first-page":"115","volume":"3","author":"M.O. Rabin","year":"1959","unstructured":"Rabin, M.O. and D. Scott, Finite automata and their decision problems, IBM Jour. Research and Development, 3 (1959), 115\u2013125.","journal-title":"IBM Jour. Research and Development"},{"key":"4_CR14","doi-asserted-by":"publisher","first-page":"139","DOI":"10.1090\/S0002-9947-1963-0158822-2","volume":"106","author":"R.W. Ritchie","year":"1963","unstructured":"Ritchie, R.W. Classes of predictably computable functions, Trans. AMS, 106 (1963), 139\u2013173.","journal-title":"Trans. AMS"},{"key":"4_CR15","doi-asserted-by":"crossref","unstructured":"Stearns, R.E., J. Hartmanis, and P.M. Lewis, III, Hierarchies of memory-limited computations, 6 th Switching Theory and Logical Design Symp. (1965), IEEE, 179\u2013190.","DOI":"10.1109\/FOCS.1965.11"},{"key":"4_CR16","doi-asserted-by":"crossref","unstructured":"Stockmeyer, L.J. and A.R. Meyer, Word problems requiring exponential time, 5 th ACM Symp. Theory of Computing (April, 1973), 1\u20139.","DOI":"10.1145\/800125.804029"}],"container-title":"Lecture Notes in Mathematics","original-title":[],"link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/BFb0064872","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,4,22]],"date-time":"2019-04-22T09:19:33Z","timestamp":1555924773000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/BFb0064872"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[1975]]},"ISBN":["9783540071556","9783540374831"],"references-count":16,"URL":"http:\/\/dx.doi.org\/10.1007\/BFb0064872","relation":{},"ISSN":["0075-8434","1617-9692"],"published":{"date-parts":[[1975]]}} \ No newline at end of file diff --git a/test/testdata/7aef1520fa9406d0d20860a4413187f663f12c4e.json b/test/testdata/7aef1520fa9406d0d20860a4413187f663f12c4e.json new file mode 100644 index 00000000..0b7107f0 --- /dev/null +++ b/test/testdata/7aef1520fa9406d0d20860a4413187f663f12c4e.json @@ -0,0 +1,24 @@ +{ + "encoding": null, + "headers": { + "access-control-allow-headers": "X-Requested-With, Accept, Accept-Encoding, Accept-Charset, Accept-Language, Accept-Ranges, Cache-Control", + "access-control-allow-origin": "*", + "access-control-expose-headers": "Link", + "connection": "close", + "content-encoding": "gzip", + "content-length": "2313", + "content-type": "application/vnd.citationstyles.csl+json", + "date": "Thu, 09 Jun 2022 10:37:23 GMT", + "link": "; rel=\"canonical\", ; version=\"vor\"; rel=\"item\"", + "permissions-policy": "interest-cohort=()", + "server": "Jetty(9.4.40.v20210413)", + "vary": "Accept, Accept-Encoding", + "x-api-pool": "public", + "x-rate-limit-interval": "1s", + "x-rate-limit-limit": "50", + "x-ratelimit-interval": "1s", + "x-ratelimit-limit": "50" + }, + "status_code": 200, + "url": "https://api.crossref.org/v1/works/10.1007%2FBFb0064872/transform" +} \ No newline at end of file diff --git a/test/testdata/7b15a2abae5c75dbb5d2d35affbbc045284db11b.html b/test/testdata/7b15a2abae5c75dbb5d2d35affbbc045284db11b.html new file mode 100644 index 00000000..0d546be5 --- /dev/null +++ b/test/testdata/7b15a2abae5c75dbb5d2d35affbbc045284db11b.html @@ -0,0 +1,684 @@ + + + + + + دیوان-خاقانی-شروانی | نگاه | خانه کتاب و ادبیات ایران + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
        + + + + + + +
        +
        +
        +
        + + + + + + + +
        + + + + + +
        + + + ورود + + ثبت نام + + + + + + +
        +
        +
        +
        + + +
        + + + + +
        +
        +
        +
        +
        + دیوان خاقانی شروانی | خانه کتاب و ادبیات ایران +
        +
        +
        + صفحات اولیه کتاب +

        + دیوان خاقانی شروانی

        +

        + + + شعر فارسی - قرن 6ق. + + +

        + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
        پدیدآور + + شاعر : + + خاقانی ، بدیل‌بن‌علی + - + + + به‌اهتمام : + + منصور ، جهانگیر + - + + + مقدمه : + + فروزانفر ، محمدحسن + + +
        ناشر + + + + نگاه + + (برای تماس با ناشر و خرید کتاب کلیک کنید) + + + + +
        شابک978-964-6736-71-9
        تاریخ نشر + +13960522 +
        قیمت +
        کد دیویی8fa1.23
        زبان کتابفارسی
        محل نشرتهران - تهران
        توضیحات + جلد - + 836 صفحه - + تالیف - + چاپ 3 +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        معرفی مختصر کتاب
        +

        + +

        +
        +
        +
        +
        +
        + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/testdata/7b15a2abae5c75dbb5d2d35affbbc045284db11b.json b/test/testdata/7b15a2abae5c75dbb5d2d35affbbc045284db11b.json new file mode 100644 index 00000000..1bc6e7ed --- /dev/null +++ b/test/testdata/7b15a2abae5c75dbb5d2d35affbbc045284db11b.json @@ -0,0 +1,22 @@ +{ + "encoding": "utf-8", + "headers": { + "AR-ATIME": "0.093", + "AR-CACHE": "BYPASS", + "AR-PoweredBy": "Arvan Cloud (arvancloud.com)", + "AR-Request-ID": "c075775321b77d840111d6b01f2fbc8f", + "AR-SID": "2012", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Security-Policy": "upgrade-insecure-requests", + "Content-Type": "text/html; charset=utf-8", + "Date": "Sat, 27 Aug 2022 23:41:53 GMT", + "Keep-Alive": "timeout=65", + "Server": "ArvanCloud", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding, Accept-Encoding", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://ketab.ir/book/d3c92d14-e702-45fa-b44a-83092702ddf1" +} \ No newline at end of file diff --git a/test/testdata/8119947f1f0f53567e3663cb4b963b5ba51a1787.html b/test/testdata/8119947f1f0f53567e3663cb4b963b5ba51a1787.html new file mode 100644 index 00000000..b8622b1f --- /dev/null +++ b/test/testdata/8119947f1f0f53567e3663cb4b963b5ba51a1787.html @@ -0,0 +1,100 @@ +{ + "header": { + "type": "esummary", + "version": "0.3" + }, + "result": { + "uids": [ + "123455" + ], + "123455": { + "uid": "123455", + "pubdate": "1975 Apr", + "epubdate": "", + "source": "Bol Oficina Sanit Panam", + "authors": [ + { + "name": "Mendozo Hernández P", + "authtype": "Author", + "clusterid": "" + } + ], + "lastauthor": "Mendozo Hernández P", + "title": "[Clinical diagnosis and therapy. Intravenous and oral rehydration].", + "sorttitle": "clinical diagnosis and therapy intravenous and oral rehydration", + "volume": "78", + "issue": "4", + "pages": "307-17", + "lang": [ + "spa" + ], + "nlmuniqueid": "0414762", + "issn": "0030-0632", + "essn": "", + "pubtype": [ + "Journal Article" + ], + "recordstatus": "PubMed - indexed for MEDLINE", + "pubstatus": "4", + "articleids": [ + { + "idtype": "pubmed", + "idtypen": 1, + "value": "123455" + }, + { + "idtype": "rid", + "idtypen": 8, + "value": "123455" + }, + { + "idtype": "eid", + "idtypen": 8, + "value": "123455" + } + ], + "history": [ + { + "pubstatus": "pubmed", + "date": "1975/04/01 00:00" + }, + { + "pubstatus": "medline", + "date": "1975/04/01 00:01" + }, + { + "pubstatus": "entrez", + "date": "1975/04/01 00:00" + } + ], + "references": [ + ], + "attributes": [ + ], + "pmcrefcount": "", + "fulljournalname": "Boletin de la Oficina Sanitaria Panamericana. Pan American Sanitary Bureau", + "elocationid": "", + "doctype": "citation", + "srccontriblist": [ + ], + "booktitle": "", + "medium": "", + "edition": "", + "publisherlocation": "", + "publishername": "", + "srcdate": "", + "reportnumber": "", + "availablefromurl": "", + "locationlabel": "", + "doccontriblist": [ + ], + "docdate": "", + "bookname": "", + "chapter": "", + "sortpubdate": "1975/04/01 00:00", + "sortfirstauthor": "Mendozo Hernández P", + "vernaculartitle": "Diagnóstico clínico y terapéutica. Rehidratación oral e intravenosa" + } + } +} + diff --git a/test/testdata/8119947f1f0f53567e3663cb4b963b5ba51a1787.json b/test/testdata/8119947f1f0f53567e3663cb4b963b5ba51a1787.json new file mode 100644 index 00000000..008092d5 --- /dev/null +++ b/test/testdata/8119947f1f0f53567e3663cb4b963b5ba51a1787.json @@ -0,0 +1,26 @@ +{ + "encoding": "UTF-8", + "headers": { + "Access-Control-Allow-Origin": "*", + "Cache-Control": "private", + "Connection": "Keep-Alive", + "Content-Security-Policy": "upgrade-insecure-requests", + "Content-Type": "application/json; charset=UTF-8", + "Date": "Fri, 01 Mar 2019 07:22:43 GMT", + "Keep-Alive": "timeout=4, max=40", + "NCBI-PHID": "D0BD25D15D723FE50000236D6E04FBE0.1.1.m_1", + "NCBI-SID": "E040C89A49735989_C22CSID", + "Server": "Finatra", + "Set-Cookie": "ncbi_sid=E040C89A49735989_C22CSID; domain=.nih.gov; path=/; expires=Sun, 01 Mar 2020 07:22:43 GMT", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "Transfer-Encoding": "chunked", + "X-RateLimit-Limit": "3", + "X-RateLimit-Remaining": "3", + "X-UA-Compatible": "IE=Edge", + "X-XSS-Protection": "1; mode=block", + "content-encoding": "gzip", + "l5d-success-class": "1.0" + }, + "status_code": 200, + "url": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?api_key=dad47b304cafdc0790b32d335e3e3a403c08&retmode=json&tool=5j9.citer@github.com&email=dalba.wiki@gmail.com&db=pubmed&id=123455" +} \ No newline at end of file diff --git a/test/testdata/849b623b1fd15d27f08dd9609c6fac990aa5405e.html b/test/testdata/849b623b1fd15d27f08dd9609c6fac990aa5405e.html new file mode 100644 index 00000000..3fa0165d --- /dev/null +++ b/test/testdata/849b623b1fd15d27f08dd9609c6fac990aa5405e.html @@ -0,0 +1,11 @@ +@Book{noorlib3232, +Title = {الكامل في التاريخ}, +Year = {}, +Url = {https://www.noorlib.ir/View/fa/Book/BookView/Image/3232}, +publisher = {دار صادر}, +address = {بیروت - لبنان}, +author = {ابن اثیر, علی بن محمد}, +Series = {الكامل في التاريخ}, +Volume = {13} +Language = {عربی} +} \ No newline at end of file diff --git a/test/testdata/849b623b1fd15d27f08dd9609c6fac990aa5405e.json b/test/testdata/849b623b1fd15d27f08dd9609c6fac990aa5405e.json new file mode 100644 index 00000000..2177ae8c --- /dev/null +++ b/test/testdata/849b623b1fd15d27f08dd9609c6fac990aa5405e.json @@ -0,0 +1,16 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "private", + "Content-Disposition": "attachment; filename=\"noorlib-3232.bib\"", + "Content-Length": "339", + "Content-Type": "application/x-bibtex; charset=UTF-8", + "Date": "Fri, 13 Apr 2018 07:57:53 GMT", + "Server": "Microsoft-IIS/7.5", + "Set-Cookie": "ASP.NET_SessionId=40ppz5uynio5hhaf5xq5opww; path=/; HttpOnly", + "X-AspNet-Version": "4.0.30319", + "X-Powered-By": "ASP.NET" + }, + "status_code": 200, + "url": "https://www.noorlib.ir/View/HttpHandler/CitationHandler.ashx?id=3232&format=BibTex" +} \ No newline at end of file diff --git a/test/testdata/86c4cac52079ad2c72968d0ef8e77222ca5584e6.html b/test/testdata/86c4cac52079ad2c72968d0ef8e77222ca5584e6.html new file mode 100644 index 00000000..2e851072 --- /dev/null +++ b/test/testdata/86c4cac52079ad2c72968d0ef8e77222ca5584e6.html @@ -0,0 +1,681 @@ + + + + + + دیوان-کامل-حافظ-همراه-با-فالنامه | دیوان | خانه کتاب و ادبیات ایران + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
        + + + + + + +
        +
        +
        +
        + + + + + + + +
        + + + + + +
        + + + ورود + + ثبت نام + + + + + + +
        +
        +
        +
        + + +
        + + + + +
        +
        +
        +
        +
        + دیوان کامل حافظ همراه با فالنامه | خانه کتاب و ادبیات ایران +
        +
        +
        + صفحات اولیه کتاب +

        + دیوان کامل حافظ همراه با فالنامه

        +

        + + + حافظ، شمس‌الدین‌محمد، - 792 ق. - فال‌گیری + + + شعر فارسی - قرن 8ق. + + +

        + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
        پدیدآور + + نويسنده : + + حافظ ، شمس‌الدین‌محمد + - + + + به‌اهتمام : + + نظرزاده ، رضا + + +
        ناشر + + + + دیوان + + (برای تماس با ناشر و خرید کتاب کلیک کنید) + + + + +
        شابک978-964-92962-6-5
        تاریخ نشر + +13850904 +
        قیمت +15,000
        کد دیویی8fa1.32
        زبان کتابفارسی
        محل نشرقم - قم
        توضیحات + جلد - + 494 صفحه - + تالیف - + چاپ 1 +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        معرفی مختصر کتاب
        +

        + +

        +
        +
        +
        +
        +
        + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/testdata/86c4cac52079ad2c72968d0ef8e77222ca5584e6.json b/test/testdata/86c4cac52079ad2c72968d0ef8e77222ca5584e6.json new file mode 100644 index 00000000..5cdae01d --- /dev/null +++ b/test/testdata/86c4cac52079ad2c72968d0ef8e77222ca5584e6.json @@ -0,0 +1,22 @@ +{ + "encoding": "utf-8", + "headers": { + "AR-ATIME": "0.296", + "AR-CACHE": "BYPASS", + "AR-PoweredBy": "Arvan Cloud (arvancloud.com)", + "AR-Request-ID": "33cee1c2281f991c24c43af810c199e4", + "AR-SID": "2012", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Security-Policy": "upgrade-insecure-requests", + "Content-Type": "text/html; charset=utf-8", + "Date": "Sat, 27 Aug 2022 23:28:57 GMT", + "Keep-Alive": "timeout=65", + "Server": "ArvanCloud", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding, Accept-Encoding", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://ketab.ir/book/bb12c0da-6ecc-4e84-90b8-3477998ba644" +} \ No newline at end of file diff --git a/test/testdata/881f3ddad23a19b062cd1ab47b768046657d90bc.html b/test/testdata/881f3ddad23a19b062cd1ab47b768046657d90bc.html new file mode 100644 index 00000000..83275f8b --- /dev/null +++ b/test/testdata/881f3ddad23a19b062cd1ab47b768046657d90bc.html @@ -0,0 +1,88 @@ + + + + + + + + + +ページが見つかりませんでした | 私とあの人の運命の出会い + + + + + + + + + + + + + + +
        + + +
        +
        +
        + +
        +
        +

        おっと、失礼しました。

        +
        + +
        +

        お探しのコンテンツを見つけられませんでした。検索をお試しください。

        +
        +
        + +
        +
        + +
        + +
        + + + + \ No newline at end of file diff --git a/test/testdata/881f3ddad23a19b062cd1ab47b768046657d90bc.json b/test/testdata/881f3ddad23a19b062cd1ab47b768046657d90bc.json new file mode 100644 index 00000000..82f20238 --- /dev/null +++ b/test/testdata/881f3ddad23a19b062cd1ab47b768046657d90bc.json @@ -0,0 +1,15 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "no-cache, must-revalidate, max-age=0", + "Connection": "close", + "Content-Type": "text/html; charset=UTF-8", + "Date": "Sat, 14 Apr 2018 14:57:47 GMT", + "Expires": "Wed, 11 Jan 1984 05:00:00 GMT", + "Pragma": "no-cache", + "Server": "Apache", + "X-Pingback": "http://www.londondevelopmentcentre.org/xmlrpc.php" + }, + "status_code": 404, + "url": "http://www.londondevelopmentcentre.org/page.php?s=1&p=2462" +} \ No newline at end of file diff --git a/test/testdata/88bd207777d978a9476c55377d5f483c6cf8c523.html b/test/testdata/88bd207777d978a9476c55377d5f483c6cf8c523.html new file mode 100644 index 00000000..01eb8416 --- /dev/null +++ b/test/testdata/88bd207777d978a9476c55377d5f483c6cf8c523.html @@ -0,0 +1,2548 @@ + + + + + +Israel says it has shot down drone launched from Gaza | World news | The Guardian + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
        +
        +
        +
        + + +
        +
        +
        +
        +
        + +
        + +
        + + + +
        + +
        +
        +
        +
        +
        +
        + +

        +Israel says it has shot down drone launched from Gaza +

        +
        +
        +
        +
        +
        +
        +
        + +Unmanned aircraft is said to have been downed using Patriot missile near city of Ashdod on southern coast +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        + + + + +
        + + + + + + + + + + + + + +Gaza City + +
        + + +
        + + + + + +Palestinians drive past a building destroyed by an Israeli strike in Gaza City. Photograph: Lefteris Pitarakis/AP +
        +
        +
        +
        +
        +
        +
        + +

        +Israel says it has shot down drone launched from Gaza +

        +
        +
        +
        +
        +
        +
        +
        + +Unmanned aircraft is said to have been downed using Patriot missile near city of Ashdod on southern coast +
        +
        +
        +
        +
        +
        + +
        +

        Israel claims to have downed a drone from the skies above its southern coastline. The unmanned aircraft, which Israel says was launched from Gaza and shot down with a Patriot missile near the city of Ashdod, is the first weapon of its kind Israel has encountered in this conflict.

        +

        Rockets were also fired into Israel from Lebanon early on Monday morning, drawing retaliatory fire from Israeli forces. This is the third such rocket attack from Lebanon since Friday. There were no reported casualties.

        +

        In the West Bank, a 21-year-old Palestinian was killed in Hebron after clashes between the Israeli military and protesters against the war in Gaza – the first Palestinian casualty in the West Bank since the conflict began.

        +

        The dead man – identified in unconfirmed reports as Muneer al-Bardeen – was shot during a protest at al-Samoua junction, 20 minutes south of Hebron.

        +
        +
        + +
        +
        +

        Witnesses said Bardeen was shot with live ammunition at around 3am, after hours of clashes that began after Iftar, the breaking of the Ramadan fast, and carried on throughout the night.

        +

        On Monday morning, Palestinian media reported that two other men were injured in the clashes – Mahmoud Yasser Muhammad Breghith, 21, who was shot in the leg, and Mahmoud Nasser Juma Hitawi, 20, shot in the foot.

        +

        Hebron is one of the cities most heavily affected by Israel's incursions into the West Bank following the kidnapping of three Israeli teenagers in June. The two main suspects in the murder of the three boys are Hamas members living in Hebron. They are still at large.

        +

        An Israeli Defence Force spokesman could not be reached for comment.

        +

        At least 172 Palestinians have been killed in the Gaza Strip since the Israeli offensive began seven days ago. The UN estimates that 77% of those killed have been civilians.

        +

        The Palestinian president, Mahmoud Abbas, was set to discuss moves to seek UN intervention at an emergency meeting of Arab foreign ministers in Cairo on Monday after another night of air strikes on Gaza in which Israeli war planes struck more than 40 sites, including three training facilities used by Hamas's armed wing, killing two people.

        +

        The UN chief, Ban Ki-moon, urged Israel to scrap plans for a ground offensive, saying "too many" Palestinian civilians had been killed.

        +

        Thousands of Palestinians fled their homes in two northern areas of the coastal enclave on Sunday after Israel warned it would "strike with might" against what it says are rocket-launching sites – an attack that has not yet materialised.

        +

        The exodus from Beit Lahia and Attatra came after Israel dropped leaflets and sent text messages warning civilians to evacuate northern Gaza by midday on Sunday in advance of a large-scale bombing campaign. The area is home to at least 100,000 people.

        +

        The UN says 17,000 people have sought refuge in its facilities.

        +

        A senior Israeli military officer, in a telephone briefing with foreign reporters, said Israel would strike the Beit Lahia area from late evening on Sunday. "The enemy has built rocket infrastructure in between the houses [in Beit Lahia]," the officer said. "He wants to trap me into an attack and into hurting civilians."

        +

        The leaflet warned: "Those who fail to comply with the instructions will endanger their lives and the lives of their families. Beware."

        +
        + + + +
        + + + + + + + + + +Israel Gaza raid + +
        +
        +

        As the ultimatum drew near, large numbers raced by in pickup trucks or on donkey carts, waving white flags, with many heading to UN-run schools that were taking in refugees. "They are sending warning messages," said one resident, Mohammad Abu Halemah. "Once we received the message, we felt scared to stay in our homes. We want to leave."

        +

        Outside one UN school, there were rows of horses tied up by families anxious to protect their animals.

        +

        During a visit to Beit Lahia after the deadline had expired, the Guardian saw that most residents had opted to stay in their homes. Some shops were open and hospitals called for volunteers from medical schools to help treat an expected influx of casualties.

        +

        The warning was issued hours after Israeli naval commandos launched an early morning raid on a beach in the Sudaniya neighbourhood in the north of Gaza City, targeting another rocket-launching site. On Saturday, the coastal enclave suffered the bloodiest day of the six-day Israeli assault, with 54 Palestinians reported killed.

        +

        There has been speculation that Israel may launch a ground offensive into Gaza, a move likely to sharply increase the number of civilian casualties. At least 30 children were among those killed, according to Gaza's health ministry. There have been several Israeli injuries but no fatalities.

        +

        In the worst single incident of the conflict so far, at least 17 people were killed and 45 injured when two large Israeli bombs hit a house in the Tuffah neighbourhood of Gaza City where the city's chief of police, Tayseer al-Batsh, was sheltering. Five other people were missing, presumed dead.

        +

        Most of the injured were returning home from a mosque when they were caught by shrapnel from the blast.

        +

        Israel has been massing tanks and soldiers at Gaza's borders, which some fear could signal a wider ground offensive that would cause heavy casualties. "We don't know when the operation will end," the Israeli prime minister, Binyamin Netanyahu, told a cabinet meeting on Sunday. "It might take a long time."

        +

        The beach raid by several dozen commandos at 2am on Sunday was the first time Israeli forces have set foot in Gaza since the beginning of the current campaign. Four commandos were reportedly lightly injured after apparently being spotted approaching and being engaged by waiting Palestinian fighters.

        +

        Saad al-Dawla, the night watchman of the Mathaf hotel, said he was sleeping when the commandos came to the beach. "I was sleeping in the lobby with a friend. At the beginning we heard shooting from the Palestinian side. I got up and looked out the window and saw that there were people shooting from the water. Almost immediately an [Israeli] helicopter came and started shooting at the water as well," he said. "Later I heard shelling from the sea and the sounded of a warship's siren. The whole thing last about two hours."

        +

        Asked whether Hamas or other groups had watchers near the beach, Dawla said he did not know. Ladders at a mosque overlooking the beachfront and leading to its tower strongly suggested that a sentry had been posted there.

        +

        Israel has launched more than 1,300 air strikes since the offensive began, the military spokesman Lt Col Peter Lerner said. Palestinian militants have launched more than 800 rockets at Israel, according to the Israeli military.

        +

        Israel has said it is acting in self-defence against rockets that have disrupted life across much of the country. It also accuses Hamas of using Gaza's civilians as human shields.

        +

        Critics say Israel's heavy bombardment of one of the most densely populated territories in the world is the main factor putting civilians at risk.

        +
        + +
        +
        + +
        +
        +
        + +
        + + + + + + + + + + + diff --git a/test/testdata/88bd207777d978a9476c55377d5f483c6cf8c523.json b/test/testdata/88bd207777d978a9476c55377d5f483c6cf8c523.json new file mode 100644 index 00000000..8cc43e5e --- /dev/null +++ b/test/testdata/88bd207777d978a9476c55377d5f483c6cf8c523.json @@ -0,0 +1,34 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "0", + "Cache-Control": "max-age=60, stale-while-revalidate=6, stale-if-error=864000, private", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "87573", + "Content-Security-Policy": "default-src https:; script-src https: 'unsafe-inline' 'unsafe-eval'; style-src https: 'unsafe-inline'; img-src https: data: blob:; media-src https: data: blob:; font-src https: data:; connect-src https: wss:; report-uri https://beacon.gu-web.net/csp-report", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:55:01 GMT", + "ETag": "W/\"hash-164974856314823728\"", + "Expires": "Tue, 23 May 2017 17:56:00 GMT", + "Fastly-Debug-Digest": "3d57f6073722ba18a938103b91de9f1050298a4626267a5a32cc0c30af9cbb80", + "Link": "; rel=preload; as=style; nopush,; rel=preload; as=script; nopush,; rel=preload; as=script; nopush,; rel=preload; as=script; nopush", + "Set-Cookie": "GU_mvt_id=337463; expires=Mon, 21 Aug 2017 17:55:01 GMT; path=/; domain=.theguardian.com, GU_geo_continent=NA; path=/;", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "Vary": "Accept-Encoding,User-Agent", + "Via": "1.1 varnish, 1.1 varnish", + "X-Cache": "MISS, MISS", + "X-Cache-Hits": "0, 0", + "X-Content-Type-Options": "nosniff", + "X-Frame-Options": "SAMEORIGIN", + "X-GU-Edition": "us", + "X-GU-Platform": "next-gen-router", + "X-Gu-Backend-App": "article", + "X-Served-By": "cache-lcy1133-LCY, cache-iad2651-IAD", + "X-Timer": "S1495562101.825155,VS0,VE282", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://www.theguardian.com/world/2014/jul/14/israel-drone-launched-gaza-ashdod" +} \ No newline at end of file diff --git a/test/testdata/89483dcac9ba26766a955a56f5aac3885ce76b60.html b/test/testdata/89483dcac9ba26766a955a56f5aac3885ce76b60.html new file mode 100644 index 00000000..1f896cb2 --- /dev/null +++ b/test/testdata/89483dcac9ba26766a955a56f5aac3885ce76b60.html @@ -0,0 +1 @@ +{"indexed":{"date-parts":[[2022,5,31]],"date-time":"2022-05-31T16:28:05Z","timestamp":1654014485081},"reference-count":27,"publisher":"Elsevier BV","issue":"6","license":[{"start":{"date-parts":[[2012,6,1]],"date-time":"2012-06-01T00:00:00Z","timestamp":1338508800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"}],"content-domain":{"domain":[],"crossmark-restriction":false},"published-print":{"date-parts":[[2012,6]]},"DOI":"10.1016\/j.mayocp.2012.02.015","type":"journal-article","created":{"date-parts":[[2012,6,4]],"date-time":"2012-06-04T16:10:01Z","timestamp":1338826201000},"page":"596-602","source":"Crossref","is-referenced-by-count":36,"title":"Evaluating the Patient With Diarrhea: A Case-Based Approach","prefix":"10.1016","volume":"87","author":[{"given":"Seth","family":"Sweetser","sequence":"first","affiliation":[]}],"member":"78","reference":[{"issue":"4","key":"10.1016\/j.mayocp.2012.02.015_bib1","doi-asserted-by":"crossref","first-page":"927","DOI":"10.1016\/0016-5085(91)90717-Y","article-title":"Epidemiology of colonic symptoms and irritable bowel syndrome","volume":"101","author":"Talley","year":"1991","journal-title":"Gastroenterology"},{"issue":"8","key":"10.1016\/j.mayocp.2012.02.015_bib2","first-page":"1160","article-title":"Self-reported diarrhea: what does it mean?","volume":"89","author":"Talley","year":"1994","journal-title":"Am J Gastroenterol"},{"issue":"2","key":"10.1016\/j.mayocp.2012.02.015_bib3","doi-asserted-by":"crossref","first-page":"164","DOI":"10.1136\/gut.27.2.164","article-title":"Bowel function measurements of individuals with different eating patterns","volume":"27","author":"Davies","year":"1986","journal-title":"Gut"},{"issue":"5","key":"10.1016\/j.mayocp.2012.02.015_bib4","doi-asserted-by":"crossref","first-page":"1259","DOI":"10.1016\/S0025-7125(05)70286-8","article-title":"Diarrhea","volume":"84","author":"Schiller","year":"2000","journal-title":"Med Clin North Am"},{"issue":"8285","key":"10.1016\/j.mayocp.2012.02.015_bib5","doi-asserted-by":"crossref","first-page":"1349","DOI":"10.1016\/S0140-6736(82)92413-8","article-title":"Faecal incontinence: the unvoiced symptom","volume":"1","author":"Leigh","year":"1982","journal-title":"Lancet"},{"issue":"3","key":"10.1016\/j.mayocp.2012.02.015_bib6","doi-asserted-by":"crossref","first-page":"481","DOI":"10.1016\/j.gtc.2009.06.008","article-title":"Diarrhea and malabsorption in the elderly","volume":"38","author":"Schiller","year":"2009","journal-title":"Gastroenterol Clin N Am"},{"issue":"1","key":"10.1016\/j.mayocp.2012.02.015_bib7","doi-asserted-by":"crossref","first-page":"53","DOI":"10.2165\/00002018-200022010-00005","article-title":"Drug-induced diarrhoea","volume":"22","author":"Chassany","year":"2000","journal-title":"Drug Saf"},{"issue":"5","key":"10.1016\/j.mayocp.2012.02.015_bib8","doi-asserted-by":"crossref","first-page":"365","DOI":"10.1007\/s11894-007-0044-x","article-title":"Drug-induced diarrhea","volume":"9","author":"Abraham","year":"2007","journal-title":"Curr Gastroenterol Rep"},{"issue":"3","key":"10.1016\/j.mayocp.2012.02.015_bib9","doi-asserted-by":"crossref","first-page":"245","DOI":"10.2165\/00002512-199813030-00007","article-title":"Mechanisms of drug-induced diarrhoea in the elderly","volume":"13","author":"Ratnaike","year":"1998","journal-title":"Drugs Aging"},{"key":"10.1016\/j.mayocp.2012.02.015_bib10","series-title":"Sleisenger and Fordtran's Gastrointestinal and Liver Disease: Pathophysiology, Diagnosis, Management","first-page":"211","article-title":"Diarrhea","author":"Schiller","year":"2010"},{"issue":"5","key":"10.1016\/j.mayocp.2012.02.015_bib11","doi-asserted-by":"crossref","first-page":"738","DOI":"10.1016\/S0016-5085(76)80353-8","article-title":"Effect of caffeine on the human small intestine","volume":"71","author":"Wald","year":"1976","journal-title":"Gastroenterology"},{"issue":"1","key":"10.1016\/j.mayocp.2012.02.015_bib12","doi-asserted-by":"crossref","first-page":"31","DOI":"10.1097\/SPC.0b013e32832531bb","article-title":"Chemotherapy-induced diarrhoea","volume":"3","author":"Gibson","year":"2009","journal-title":"Curr Opin Support Palliat Care"},{"issue":"11","key":"10.1016\/j.mayocp.2012.02.015_bib13","doi-asserted-by":"crossref","first-page":"725","DOI":"10.1056\/NEJM199503163321107","article-title":"Evaluation of patients with chronic diarrhea","volume":"332","author":"Donowitz","year":"1995","journal-title":"N Engl J Med"},{"issue":"18","key":"10.1016\/j.mayocp.2012.02.015_bib14","doi-asserted-by":"crossref","first-page":"973","DOI":"10.1056\/NEJM197205042861804","article-title":"Floating stools\u2014flatus versus fat","volume":"286","author":"Levitt","year":"1972","journal-title":"N Engl J Med"},{"issue":"4","key":"10.1016\/j.mayocp.2012.02.015_bib15","doi-asserted-by":"crossref","first-page":"1079","DOI":"10.1093\/ajcn\/48.4.1142","article-title":"The acceptability of milk and milk products in populations with a high prevalence of lactose intolerance","volume":"48","author":"Scrimshaw","year":"1988","journal-title":"Am J Clin Nutr"},{"issue":"2","key":"10.1016\/j.mayocp.2012.02.015_bib16","doi-asserted-by":"crossref","first-page":"545","DOI":"10.1016\/0016-5085(92)90845-P","article-title":"Fecal osmotic gap and pH in experimental diarrhea of various causes","volume":"103","author":"Eherer","year":"1992","journal-title":"Gastroenterology"},{"issue":"6","key":"10.1016\/j.mayocp.2012.02.015_bib17","doi-asserted-by":"crossref","first-page":"1936","DOI":"10.1172\/JCI114927","article-title":"Carbohydrate malabsorption: its measurement and its contribution to diarrhea","volume":"86","author":"Hammer","year":"1990","journal-title":"J Clin Invest"},{"issue":"5","key":"10.1016\/j.mayocp.2012.02.015_bib18","doi-asserted-by":"crossref","first-page":"389","DOI":"10.1007\/s11894-999-0020-8","article-title":"Secretory diarrhea","volume":"1","author":"Schiller","year":"1999","journal-title":"Curr Gastroenterol Rep"},{"issue":"10","key":"10.1016\/j.mayocp.2012.02.015_bib19","doi-asserted-by":"crossref","first-page":"2216","DOI":"10.1007\/BF02090374","article-title":"Diagnostic value of fasting plasma peptide concentrations in patients with chronic diarrhea","volume":"39","author":"Schiller","year":"1994","journal-title":"Dig Dis Sci"},{"issue":"10","key":"10.1016\/j.mayocp.2012.02.015_bib20","doi-asserted-by":"crossref","first-page":"1379","DOI":"10.1136\/gut.30.10.1379","article-title":"Prevalence of surreptitious laxative abuse in patients with diarrhoea of uncertain origin: a cost benefit analysis of a screening procedure","volume":"30","author":"Bytzer","year":"1989","journal-title":"Gut"},{"issue":"suppl 5","key":"10.1016\/j.mayocp.2012.02.015_bib21","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1136\/gut.52.suppl_5.v1","article-title":"Guidelines for the investigation of chronic diarrhea, 2nd edition","volume":"52","author":"Thomas","year":"2003","journal-title":"Gut"},{"issue":"20","key":"10.1016\/j.mayocp.2012.02.015_bib22","doi-asserted-by":"crossref","first-page":"1418","DOI":"10.1056\/NEJM199405193302004","article-title":"Brief report: factitious diarrhea detected by measurement of stool osmolality","volume":"330","author":"Topazian","year":"1994","journal-title":"N Engl J Med"},{"issue":"6","key":"10.1016\/j.mayocp.2012.02.015_bib23","first-page":"609","article-title":"Dilutional diarrhea: underdiagnosed and over-investigated","volume":"12","author":"Pollock","year":"2000","journal-title":"Eur J Gastroenterol Hepatol"},{"issue":"4","key":"10.1016\/j.mayocp.2012.02.015_bib24","doi-asserted-by":"crossref","first-page":"584","DOI":"10.1097\/00005176-200210000-00026","article-title":"More on factitious diarrhea","volume":"35","author":"Zimmer","year":"2002","journal-title":"J Pediatr Gastroenterol Nutr"},{"key":"10.1016\/j.mayocp.2012.02.015_bib25","doi-asserted-by":"crossref","first-page":"723","DOI":"10.1016\/S0300-5089(21)00746-X","article-title":"Factitious diarrhoea","volume":"15","author":"Ewe","year":"1986","journal-title":"Clin Gastroenterol"},{"issue":"5","key":"10.1016\/j.mayocp.2012.02.015_bib26","doi-asserted-by":"crossref","first-page":"631","DOI":"10.1016\/0002-9610(51)90432-1","article-title":"Melanosis coli: experimental observations of its production and elimination in twenty-three cases","volume":"82","author":"Speare","year":"1951","journal-title":"Am J Surg"},{"issue":"8","key":"10.1016\/j.mayocp.2012.02.015_bib27","doi-asserted-by":"crossref","first-page":"2165","DOI":"10.1111\/j.1572-0241.1999.01289.x","article-title":"The prevalence of chronic diarrhea among diabetic patients","volume":"94","author":"Lysy","year":"1999","journal-title":"Am J Gastroenterol"}],"container-title":"Mayo Clinic Proceedings","original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0025619612003825?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0025619612003825?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2022,1,16]],"date-time":"2022-01-16T17:03:28Z","timestamp":1642352608000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0025619612003825"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2012,6]]},"references-count":27,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2012,6]]}},"alternative-id":["S0025619612003825"],"URL":"http:\/\/dx.doi.org\/10.1016\/j.mayocp.2012.02.015","relation":{},"ISSN":["0025-6196"],"subject":["General Medicine"],"container-title-short":"Mayo Clinic Proceedings","published":{"date-parts":[[2012,6]]}} \ No newline at end of file diff --git a/test/testdata/89483dcac9ba26766a955a56f5aac3885ce76b60.json b/test/testdata/89483dcac9ba26766a955a56f5aac3885ce76b60.json new file mode 100644 index 00000000..e1c4676e --- /dev/null +++ b/test/testdata/89483dcac9ba26766a955a56f5aac3885ce76b60.json @@ -0,0 +1,24 @@ +{ + "encoding": null, + "headers": { + "access-control-allow-headers": "X-Requested-With, Accept, Accept-Encoding, Accept-Charset, Accept-Language, Accept-Ranges, Cache-Control", + "access-control-allow-origin": "*", + "access-control-expose-headers": "Link", + "connection": "close", + "content-encoding": "gzip", + "content-length": "2920", + "content-type": "application/vnd.citationstyles.csl+json", + "date": "Thu, 09 Jun 2022 10:37:35 GMT", + "link": "; rel=\"canonical\", ; version=\"vor\"; type=\"text/xml\"; rel=\"item\", ; version=\"vor\"; type=\"text/plain\"; rel=\"item\", ; version=\"tdm\"; rel=\"license\"", + "permissions-policy": "interest-cohort=()", + "server": "Jetty(9.4.40.v20210413)", + "vary": "Accept, Accept-Encoding", + "x-api-pool": "public", + "x-rate-limit-interval": "1s", + "x-rate-limit-limit": "50", + "x-ratelimit-interval": "1s", + "x-ratelimit-limit": "50" + }, + "status_code": 200, + "url": "https://api.crossref.org/v1/works/10.1016%2Fj.mayocp.2012.02.015/transform" +} \ No newline at end of file diff --git a/test/testdata/89844d21505f5247724663e0d2aa3af6ab03b52a.html b/test/testdata/89844d21505f5247724663e0d2aa3af6ab03b52a.html new file mode 100644 index 00000000..8f3a3b34 --- /dev/null +++ b/test/testdata/89844d21505f5247724663e0d2aa3af6ab03b52a.html @@ -0,0 +1,495 @@ + + + + + + +تحقیقات تاریخ اجتماعی + + + + + + + + + + + + + + + + + +

        تحقیقات تاریخ اجتماعی (SHC)

        +
        + + + + +
        +
        +
        +
        + +
        + + + + + +
        + +
        + +
        +
        +
        +
        +
        + + + + +
        + +
        + + + diff --git a/test/testdata/89844d21505f5247724663e0d2aa3af6ab03b52a.json b/test/testdata/89844d21505f5247724663e0d2aa3af6ab03b52a.json new file mode 100644 index 00000000..9337e9cc --- /dev/null +++ b/test/testdata/89844d21505f5247724663e0d2aa3af6ab03b52a.json @@ -0,0 +1,21 @@ +{ + "encoding": "ISO-8859-1", + "headers": { + "Cache-Control": "no-store, no-cache, must-revalidate, post-check=0, pre-check=0", + "Connection": "close", + "Content-Encoding": "gzip", + "Content-Length": "8966", + "Content-Type": "text/html", + "Date": "Sun, 28 May 2017 05:54:52 GMT", + "Expires": "Thu, 19 Nov 1981 08:52:00 GMT", + "Pragma": "no-cache", + "Server": "Apache/2.4.7 (Ubuntu)", + "Set-Cookie": "juFirstLang=fa; expires=Tue, 27-Jun-2017 05:46:07 GMT; Max-Age=2592000; path=/; httponly, juSecondLang=en; expires=Tue, 27-Jun-2017 05:46:07 GMT; Max-Age=2592000; path=/; httponly, PHPSESSID=e84214a3gdhku7fov1mdmscjj6; path=/; HttpOnly", + "Vary": "Accept-Encoding", + "X-Cache": "MISS from google.com", + "X-Cache-Lookup": "MISS from google.com:85", + "X-Powered-By": "PHP/5.5.9-1ubuntu4.5" + }, + "status_code": 200, + "url": "http://socialhistory.ihcs.ac.ir/" +} \ No newline at end of file diff --git a/test/testdata/8cd9257aef43fabeed45d235cf7cebfd350218e3.html b/test/testdata/8cd9257aef43fabeed45d235cf7cebfd350218e3.html new file mode 100644 index 00000000..14c95dbd --- /dev/null +++ b/test/testdata/8cd9257aef43fabeed45d235cf7cebfd350218e3.html @@ -0,0 +1,725 @@ + + + + + + تصویر کتاب ایران در زمان ساسانیان: تاریخ ایران ساسانی تا حمله عرب و وضع دولت و ملت در زمان ساسانیان - جلد 1 - صفحه 1 - کریستن سن، آرتور امانویل + + + + + + + + + + + + + + + + + + +
        +
        + + + + +
        +
        + + + +
        + + +
        +
        +
        + + + +
        +
        +
        + + +
        + +
        +
        + + + + + + + + + +
        + +
        +
        +
        +
        + + +
        +
        +
        +
        + + +
        + +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        + + + + + + + + +
        +
        +
        + +
        + + + + +
        +
        + +
        + +
        + +
        + +
        + + +
        +
        + + + + + diff --git a/test/testdata/8cd9257aef43fabeed45d235cf7cebfd350218e3.json b/test/testdata/8cd9257aef43fabeed45d235cf7cebfd350218e3.json new file mode 100644 index 00000000..9ad02c84 --- /dev/null +++ b/test/testdata/8cd9257aef43fabeed45d235cf7cebfd350218e3.json @@ -0,0 +1,17 @@ +{ + "encoding": "utf-8", + "headers": { + "Cache-Control": "private", + "Content-Encoding": "gzip", + "Content-Length": "15641", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:51:50 GMT", + "Server": "Microsoft-IIS/7.5", + "Set-Cookie": "ASP.NET_SessionId=j4r143dsyn3nxit12mahdp1m; path=/; HttpOnly", + "Vary": "Accept-Encoding", + "X-AspNet-Version": "4.0.30319", + "X-Powered-By": "ASP.NET" + }, + "status_code": 200, + "url": "http://www.noorlib.ir/View/fa/Book/BookView/Image/6120" +} \ No newline at end of file diff --git a/test/testdata/8d20a9197e5197bb6d18de551e6364c3aafa30a8.html b/test/testdata/8d20a9197e5197bb6d18de551e6364c3aafa30a8.html new file mode 100644 index 00000000..e5962a10 --- /dev/null +++ b/test/testdata/8d20a9197e5197bb6d18de551e6364c3aafa30a8.html @@ -0,0 +1 @@ +Woman who lost brother on MH370 mourns relatives on board MH17 | The Times & The Sunday Times
        Subscription Notification
        We have noticed that there is an issue with your subscription billing details. Please update your billing details here
        Please update your billing information
        The subscription details associated with this account need to be updated. Please update your billing details here to continue enjoying your subscription.
        Your subscription will end shortly
        Please update your billing details here to continue enjoying your access to the most informative and considered journalism in the UK.
        Read the full article
        Just register a few details.

        Woman who lost brother on MH370 mourns relatives on board MH17

        Victims of missing flight MH370 are remembered. The brother and sister-in-law of Kaylene Mann, of Brisbane, were on board the flight, which vanished in MarchGetty Images

        An Australian woman whose brother and sister-in-law died when Malaysian Airlines Flight MH370 vanished four months ago has lost two more close relatives aboard the airliner shot down in Ukraine.

        Kaylene Mann, from Brisbane, was too distraught to speak publicly yesterday. She was the sister of Rod Burrows, who died with his wife, Mary, on MH370, which disappeared over the southern Indian Ocean on March 8. No trace of the aircraft has been found. Both the lost aircraft were identical long-range Boeing 777 ER twinjets

        Mrs Mann’s stepdaughter, Marie Rizk, and her stepdaughter’s husband, Albert Rizk, from Melbourne, were returning from a European holiday aboard Flight MH17. “It’s just brought everyone, everything back,” Mrs Mann’s brother, Greg Burrows, said. “It’s just ripped our guts again.”

        Want to read more?
        Register with a few details to continue reading this article.
        Already a subscriber? Login

        You are now logged out

        Your choice of two articles a week

        Unlock quality journalism on the topics that you decide matter most

        Register now

        Or enjoy full access

        Subscribe and catch up with all the stories behind the headlines

        Subscribe today

        Already a member? Log in

        \ No newline at end of file diff --git a/test/testdata/8d20a9197e5197bb6d18de551e6364c3aafa30a8.json b/test/testdata/8d20a9197e5197bb6d18de551e6364c3aafa30a8.json new file mode 100644 index 00000000..9cf64ac7 --- /dev/null +++ b/test/testdata/8d20a9197e5197bb6d18de551e6364c3aafa30a8.json @@ -0,0 +1,21 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Cache-Control": "max-age=0, no-cache, no-store", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "9537", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:53:54 GMT", + "ETag": "W/\"840a-rIsPGICF7owghuWSZz7cSA\"", + "Expires": "Tue, 23 May 2017 17:53:54 GMT", + "Pragma": "no-cache", + "Set-Cookie": "bm_sv=B6B7F171ACE12C74D7FC25D5ECA1DC41~U2W/SuGXhpvBrhgEQcCAnj946kyEVued6XSIrhGAhL9LXljoTBmzHdQINTDL87NNqHWarFT9j8eiiDb+HIP9plMENQAeXL23WIad0fzuloxYyP+8cJKuAOrQOfC+DalJuXyZxoQWE72agH7wtXkVPAvtDoEewpdj6Nh9YwE4pVY=; Domain=.thetimes.co.uk; Path=/; Max-Age=7200; HttpOnly", + "Vary": "Accept-Encoding", + "X-NU-AKA-ACS-Version": "2.0", + "X-Varnish": "3840699" + }, + "status_code": 200, + "url": "https://www.thetimes.co.uk/article/woman-who-lost-brother-on-mh370-mourns-relatives-on-board-mh17-r07q5rwppl0" +} \ No newline at end of file diff --git a/test/testdata/8dbd310714017763468c2712dd09e5d544808048.html b/test/testdata/8dbd310714017763468c2712dd09e5d544808048.html new file mode 100644 index 00000000..33f3c7d0 --- /dev/null +++ b/test/testdata/8dbd310714017763468c2712dd09e5d544808048.html @@ -0,0 +1 @@ +[{"itemType":"book","date":"2007","publisher":"Abacus","title":"The war for all the oceans : from Nelson at the Nile to Napoleon at Waterloo","oclc":"137313052","url":"https://www.worldcat.org/oclc/137313052","ISBN":["978-0-349-11916-8","0-349-11916-3"],"place":"London","numPages":"xxvi, 534 pages, 16 unnumbered pages of plates","abstractNote":"Covering large fleet actions, the attempts to destroy the French invasion flotilla, many duels between pairs of ships and small groups of ships, and feats of daring, this title presents an account of the action-packed years of British naval history, during the period of 1800-15. Originally published: London: Little, Brown, 2006","contributor":[["Lesley","Adkins"]],"author":[["Roy","Adkins"]],"accessDate":"2022-01-08","source":["WorldCat"]}] \ No newline at end of file diff --git a/test/testdata/8dbd310714017763468c2712dd09e5d544808048.json b/test/testdata/8dbd310714017763468c2712dd09e5d544808048.json new file mode 100644 index 00000000..2b1ebc33 --- /dev/null +++ b/test/testdata/8dbd310714017763468c2712dd09e5d544808048.json @@ -0,0 +1,37 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "0", + "Connection": "keep-alive", + "NEL": "{ \"report_to\": \"wm_nel\", \"max_age\": 86400, \"failure_fraction\": 0.05, \"success_fraction\": 0.0}", + "Permissions-Policy": "interest-cohort=()", + "Report-To": "{ \"group\": \"wm_nel\", \"max_age\": 86400, \"endpoints\": [{ \"url\": \"https://intake-logging.wikimedia.org/v1/events?stream=w3c.reportingapi.network_error&schema_uri=/w3c/reportingapi/network_error/1.0.0\" }] }", + "Server-Timing": "cache;desc=\"pass\", host;desc=\"cp3052\"", + "Set-Cookie": "WMF-Last-Access=08-Jan-2022;Path=/;HttpOnly;secure;Expires=Wed, 09 Feb 2022 12:00:00 GMT, WMF-Last-Access-Global=08-Jan-2022;Path=/;Domain=.wikipedia.org;HttpOnly;secure;Expires=Wed, 09 Feb 2022 12:00:00 GMT, GeoIP=US:::37.75:-97.82:v4; Path=/; secure; Domain=.wikipedia.org", + "Strict-Transport-Security": "max-age=106384710; includeSubDomains; preload", + "X-Cache": "cp3058 miss, cp3052 pass", + "X-Cache-Status": "pass", + "X-Client-IP": "185.15.56.50", + "access-control-allow-headers": "accept, content-type, content-length, cache-control, accept-language, api-user-agent, if-match, if-modified-since, if-none-match, dnt, accept-encoding", + "access-control-allow-methods": "GET,HEAD", + "access-control-allow-origin": "*", + "access-control-expose-headers": "etag", + "cache-control": "private, max-age=0, s-maxage=0, must-revalidate", + "content-length": "791", + "content-location": "https://en.wikipedia.org/api/rest_v1/data/citation/mediawiki/9780349119168", + "content-security-policy": "default-src 'none'; frame-ancestors 'none'", + "content-type": "application/json; charset=utf-8", + "date": "Sat, 08 Jan 2022 14:55:01 GMT", + "referrer-policy": "origin-when-cross-origin", + "server": "restbase1028", + "vary": "Accept-Encoding", + "x-content-security-policy": "default-src 'none'; frame-ancestors 'none'", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-webkit-csp": "default-src 'none'; frame-ancestors 'none'", + "x-xss-protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://en.wikipedia.org/api/rest_v1/data/citation/mediawiki/9780349119168" +} \ No newline at end of file diff --git a/test/testdata/8e908515af4d407dd1d2f15bdd7591cf6326512b.html b/test/testdata/8e908515af4d407dd1d2f15bdd7591cf6326512b.html new file mode 100644 index 00000000..f3e37ce3 --- /dev/null +++ b/test/testdata/8e908515af4d407dd1d2f15bdd7591cf6326512b.html @@ -0,0 +1,881 @@ + + + + + + + + Business News Daily: Small Business Solutions & Inspiration + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
        + + +
        + +
        + + + +
        + +
        +
        + +
        + +
        +
        + +
        +
        + +
        + +
        + +
        + +
        + + + + All Latest Headlines +
        + + +
        + +
        +
        +

        Start Your Business

        + See More +
        + + +
        +
        +
        +

        Grow Your Business

        + See More +
        + + +
        +
        +
        +

        Build Your Career

        + See More +
        + + +
        +
        +
        +

        Lead Your Team

        + See More +
        + + +
        +
        +
        +

        Find a Solution

        + See More +
        + + +
        + +
        + +
        + + + + + + + + +
        + +
        + +
        + +
        +
        + +
        + + + + + + + + + + + \ No newline at end of file diff --git a/test/testdata/8e908515af4d407dd1d2f15bdd7591cf6326512b.json b/test/testdata/8e908515af4d407dd1d2f15bdd7591cf6326512b.json new file mode 100644 index 00000000..90be2346 --- /dev/null +++ b/test/testdata/8e908515af4d407dd1d2f15bdd7591cf6326512b.json @@ -0,0 +1,21 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "max-age=0, no-cache", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "12450", + "Content-Type": "text/html; charset=UTF-8", + "Date": "Tue, 23 May 2017 17:53:56 GMT", + "Expires": "Tue, 23 May 2017 17:53:56 GMT", + "Link": ";rel=preconnect, ;rel=preconnect, ;rel=preconnect, ;rel=preconnect, ;rel=preconnect, ;rel=preconnect, ;rel=preconnect, ;rel=preconnect, ;rel=preconnect, ;rel=preconnect", + "Pragma": "no-cache", + "Server": "nginx", + "Set-Cookie": "__uzma=592477349c7dc2.76321550; expires=Fri, 21-May-2027 17:53:56 GMT; Max-Age=315360000; path=/, __uzmd=1495562036; expires=Fri, 21-May-2027 17:53:56 GMT; Max-Age=315360000; path=/, __uzmc=623491073517; expires=Fri, 21-May-2027 17:53:56 GMT; Max-Age=315360000; path=/, __uzmb=1495562036; expires=Fri, 21-May-2027 17:53:56 GMT; Max-Age=315360000; path=/", + "Surrogate-Control": "content=\"ESI/1.0\"", + "Vary": "Accept-Encoding, User-Agent", + "X-Akamai-Transformed": "c 9552 0 -" + }, + "status_code": 200, + "url": "http://www.businessnewsdaily.com/" +} \ No newline at end of file diff --git a/test/testdata/8f4699babcfc3823ff41490452a0c47d1c4183d1.html b/test/testdata/8f4699babcfc3823ff41490452a0c47d1c4183d1.html new file mode 100644 index 00000000..1c220162 --- /dev/null +++ b/test/testdata/8f4699babcfc3823ff41490452a0c47d1c4183d1.html @@ -0,0 +1,1290 @@ + + + + + + + Election Rules Complicate Kenya Race - The New York Times + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
        + + + + + + + + + +
        + + +
        +
        + + + + +
        + +
        + + + + +
        +
        +
        + Photo +
        + + + +
        +
        + What’s left of a poster for Raila Odinga adorns an outhouse in Kibera, a Nairobi slum. Mr. Odinga is running for president, but must win a seat in Kibera, where he faces united opposition. + + Credit + Evelyn Hockstein for The New York Times +
        +
        + +

        NAIROBI, Kenya — The fate of Kenya’s hotly contested presidential election could come down to a single slum.

        Kibera, known as the biggest slum in Africa, is a sprawling settlement on the outskirts of Nairobi, Kenya’s capital, with one million people squeezed into a warren of rusted roof shacks, linked by muddy footpaths and streams of greenish-grayish sewage trickling alongside.

        Made famous by the movie “The Constant Gardener,” which featured several scenes here, Kibera is concentrated poverty on a stunning scale. Half-naked children play in 10-foot-high piles of garbage. Drunken men stumble down the dirt boulevards, begging for work.

        But Kibera is not only a treasure trove of votes in this election, which will be held on Thursday and is predicted to be the closest contest in Kenya’s history and possibly the greatest test yet of this young, multiparty democracy.

        Continue reading the main story +
        +
        +
        +
        +
        + +
        +
        +
        +

        Kibera is also the heart of Raila Odinga’s parliamentary district, and Mr. Odinga, a rich businessman who has campaigned as a champion of the poor, is the leading contender for president. Most polls show him several percentage points ahead of Kenya’s president, Mwai Kibaki, who has improved the country’s economy but has alienated many voters.

        +

        Many Kenyans say Mr. Kibaki has shared the fruits of Kenya’s growing prosperity primarily with members of his own tribe, the Kikuyu.

        And here is where things get interesting.

        The fine print of Kenyan election law says that to become president, a candidate must win not only the most votes nationwide, but also a seat in Parliament and at least 25 percent of the votes in five of the country’s eight provinces. This may pose problems for both candidates, which could result in an inconclusive and turbulent post-election period.

        In Kibera, Mr. Odinga faces Stanley Livondo, a spirited challenger, who according to many residents, has been sprinkling around 500-shilling notes (the equivalent of about $8) and winning over converts. Mr. Livondo, also a businessman, was a political nobody until the president’s party got behind him a couple of months ago.

        + Photo +
        + + + +
        +
        + A supporter of presidential candidate Raila Odinga runs as a poster of President Mwai Kibaki burns in Nairobi. Mr. Odinga is running for president, but must win a seat in Kibera, where he faces united opposition. + + Credit + Thomas Mukoya/Reuters +
        +
        +

        Mr. Livondo said in a campaign advertisement that Kibera’s residents had been “reduced to tourist attractions” and that if he wins a Parliament seat, he will bring the area 50 new toilets and at least one new fire engine.

        +

        Many residents like the sound of that.

        “Just look at this place,” said Simon Mugambe, a shopkeeper, jerking his head toward a river of sewage running by his feet. “Somebody needs to do something.”

        +
        +

        Mr. Odinga has represented Kibera and the surrounding neighborhoods for the last 15 years. He won about 80 percent of the vote in the last parliamentary race in 2002, his campaign said. But this time he must reckon with a united front to defeat him.

        +

        Several other parliamentary candidates recently pulled out of the race, throwing their support behind Mr. Livondo. Mr. Odinga’s campaign is now worried that any election irregularities, like buying votes, could cost Mr. Odinga his seat — and therefore the presidency.

        Mr. Kibaki, though, has to contend with the other electoral wrinkle — the five out of eight rule. Recent polls show the president’s support heavily concentrated in the few provinces home to many Kikuyus. In the other provinces, his support is very thin.

        The result, after a campaign season in which more than 20 people have been killed in election-related violence, could be an unclear outcome with neither major candidate declared the winner. That would be bad, just about everyone agrees.

        +

        “The law is completely vague,” said Maina Kiai, chairman of the Kenya National Commission on Human Rights. “These things were not envisioned in the Constitution and there will be a lot of confusion. If this happens, the leadership will really have to step up. If not, we could have trouble.”

        One Western diplomat said, “We just hope it’s not close.”

        Kenya’s election commission said it was not sure what would happen if neither major candidate met the criteria.

        +

        “It’s a tricky one,” said Mani Lemayian, a spokesman for the election commission.

        There is a possibility, Mr. Lemayian said, that there could a runoff between the second and third place finishers, if the first-place finisher did not win a Parliament seat. That would open the door for Kalonzo Musyoka, a former foreign minister, who is also running for president and has been ranked a distant third.

        + +
        + Continue reading the main story +
        +
        +
        +
        + + + + + + +
        + + + + + +
        +
        +
        +
        +

        Go to Home Page »

        +

        + Site Index + + The New York Times + +

        + +
        + + + +
        + + +
        +
        + + + + + + + + + + + + + + + + diff --git a/test/testdata/8f4699babcfc3823ff41490452a0c47d1c4183d1.json b/test/testdata/8f4699babcfc3823ff41490452a0c47d1c4183d1.json new file mode 100644 index 00000000..852b6126 --- /dev/null +++ b/test/testdata/8f4699babcfc3823ff41490452a0c47d1c4183d1.json @@ -0,0 +1,27 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes, bytes", + "Cache-Control": "no-cache", + "Connection": "close", + "Content-Encoding": "gzip", + "Content-Type": "text/html; charset=utf-8", + "Cteonnt-Length": "83463", + "Date": "Tue, 23 May 2017 17:53:35 GMT", + "Server": "Apache", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding, Fastly-SSL", + "X-API-Version": "F-5-5", + "X-Age": "0", + "X-Cache": "MISS", + "X-Cache-Hits": "0", + "X-ESI": "1", + "X-Frame-Options": "DENY", + "X-Origin-Time": "2017-05-23 13:53:35 EDT", + "X-PageType": "article", + "X-Served-By": "cache-iad2144-IAD", + "X-Timer": "S1495562015.877668,VS0,VE251" + }, + "status_code": 200, + "url": "http://www.nytimes.com/2007/12/25/world/africa/25kenya.html" +} \ No newline at end of file diff --git a/test/testdata/90b7c41bcdc6e84c7c8473b86233c25ab9d42dee.html b/test/testdata/90b7c41bcdc6e84c7c8473b86233c25ab9d42dee.html new file mode 100644 index 00000000..18769136 --- /dev/null +++ b/test/testdata/90b7c41bcdc6e84c7c8473b86233c25ab9d42dee.html @@ -0,0 +1,7671 @@ + + +Abu Hamza found guilty in US court of helping Al-Qaeda terrorists | Daily Mail Online + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
        + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
        + + +
         
        + + + + + + +
        + + + +
        +
        + +
        + + + + + +
        +
        +
        +

        Abu Hamza to die behind bars: New York court finds hate preacher GUILTY of setting up terror training camps in US

        +
        • Abu Hamza found guilty of terrorism offences in a Manhattan court
        • Hate preacher was extradited from the UK in 2012 to face U.S. justice
        • Guilty of aiding Al-Qaeda terrorists and may be jailed for life
        + + + +

        Hate preacher Abu Hamza is expected to die behind bars after he was last night found guilty in the US of a string of terror offences.

        The hook-handed Islamist cleric – who for years spouted evil on Britain’s streets – was convicted by a New York court of 11 charges after a five-week trial.

        The firebrand, 56, had denied helping to organise a hostage-taking in Yemen in 1998 when three Britons were killed.

        Scroll down for video

        + +
        + Guilty verdict: Hate preacher Abu Hamza, 56, who was extradited from the UK in 2012, has been found guilty of terrorism offences in New York on Monday night +
        + + +

        Guilty verdict: Hate preacher Abu Hamza, 56, who was extradited from the UK in 2012, has been found guilty of terrorism offences in New York on Monday night

        +

        He also denied supporting Al Qaeda in +Afghanistan by sending volunteers and money and trying to set up a +terrorist training camp Oregon in 1999. But a jury of eight men and four + women took 11 hours to reach their guilty verdict.

        The + trial took place just a few streets from the site of the attacks on the + World Trade Center on September 11, 2001, which Hamza gloated was a +‘towering day in history’.

        + + +

        Hamza, also known as Mustafa Kamel +Mustafa headed the Finsbury Park Mosque in the 1990s, reportedly +attended by 9/11 conspirator Zacarias Moussaoui and shoe bomber Richard +Reid.

        The + conviction marks the end of a relentless ten-year mission by the US +authorities to extradite Hamza, who only has one eye and stumps instead +of hands.

        He cost British +taxpayers millions of pounds resisting American justice and went all the + way to the European Court of Human Rights but lost his case and was +extradited to New York in 2012.

         
        +
        + +
        + +
        +
        +
        + +
        + In court: Hamza, pictured in a court sketch from last month, looked straight ahead as the Manhattan jury found him guilty of providing material support to al-Qaeda-linked organizations in Yemen and Afghanistan sending men to establish an al-Qaeda training camp in Oregon, U.S. +
        + + +

        In court: Hamza, pictured in a court sketch from last month, looked straight ahead as the Manhattan jury found him guilty of providing material support to al-Qaeda-linked organizations in Yemen and Afghanistan sending men to establish an al-Qaeda training camp in Oregon, U.S.

        + +
        + One-eyed preacher: Muslim cleric Abu Hamza al-Masri addresses followers during prayer near the Finsbury Park mosque in 2004 +
        + + +

        One-eyed preacher: Muslim cleric Abu Hamza al-Masri addresses followers during prayer near the Finsbury Park mosque in 2004

        + +
        + Finally: The hook-handed father-of-eight, formerly the imam of Finsbury Park Mosque in London, fought extradition to the U.S. for eight years +
        + + +

        Finally: The hook-handed father-of-eight, formerly the imam of Finsbury Park Mosque in London, fought extradition to the U.S. for eight years

        +

        The + jury at Manhattan’s Federal Court agreed with the prosecution claims +that Hamza, 56, was a ‘trainer of terrorists’ who recruited and +indoctrinated young men at the Finsbury Park Mosque as part of his +global empire.

        In his +opening statement Edward Kim, Assistant District Attorney for the +Southern District of New York, also said that Hamza used the cover of +religion to ‘hide in plain sight’ under the noses of the British +authorities for years.

        Hamza + was convicted of 11 counts of criminal conduct related to the taking of + 16 hostages in Yemen in 1998 that left three Britons and an Australian +dead.

        He was also found +guilty of advocating violent jihad in Afghanistan in 2001 and conspiring + to establish a jihad training camp in Bly, Oregon, between June 2000 +and December 2001.

        During +the trial Hamza’s defence team tried to claim that the prosecution case, + which included videos of his fiery sermons at the Finsbury Park Mosque, + was just about ‘words, not deeds’.

        They portrayed him as an ‘independent thinker’ who was ‘on his own island’.

        They + admitted the jury might not like some of things that Hamza said but +defence lawyer Joshua Dratel told them: ‘These are views, not acts. This + is expression, not crimes.’

        In + his testimony Hamza did little to enamour himself with the jury when he + brazenly said that even today he loves Osama bin Laden and that he +thought 9/11 was a good thing.

        +

        +
        + +
        + Hiding in plain sight: Abu Hamza, sketched giving testimony earlier this month, was accused by the judge of hiding his terror message behind his religion +
        + + +

        Hiding in plain sight: Abu Hamza, sketched giving testimony earlier this month, was accused by the judge of hiding his terror message behind his religion

        + +
        + Abu Hamza al-Masri facing U.S. terrorism charges, gives testimony in Manhattan federal court in New York in this artist's sketch +
        + + +

        Abu Hamza al-Masri facing U.S. terrorism charges, gives testimony in Manhattan federal court in New York in this artist's sketch

         
        +
        + +
        + +
        +
        +

        Ironically it was only due to his personal intervention that he spoke at all about such matters.

        Hamza + wrote a series of letters to District Judge Katherine Forrest saying +his evidence would be 'important for historians' and overruling his +lawyers’ objections to him talking about anything inflammatory not +strictly to do with the case.

        A + major defeat for Hamza was the decision by Judge Forrest to ban him +from discussing his supposed links with MI5 or Scotland Yard.

        Mr + Dratel had had told the court that he had 50 pages of notes from +meetings between Hamza and British intelligence agencies between 1997 +and 2000 which had been handed to him by the UK.

        Mr Dratel said this showed a ‘constant dialogue’ between them Hamza - but the jury never got to hear it.

        As + a result Hamza was left claiming that he was just the ‘mouthpiece’ for +the Yemeni kidnappers and nothing else, a role he likened to that of +Gerry Adams with the IRA.

        + +
        + Muslim cleric Abu Hamza al-Masri in the street outside the closed Finsbury Park Mosque in London +
        + + +

        Muslim cleric Abu Hamza al-Masri in the street outside the closed Finsbury Park Mosque in London

        + +
        + Hamza_Dodging_Justice +
        + + + +

        +

        He + claimed that he tried to ‘de-escalate’ the situation and told the jury +he had nothing to do with the camp in Oregon and did not send anyone to +fight with Al Qaeda in Afghanistan.

        Hamza + - who had already claimed decades of disability benefit - cost British +taxpayers millions of pounds fighting his extradition to the US but in +2012 the European Court of Human Rights ruled it could go ahead.

        During + the trial there were moments of levity where Hamza displayed an offbeat + wit that was accentuated by his strong Egyptian accent.

        At + one point Hamza said that changing your name in Britain was so easy +that ‘if you want to be John Travolta, you become John Travolta’.

        District Judge Katherine Forrest asked: ‘And did you become John Travolta’ Hamza replied: ‘No madam’.

        Among + those the jury heard from were Saajid Badat, the former Al Qaeda +operative turned British ‘Supergrass’ who told the court that he saw +Feroz Abbasi, one of Hamza’s followers, at a terrorist training camp in +Afghanistan.

        There was also +moving testimony from Mary Quin, one of the Yemeni hostages who flew to +Britain and confronted Hamza at the Finsbury Park mosque in 2000 - her +interview proved to be a key piece of evidence.

        ABU HAMZA - THE EGYPTIAN ENGINEER WHO BECAME A PREACHER OF HATE

        Abu Hamza al-Masri was born in Alexandria, Egypt in 1958 as Mustafa Kamel Mustafa, the son of a naval officer and a primary school headmistress.

        After initially studying civil engineering he entered the UK in 1979 on a student visa.

        He was granted UK citizenship when he met and married his first wife, a British Muslim convert, in 1980.
        Hamza has previously said she was the one who got him interested in Islam and he converted after taking time off from his job as a nightclub bouncer in London’s Soho.

        + +
        + Abu Hamza continued to preach near the Finsbury Park Mosque after the congregation was thrown out in 2003 +
        + + +

        Abu Hamza continued to preach near the Finsbury Park Mosque after the congregation was thrown out in 2003

        As he found his new religion and his +job incompatible, he instead resumed his civil engineering studies at +Brunel University and Brighton Polytechnic, gaining a degree.

        He + then divorced his first wife, the mother of his oldest son, Muhammed +Kamel, who at the age of 17 was convicted of being part of a bomb plot +in Yemen and imprisoned for three years in 1999.

        He met and married his second wife in 1984 in a Muslim ceremony in London and had a further seven childen.

        Heavily + influenced by the Iranian revolution, he took an interest in Islam and +politics, in particularly the occupation of Afghanistan by the Soviet +Union.

        After meeting the +founder of Afghan Mujahideen in 1987, he moved to Egypt and then to Afghanistan, and it +was in the following years that he lost his hands and one eye.

        Over the years, Hamza has given several different reasons +for the loss of his hands and eye. These include a road project in Pakistan, an +explosion during a de-mining project in Jalalabad, Afghanistan, fighting the jihad as a Pakistani Mujahideen, and +working with Pakistani military in Lahore when an explosives experiment +went wrong.

        After spending +time in Afghanistan and Bosnia in the early 90s, he returned to Britain +and adopted a new name - Sheikh Abu Hamza al-Masri.

        It was in London that Hamza began his rise to public notoriety as the Finsbury Park mosque imam, where he arrived in 1997.

        One + year later, in 1998, he helped organise hostage-taking of 16 mostly +British tourists in Yemen. Three Britons and an Australian killed in +rescue mission.

        In 2000, he set up a terrorist training camp in Bly, +Oregon, sending volunteers and money to Afghanistan to support al Qaeda +and the Taliban.

        He firmly +placed himself on the national radar in 2001 after speaking out in +support of Osama bin Laden following the September 11 attacks.

        His + inflammatory speeches led to the Charity Commission suspending him from + his position at Finsbury Park Mosque the following year.

        In 2003, legal moves begin to get Hamza deported to Yemen, a move which he appealed.

        In + 2004 Hamza was arrested on a US extradition warrant over charges of +conspiring to take hostages in Yemen, funding terrorism, and organising a + terrorist training camp in Oregon. Charged with 15 offences under the +Terrorism Act, temporarily staying US extradition.

        In + 2006, Hamza  was jailed for seven years at the Old Bailey after being +found guilty of 11 of 15 charges, but the courts still battle to have +him extradited. 

        He was finally extradited in October 2012, and appeared in a U.S. court, indicted under the name Mustafa Kamel Mustafa, where he pleaded not guilty to terrorism charges.

        Tonight, + Hamza was convicted of all 11 charges on terrorism offences at Manhattan’s Federal Court. Sentencing on September +9th.

        +

        +
        + + + + + +
        +
        + + + + + + +
        +
        + + +
        + + + + + + + + + + +
        + +
        + + + + + + + + + + + + + + + +
        + +
        + + + + + + + + +
        + +
        + + +

        The comments below have been moderated in advance.

        + + +
        + +
        + +
        + +

        + The views expressed in the contents above are those of our users and do not necessarily reflect the views of MailOnline. +

        + +
        + + + + + + +

        We are no longer accepting comments on this article.

        + + +
        +
        +
        + + + +
        +
        +

        More top stories

        + +
        + + + +
        + +
        +
        + +
        +
        +
        + Bing +
        + + + + + + + + +
        + +
        + + + + +
        +
        + +
        + +
        + +
        + + +
        + + + +
        +
        + +   +   +

        Femail Today

        + + +
        + +
        + +
        +
        + + MailOnline iPhone app + +
        + +
        + +
        + +
        +
        + +   +   +

        DON'T MISS

        + + +
        + +
        + +
        +
        + +
        + +
        +
        + +
        + +
        + +
        +
        +
        + + match.png + +
        +

        From the Makers of Candy Crush

        + Farm Heroes Saga, the #4 Game on iTunes. Play it now! + +
        +
        + +
        +
        + +
        + +
        + +
        + +
        + +
        + + +
        +
        + + + + + + + +
        + +
        +
        + +
        +
        + + + + + + + +
        + +
        +
        +
        + + + + + + + + +
        + + + + + + + + + + +
        +
        +
        +
        +
        +
        + + +
        +
        +
        +
        + + +
        +
        +
        +
        + + + + + + + + + + + + +
        + + + + + + + + + + + + + +
        + + + + + + + + +
         
        + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/testdata/90b7c41bcdc6e84c7c8473b86233c25ab9d42dee.json b/test/testdata/90b7c41bcdc6e84c7c8473b86233c25ab9d42dee.json new file mode 100644 index 00000000..7645e159 --- /dev/null +++ b/test/testdata/90b7c41bcdc6e84c7c8473b86233c25ab9d42dee.json @@ -0,0 +1,21 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "max-age=0, no-cache", + "Connection": "close", + "Content-Encoding": "gzip", + "Content-Type": "text/html;charset=UTF-8", + "Date": "Tue, 23 May 2017 17:53:10 GMT", + "Expires": "Tue, 23 May 2017 17:53:10 GMT", + "Pragma": "no-cache", + "Vary": "User-Agent, Accept-Encoding", + "X-MOL-GEORESP": "us", + "X-rs-ops": "10.250.203.249:6081", + "x-rs-ben": "cljfe-a7:8181", + "x-rs-ctime": "1800", + "x-rs-time": "Tue, 23 May 2017 17-53-10 GMT", + "x-storage": "dmoldarticles" + }, + "status_code": 200, + "url": "http://www.dailymail.co.uk/news/article-2633025/London-cleric-convicted-NYC-terrorism-trial.html" +} \ No newline at end of file diff --git a/test/testdata/919578170c6abe356deb6aea2e8b421e3abf7d29.html b/test/testdata/919578170c6abe356deb6aea2e8b421e3abf7d29.html new file mode 100644 index 00000000..039f104d --- /dev/null +++ b/test/testdata/919578170c6abe356deb6aea2e8b421e3abf7d29.html @@ -0,0 +1 @@ +[{"itemType":"book","date":"1379 [2000 or 2001]","publisher":"Muʼassasah-ʼi Intishārāt-i Nigāh","title":"Rāz-i gul-i surkh","oclc":"53446327","url":"https://www.worldcat.org/oclc/53446327","ISBN":["964-6736-34-3","978-964-6736-34-4"],"edition":"Chāp-i 2","place":"Tihrān","numPages":"239 pages","author":[["Suhrāb.","Sipihrī"]],"accessDate":"2022-01-08","source":["WorldCat"]}] \ No newline at end of file diff --git a/test/testdata/919578170c6abe356deb6aea2e8b421e3abf7d29.json b/test/testdata/919578170c6abe356deb6aea2e8b421e3abf7d29.json new file mode 100644 index 00000000..d39a8b03 --- /dev/null +++ b/test/testdata/919578170c6abe356deb6aea2e8b421e3abf7d29.json @@ -0,0 +1,37 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "0", + "Connection": "keep-alive", + "NEL": "{ \"report_to\": \"wm_nel\", \"max_age\": 86400, \"failure_fraction\": 0.05, \"success_fraction\": 0.0}", + "Permissions-Policy": "interest-cohort=()", + "Report-To": "{ \"group\": \"wm_nel\", \"max_age\": 86400, \"endpoints\": [{ \"url\": \"https://intake-logging.wikimedia.org/v1/events?stream=w3c.reportingapi.network_error&schema_uri=/w3c/reportingapi/network_error/1.0.0\" }] }", + "Server-Timing": "cache;desc=\"pass\", host;desc=\"cp3052\"", + "Set-Cookie": "WMF-Last-Access=08-Jan-2022;Path=/;HttpOnly;secure;Expires=Wed, 09 Feb 2022 12:00:00 GMT, WMF-Last-Access-Global=08-Jan-2022;Path=/;Domain=.wikipedia.org;HttpOnly;secure;Expires=Wed, 09 Feb 2022 12:00:00 GMT, GeoIP=US:::37.75:-97.82:v4; Path=/; secure; Domain=.wikipedia.org", + "Strict-Transport-Security": "max-age=106384710; includeSubDomains; preload", + "X-Cache": "cp3062 miss, cp3052 pass", + "X-Cache-Status": "pass", + "X-Client-IP": "185.15.56.50", + "access-control-allow-headers": "accept, content-type, content-length, cache-control, accept-language, api-user-agent, if-match, if-modified-since, if-none-match, dnt, accept-encoding", + "access-control-allow-methods": "GET,HEAD", + "access-control-allow-origin": "*", + "access-control-expose-headers": "etag", + "cache-control": "private, max-age=0, s-maxage=0, must-revalidate", + "content-length": "395", + "content-location": "https://en.wikipedia.org/api/rest_v1/data/citation/mediawiki/964-6736-34-3", + "content-security-policy": "default-src 'none'; frame-ancestors 'none'", + "content-type": "application/json; charset=utf-8", + "date": "Sat, 08 Jan 2022 15:50:55 GMT", + "referrer-policy": "origin-when-cross-origin", + "server": "restbase1017", + "vary": "Accept-Encoding", + "x-content-security-policy": "default-src 'none'; frame-ancestors 'none'", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-webkit-csp": "default-src 'none'; frame-ancestors 'none'", + "x-xss-protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://en.wikipedia.org/api/rest_v1/data/citation/mediawiki/964-6736-34-3" +} \ No newline at end of file diff --git a/test/testdata/92debb1fbd8d6d7e1cbf7b68e80446d191a5a9cf.html b/test/testdata/92debb1fbd8d6d7e1cbf7b68e80446d191a5a9cf.html new file mode 100644 index 00000000..09ff71af --- /dev/null +++ b/test/testdata/92debb1fbd8d6d7e1cbf7b68e80446d191a5a9cf.html @@ -0,0 +1,1293 @@ + + + روانچی: در ارتباط با مواضع نامناسب اخیر مقامات انگلیسی در مورد ایران گفت‌وگو خواهیم کرد - ایسنا + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
        + +
        + + +
        +
        +
        +
        +
        +
        +
        +
        + +
          +
        • چهارشنبه / ۲۹ دی ۱۳۹۵ / ۱۴:۵۹
        • +
        • دسته‌بندی: + سیاست خارجی + +
        • +
        • کد خبر: 95102918901
        • +
        • خبرنگار : 71038
        • + +
        +
        +
        + +
        +
        +
        + +

        با اعلام خبر سفر معاون وزیر خارجه انگلیس به تهران

        +

        روانچی: در ارتباط با مواضع نامناسب اخیر مقامات انگلیسی در مورد ایران گفت‌وگو خواهیم کرد

        +
        +
        +
        +
        + مصاحبه اختصاصی با تخت روانچی +
        +

        معاون اروپا و آمریکای وزیر امور خارجه با اشاره به دیدار معاون وزیر امور خارجه انگلیس با وی در روز چهارشنبه در تهران، اظهار کرد: در جریان این دیدار در ارتباط با مسائل دوجانبه، منطقه‌ای، بین‌المللی و برجام گفت‌وگو خواهیم کرد.

        + +

        مجید تخت روانچی در گفت‌وگو با خبرنگار ایسنا،  با بیان این که این دیدار عصر امروز برگزار می شود ، اعلام کرد: در این ملاقات همچنین نسبت به مواضع نامناسبی که اخیراً توسط مقامات انگلیسی در مورد ایران اعلام شده گفت‌وگو و صحبت خواهیم کرد.

        + +

         ترزا می، نخست وزیر انگلیس، چند هفته پیش  ضمن حضور در نشست سران کشورهای عضو شورای همکاری خلیج فارس از لزوم همکاری با کشورهای این حوزه در برابر فعالیت‌های منطقه‌ای ایران سخن گفته و ادعاهایی را علیه ایران مطرح کرد.

        + +

        در این نشست، می ضمن تشریح سیاست‌های دولت خود در زمینه روابط با کشورهای عضو شورای همکاری خلیج فارس، به نقش ایران در منطقه نیز پرداخت و گفت که انگلیس آماده است با کشورهای عضو شورا برای مقابله با آنچه وی «اقدامات تهاجمی ایران در منطقه» خواند، همکاری کند. 

        + +

        نخست وزیر انگلیس اظهار کرد : ما باید همچنان به مقابله با دولت‌هایی که نفوذ آن ها بی‌ثباتی منطقه را مشتعل می‌کند، ادامه دهیم و افزود: «بنابراین، می‌خواهم به شما اطمینان دهم که من به وضوح تهدیدی را که ایران متوجه منطقه خلیج فارس و در بُعدی وسیعتر، متوجه خاورمیانه می‌کند، مشاهده می‌کنم».

        + +

         این اظهارات در همان زمان با واکنش شدید جمهوری اسلامی ایران و دستگاه دیپلماسی کشور مواجه شد و بهرام قاسمی سخنگوی وزارت امور خارجه در این ارتباط گفت : کشورهایی که مداخله‌جویی‌های غیر مسئولانه آنها در سایر کشورها موجب گسترش ناامنی، جنگ، خشونت و تروریسم شده است در جایگاهی نیستند که دیگران را به مداخله در امور منطقه متهم نمایند.

        + +

        وی با اشاره به سیاست‌های تفرقه افکنانه بریتانیا افزود: این کشور در تلاش برای بازگشت به این منطقه، مجددا به سیاست های تفرقه افکنانه روی آورده است که از دیدگاه جمهوری اسلامی ایران کاری عبث و غیرسازنده است.

        + +

        قاسمی اضافه کرد: جمهوری اسلامی ایران، ریشه بخشی از این اظهارات را ناشی از تحولات در روابط این کشور با اتحادیه اروپایی می داند که مشکلات، کمبودها و پیچیدگی هایی را در منافع و جایگاه بین المللی انگلیس ایجاد کرده و باعث شده است نخست وزیر این کشور متناسب با فضای اجلاس شورای همکاری خلیج فارس و برای خوشایند تعدادی از سران کشورهای عضو این شورا، حرف هایی نسنجیده علیه دولت و ملت ایران بر زبان بیاورد.

        + +

        سخنگوی وزارت خارجه در پایان گفت: به نظر می رسد هدف از این گونه اظهارات تلاش برای عقد قراردادهای جدید هنگفت تسلیحاتی بین انگلیس و برخی کشورهای عرب حاشیه خلیج فارس و در نهایت، تشدید بحران های ناشی از جنایات جنگی آنها علیه ملت های مظلوم یمن، سوریه، بحرین، عراق و دیگر کشورهای اسلامی منطقه باشد.

        + +

        مشروح  گفت‌وگوی مجید تخت روانچی با خبرنگاران هسته‌ای و سیاست خارجی ایسنا طی روزهای آتی ارسال می‌شود .

        + +

        انتهای پیام

        + +

        + +
        + + + +
        + +
        + +
        +
        + +
        +
        +
        +
        + +
        • در زمینه انتشار نظرات مخاطبان رعایت چند مورد ضروری است:
        • -لطفا نظرات خود را با حروف فارسی تایپ کنید.
        • -«ایسنا» مجاز به ویرایش ادبی نظرات مخاطبان است.
        • - ایسنا از انتشار نظراتی که حاوی مطالب کذب، توهین یا بی‌احترامی به اشخاص، قومیت‌ها، عقاید دیگران، موارد مغایر با قوانین کشور و آموزه‌های دین مبین اسلام باشد معذور است.
        • - نظرات پس از تأیید مدیر بخش مربوطه منتشر می‌شود.
        +
        +
        +
        +
        +

        نظرات

        +
        +
        + +
        +
        +
        شما در حال پاسخ به نظر «» هستید. + +
        +
        +
        + + + + +
        +
        + +
        +
        +
        + + + +
        +
        +
        +
        +
        + +
        +
        +
        +
        +
        +
        +
        + + +
        +
        +
        +
        +
        +
        +
        + + + + + + + + + \ No newline at end of file diff --git a/test/testdata/92debb1fbd8d6d7e1cbf7b68e80446d191a5a9cf.json b/test/testdata/92debb1fbd8d6d7e1cbf7b68e80446d191a5a9cf.json new file mode 100644 index 00000000..9ad9aa27 --- /dev/null +++ b/test/testdata/92debb1fbd8d6d7e1cbf7b68e80446d191a5a9cf.json @@ -0,0 +1,21 @@ +{ + "encoding": "UTF-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "646", + "Connection": "close", + "Content-Encoding": "gzip", + "Content-Length": "17260", + "Content-Type": "text/html;charset=UTF-8", + "Date": "Mon, 29 May 2017 09:42:54 GMT", + "Server": "Apache-Coyote/1.1", + "Vary": "Accept-Encoding", + "Via": "1.1 varnish-v4", + "X-Cache": "HIT from google.com", + "X-Cache-Lookup": "HIT from google.com:85", + "X-Varnish": "448301522 447491081", + "grace": "none" + }, + "status_code": 200, + "url": "http://www.isna.ir/news/95102918901/%D8%B1%D9%88%D8%A7%D9%86%DA%86%DB%8C-%D8%AF%D8%B1-%D8%A7%D8%B1%D8%AA%D8%A8%D8%A7%D8%B7-%D8%A8%D8%A7-%D9%85%D9%88%D8%A7%D8%B6%D8%B9-%D9%86%D8%A7%D9%85%D9%86%D8%A7%D8%B3%D8%A8-%D8%A7%D8%AE%DB%8C%D8%B1-%D9%85%D9%82%D8%A7%D9%85%D8%A7%D8%AA-%D8%A7%D9%86%DA%AF%D9%84%DB%8C%D8%B3%DB%8C-%D8%AF%D8%B1-%D9%85%D9%88%D8%B1%D8%AF" +} \ No newline at end of file diff --git a/test/testdata/92e0aa4660bf2c2fe8d779967c76c0493807dc24.html b/test/testdata/92e0aa4660bf2c2fe8d779967c76c0493807dc24.html new file mode 100644 index 00000000..98bb32ad --- /dev/null +++ b/test/testdata/92e0aa4660bf2c2fe8d779967c76c0493807dc24.html @@ -0,0 +1 @@ +{"indexed":{"date-parts":[[2022,4,3]],"date-time":"2022-04-03T00:11:56Z","timestamp":1648944716989},"reference-count":0,"publisher":"University of Chicago Press","issue":"3","content-domain":{"domain":[],"crossmark-restriction":false},"published-print":{"date-parts":[[2014,3]]},"DOI":"10.1086\/677379","type":"journal-article","created":{"date-parts":[[2014,6,19]],"date-time":"2014-06-19T17:41:03Z","timestamp":1403199663000},"page":"272-281","source":"Crossref","is-referenced-by-count":0,"title":"Books of Critical Interest","prefix":"10.1086","volume":"40","member":"200","container-title":"Critical Inquiry","original-title":[],"language":"en","link":[{"URL":"http:\/\/www.journals.uchicago.edu\/doi\/pdf\/10.1086\/677379","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2018,4,7]],"date-time":"2018-04-07T08:30:03Z","timestamp":1523089803000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.journals.uchicago.edu\/doi\/10.1086\/677379"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2014,3]]},"references-count":0,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2014,3]]}},"alternative-id":["10.1086\/677379"],"URL":"http:\/\/dx.doi.org\/10.1086\/677379","relation":{},"ISSN":["0093-1896","1539-7858"],"subject":["General Arts and Humanities","Cultural Studies"],"container-title-short":"Critical Inquiry","published":{"date-parts":[[2014,3]]}} \ No newline at end of file diff --git a/test/testdata/92e0aa4660bf2c2fe8d779967c76c0493807dc24.json b/test/testdata/92e0aa4660bf2c2fe8d779967c76c0493807dc24.json new file mode 100644 index 00000000..618fdac2 --- /dev/null +++ b/test/testdata/92e0aa4660bf2c2fe8d779967c76c0493807dc24.json @@ -0,0 +1,24 @@ +{ + "encoding": null, + "headers": { + "access-control-allow-headers": "X-Requested-With, Accept, Accept-Encoding, Accept-Charset, Accept-Language, Accept-Ranges, Cache-Control", + "access-control-allow-origin": "*", + "access-control-expose-headers": "Link", + "connection": "close", + "content-encoding": "gzip", + "content-length": "700", + "content-type": "application/vnd.citationstyles.csl+json", + "date": "Thu, 09 Jun 2022 10:37:18 GMT", + "link": "; rel=\"canonical\", ; version=\"vor\"; rel=\"item\"", + "permissions-policy": "interest-cohort=()", + "server": "Jetty(9.4.40.v20210413)", + "vary": "Accept, Accept-Encoding", + "x-api-pool": "public", + "x-rate-limit-interval": "1s", + "x-rate-limit-limit": "50", + "x-ratelimit-interval": "1s", + "x-ratelimit-limit": "50" + }, + "status_code": 200, + "url": "https://api.crossref.org/v1/works/10.1086%2F677379/transform" +} \ No newline at end of file diff --git a/test/testdata/92ff5588773ca662d88b74f4ab7942c75be0984e.html b/test/testdata/92ff5588773ca662d88b74f4ab7942c75be0984e.html new file mode 100644 index 00000000..cc38abff --- /dev/null +++ b/test/testdata/92ff5588773ca662d88b74f4ab7942c75be0984e.html @@ -0,0 +1 @@ +{"indexed":{"date-parts":[[2022,4,4]],"date-time":"2022-04-04T12:45:01Z","timestamp":1649076301885},"publisher-location":"New York, New York, USA","reference-count":0,"publisher":"ACM Press","isbn-type":[{"value":"1595932550","type":"print"}],"content-domain":{"domain":[],"crossmark-restriction":false},"published-print":{"date-parts":[[2006]]},"DOI":"10.1145\/1117278","type":"proceedings","created":{"date-parts":[[2006,5,8]],"date-time":"2006-05-08T21:40:43Z","timestamp":1147124443000},"source":"Crossref","is-referenced-by-count":0,"title":"Proceedings of the international workshop on System-level interconnect prediction - SLIP'06","prefix":"10.1145","member":"320","event":"the international workshop","container-title":[],"original-title":[],"deposited":{"date-parts":[[2013,12,16]],"date-time":"2013-12-16T23:08:32Z","timestamp":1387235312000},"score":1,"resource":{"primary":{"URL":"http:\/\/portal.acm.org\/citation.cfm?doid=1117278"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2006]]},"ISBN":["1595932550"],"references-count":0,"URL":"http:\/\/dx.doi.org\/10.1145\/1117278","relation":{},"published":{"date-parts":[[2006]]}} \ No newline at end of file diff --git a/test/testdata/92ff5588773ca662d88b74f4ab7942c75be0984e.json b/test/testdata/92ff5588773ca662d88b74f4ab7942c75be0984e.json new file mode 100644 index 00000000..478ff3b3 --- /dev/null +++ b/test/testdata/92ff5588773ca662d88b74f4ab7942c75be0984e.json @@ -0,0 +1,24 @@ +{ + "encoding": null, + "headers": { + "access-control-allow-headers": "X-Requested-With, Accept, Accept-Encoding, Accept-Charset, Accept-Language, Accept-Ranges, Cache-Control", + "access-control-allow-origin": "*", + "access-control-expose-headers": "Link", + "connection": "close", + "content-encoding": "gzip", + "content-length": "597", + "content-type": "application/vnd.citationstyles.csl+json", + "date": "Thu, 09 Jun 2022 10:35:29 GMT", + "link": "; rel=\"canonical\"", + "permissions-policy": "interest-cohort=()", + "server": "Jetty(9.4.40.v20210413)", + "vary": "Accept, Accept-Encoding", + "x-api-pool": "public", + "x-rate-limit-interval": "1s", + "x-rate-limit-limit": "50", + "x-ratelimit-interval": "1s", + "x-ratelimit-limit": "50" + }, + "status_code": 200, + "url": "https://api.crossref.org/v1/works/10.1145%2F1117278/transform" +} \ No newline at end of file diff --git a/test/testdata/957494cd1e7a50282ae9002a24f431242d330134.html b/test/testdata/957494cd1e7a50282ae9002a24f431242d330134.html new file mode 100644 index 00000000..37943e08 --- /dev/null +++ b/test/testdata/957494cd1e7a50282ae9002a24f431242d330134.html @@ -0,0 +1,540 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Magiran | بانک اطلاعات نشریات کشور + + + + + + + + + + + + + +
        +
        + +
        + +
        +
        + +
        +
        + +
        +
        +
        + تا کنون بیش از 25000 پژوهشگر به سامانه نویسندگان مگیران پیوسته‌اند! +
        +
        + از نویسندگان و پژوهشگرانی که مقالات آنها در مگیران نمایه شده دعوت می‌کنیم با ایمیل منتشر شده در مقالات‌شان در سایت ثبت نام و صفحه رزومه خود را تکمیل نمایند. +
        +
        +
        +
        +
        + + +
        +
        + +
        +
        +
        + +
        +
        +
        + اشتراک سالانه مگیران +
        +
        اشتراک سالانه شخصی
        + +
        + برای استفاده از خدمات مختلف سایت و مطالعه متن مطالب و مقالات علمی مورد نظرتان، ثبت نام کنید و حق اشتراک سالانه سایت را پرداخت نمایید! پرداخت آنلاین با کارت‌های شتاب و یا کارت‌های اعتباری بین‌المللی با PayPal امکان‌پذیر است. +
        +
        + +
        +
        +
        + +
        +
        + اشتراک نسخه چاپی مجلات +
        +
        اشتراک سالانه سازمانی
        +
        + به مدیران کتابخانه‌ها، دانشگاه‌ها و مراکز پژوهشی ادارات و سازمان‌ها پیشنهاد می‌کنیم با پرداخت حق اشتراک سالانه سازمانی، + امکان دسترسی آسان دانشجویان، پژوهشگران و کارشناسان خود را به منابع این سایت مرجع فراهم آورند! +
        + + +
        +
        +
        +
        +
        + اشتراک نسخه چاپی مجلات +
        +
        اشتراک نسخه چاپی
        +
        + با استفاده از خدمات مگیران نسخه چاپی مجلات مورد علاقه‌تان را مشترک شوید تا در زمان انتشار توسط دفتر مجله به نشانی شما ارسال شود. + ما هزینه‌ای بابت خدمات از شما دریافت نخواهیم کرد و برای مشترکان خود نیز تخفیف ویژه‌ای در نظر گرفته‌ایم! +
        + + +
        +
        +
        +
        +
        + +
        +
        +
        +
        +

        «بانک اطلاعات نشریات کشور»

        +

        + پایگاه «مگیران» در سال ۱۳۸۰ راه‌اندازی شد تا مرجعی جامع و فراگیر از مطالب و مقالات مجلات ایرانی برای مطالعه علاقمندان و پژوهشگران در رشته‌های مختلف علمی و تخصصی باشد. + خوشحالیم که طی سال‌های طولانی یار و همراه نشریات، مخاطبین آن‌ها و همه دانشجویان، اساتید و پژوهشگران این مرز و بوم بوده‌ایم. +
          + اکنون مطالب ۱۳۰.۰۰۰ جلد از ۳۵۰۰ مجله علمی، تخصصی و عمومی کشور در دسترس شماست تا بتوانید به سادگی آن‌ها را جستجو کرده و متن بیش از ۲.۰۰۰.۰۰۰ مطلب را دریافت و مطالعه کنید! +
          + خدمات گوناگون این ‍ پایگاه شما را در پیگیری مجلات مورد علاقه و استفاده از مطالب آن‌ها در مطالعات و پژوهش‌های‌تان یاری خواهد داد. +
        + از سال ۱۳۸۵ بخش روزنامه‌ها نیز به سایت افزوده شد تا کاربران سایت بتوانند اخبار، گزارش‌ها و تحلیل‌های روز و موارد مرتبط با + رشته و تخصص خود را نیز دنبال کنند. خوشحالیم که پس از پانزده سال تلاش مستمر آرشیو منحصربه فردی از مطالب این سال‌های روزنامه‌های مطرح کشور + گردآورده‌ایم. +
        + + بیشتر ... + +

        + +
        +
        + + + + + + + +
        + +
        + با جستجوی پیشرفته مطالب، نتایج جستجو را بر حسب نوع مجلات، دوره زمانی انتشار، زبان و رتبه علمی بهینه کنید!
        +

        +
        + + + + +
        + +
        +
        +
        +
        +
        + روزنامه‌های تحت پوشش +
        +
        + + + + + + + + +
        +
        + + +
        + +
        + +
        +
        + + + + + +
        +
        + + +
        + +
        + درخواست پشتیبانی - گزارش اشکال +
        +
        + + + + + + + + + + + + + + + diff --git a/test/testdata/957494cd1e7a50282ae9002a24f431242d330134.json b/test/testdata/957494cd1e7a50282ae9002a24f431242d330134.json new file mode 100644 index 00000000..73666b07 --- /dev/null +++ b/test/testdata/957494cd1e7a50282ae9002a24f431242d330134.json @@ -0,0 +1,29 @@ +{ + "encoding": "utf-8", + "headers": { + "AR-ATIME": "0.044", + "AR-CACHE": "BYPASS", + "AR-Request-ID": "a3c24606f8156df11e5a2f23b9aef671", + "AR-SID": "2020", + "Accept-Ranges": "bytes", + "Cache-Control": "max-age=0", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "10389", + "Content-Security-Policy": "upgrade-insecure-requests", + "Content-Type": "text/html; charset=utf-8", + "Date": "Sat, 04 Jun 2022 08:54:01 GMT", + "Expires": "Sat, 04 Jun 2022 08:54:01 GMT", + "Keep-Alive": "timeout=65", + "Pragma": "no-cache", + "Server": "ArvanCloud", + "Strict-Transport-Security": "max-age=15768000", + "Vary": "Accept-Encoding, Accept-Encoding", + "X-Content-Type-Options": "nosniff", + "X-Frame-Options": "SAMEORIGIN", + "X-Powered-By": "My Little Pony", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://www.magiran.com/" +} \ No newline at end of file diff --git a/test/testdata/95b6685ec06107c37235062f13240de1effd4f54.html b/test/testdata/95b6685ec06107c37235062f13240de1effd4f54.html new file mode 100644 index 00000000..25ad90ad --- /dev/null +++ b/test/testdata/95b6685ec06107c37235062f13240de1effd4f54.html @@ -0,0 +1,35 @@ +
        + \ No newline at end of file diff --git a/test/testdata/95b6685ec06107c37235062f13240de1effd4f54.json b/test/testdata/95b6685ec06107c37235062f13240de1effd4f54.json new file mode 100644 index 00000000..db578761 --- /dev/null +++ b/test/testdata/95b6685ec06107c37235062f13240de1effd4f54.json @@ -0,0 +1,21 @@ +{ + "encoding": "utf-8", + "headers": { + "CF-Cache-Status": "DYNAMIC", + "CF-RAY": "744536e05873924f-FRA", + "Cache-Control": "private, no-cache, no-store, max-age=0, must-revalidate", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Security-Policy": "frame-ancestors 'none'", + "Content-Type": "text/html; charset=utf-8", + "Date": "Fri, 02 Sep 2022 09:28:14 GMT", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=15724800; includeSubDomains", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding", + "X-Frame-Options": "SAMEORIGIN", + "X-Powered-By": "Next.js" + }, + "status_code": 200, + "url": "https://www.worldcat.org/title/875039842" +} \ No newline at end of file diff --git a/test/testdata/970c142eb267d5734344dbd052796ca8221a9d81.html b/test/testdata/970c142eb267d5734344dbd052796ca8221a9d81.html new file mode 100644 index 00000000..65b522fd --- /dev/null +++ b/test/testdata/970c142eb267d5734344dbd052796ca8221a9d81.html @@ -0,0 +1 @@ +{"indexed":{"date-parts":[[2022,4,2]],"date-time":"2022-04-02T19:29:01Z","timestamp":1648927741552},"reference-count":12,"publisher":"American Geophysical Union (AGU)","issue":"10","license":[{"start":{"date-parts":[[2015,9,1]],"date-time":"2015-09-01T00:00:00Z","timestamp":1441065600000},"content-version":"tdm","delay-in-days":4857,"URL":"http:\/\/doi.wiley.com\/10.1002\/tdm_license_1.1"}],"content-domain":{"domain":[],"crossmark-restriction":false},"published-print":{"date-parts":[[2002,5,15]]},"DOI":"10.1029\/2002gl014729","type":"journal-article","created":{"date-parts":[[2002,10,27]],"date-time":"2002-10-27T18:39:25Z","timestamp":1035743965000},"page":"15-1-15-3","source":"Crossref","is-referenced-by-count":9,"title":"The effect of the July 14, 2000 \u201cBastille Day\u201d solar flare event on >70 MeV galactic cosmic rays observed at V1 and V2 in the distant heliosphere","prefix":"10.1029","volume":"29","author":[{"given":"W. R.","family":"Webber","sequence":"first","affiliation":[{"name":"Department of Astronomy; New Mexico State University; Las Cruces USA"}]},{"given":"F. B.","family":"McDonald","sequence":"additional","affiliation":[{"name":"Institute of Physical Science and Technology; University of Maryland; College Park USA"}]},{"given":"J. A.","family":"Lockwood","sequence":"additional","affiliation":[{"name":"Space Science Center; University of New Hampshire; Durham USA"}]},{"given":"B.","family":"Heikkila","sequence":"additional","affiliation":[{"name":"Institute of Physical Science and Technology; University of Maryland; College Park USA"}]}],"member":"13","published-online":{"date-parts":[[2002,5,21]]},"reference":[{"key":"10.1029\/2002GL014729-BIB0001|grl15702-cit-0001","unstructured":"Burlaga , L. F. N. F. Ness F. B. McDonald \u201cVoyagers 1 and 2 observe a GMIR and associated cosmic ray decrease at 61 and 82 AU\u201d Proc. 27th ICRC, Hamburg 3641 2001"},{"key":"10.1029\/2002GL014729-BIB0002|grl15702-cit-0002","doi-asserted-by":"crossref","first-page":"199","DOI":"10.1126\/science.262.5131.199","article-title":"\u201cRadio emission from the heliopause triggered by an interplanetary shock\u201d","volume":"262","author":"Gurnett","year":"1993","journal-title":"Science"},{"key":"10.1029\/2002GL014729-BIB0003|grl15702-cit-0003","unstructured":"Krimigis , S. M. R. B. Decker D. C. Hamilton M. E. Hill G. Gloeckler \u201cSurvey of energetic particles observed at Voyagers 1 and 2 during 1999-2001\u201d Proc. 27th ICRC, Hamburg 3607 2001"},{"key":"10.1029\/2002GL014729-BIB0004|grl15702-cit-0004","doi-asserted-by":"crossref","first-page":"4709","DOI":"10.1029\/1998JA900089","article-title":"Global merged interaction regions the heliospheric termination shock and time dependent cosmic ray modulation","volume":"104","author":"le Roux","year":"1999","journal-title":"J. Geophys. Res."},{"key":"10.1029\/2002GL014729-BIB0005|grl15702-cit-0005","unstructured":"McDonald , F. B. \u201cThe July 14th 2000 Bastille day solar event as observed by Voyagers 1 and 2 in the distant heliosphere\u201d Proc. 27th ICRC, Hamburg 3637 2001"},{"key":"10.1029\/2002GL014729-BIB0006|grl15702-cit-0006","doi-asserted-by":"crossref","first-page":"L233","DOI":"10.1086\/319100","article-title":"\u201cHeavy ion abundances and spectra in the large solar energetic particle event on July 14, 2000\u201d","volume":"548","author":"Reames","year":"2001","journal-title":"Ap.J. (Letters)"},{"key":"10.1029\/2002GL014729-BIB0007|grl15702-cit-0007","doi-asserted-by":"crossref","first-page":"355","DOI":"10.1007\/BF00211546","article-title":"\u201cCosmic ray investigation for the Voyager missions; Energetic particle studies in the outer heliosphere and beyond\u201d","volume":"21","author":"Stone","year":"1977","journal-title":"Space Sci. Rev."},{"key":"10.1029\/2002GL014729-BIB0008|grl15702-cit-0008","unstructured":"Stone , E. C. A. C. Cummings \u201cEstimate of the location of the solar wind termination shock\u201d Proc. 27 th ICRC, Hamburg 4263 2001"},{"key":"10.1029\/2002GL014729-BIB0009|grl15702-cit-0009","article-title":"\u201cPredicted Voyager 2 observations of the Bastille Day 2000 CME\u201d","author":"Wang","year":"2002","journal-title":"J. Geophys. Res"},{"key":"10.1029\/2002GL014729-BIB0010|grl15702-cit-0010","doi-asserted-by":"crossref","first-page":"7821","DOI":"10.1029\/92JA02643","article-title":"\u201cGiant transient decreases of cosmic rays in the outer heliosphere in September, 1991\u201d","volume":"98","author":"Webber","year":"1993","journal-title":"J. Geophys. Res."},{"key":"10.1029\/2002GL014729-BIB0011|grl15702-cit-0011","doi-asserted-by":"crossref","first-page":"253","DOI":"10.1029\/2000JA000285","article-title":"\u201cUsing transient decreases of cosmic rays observed by Voyagers 1 and 2 to estimate the location of the heliospheric termination shock\u201d","volume":"106","author":"Webber","year":"2001","journal-title":"J. Geophys. Res."},{"key":"10.1029\/2002GL014729-BIB0012|grl15702-cit-0012","article-title":"\u201cPredicted timing for the turn on of radiation in the outer Heliosphere due to the Bastille day shock\u201d","author":"Zank","year":"2002","journal-title":"J. Geophys. Res."}],"container-title":"Geophysical Research Letters","original-title":[],"language":"en","link":[{"URL":"https:\/\/api.wiley.com\/onlinelibrary\/tdm\/v1\/articles\/10.1029%2F2002GL014729","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2021,7,20]],"date-time":"2021-07-20T04:16:51Z","timestamp":1626754611000},"score":1,"resource":{"primary":{"URL":"http:\/\/doi.wiley.com\/10.1029\/2002GL014729"}},"subtitle":["THE EFFECT OF THE JULY 14, 2000 \u201cBASTILLE DAY\u201d"],"short-title":[],"issued":{"date-parts":[[2002,5,15]]},"references-count":12,"journal-issue":{"issue":"10","published-print":{"date-parts":[[2002,5,15]]}},"URL":"http:\/\/dx.doi.org\/10.1029\/2002GL014729","relation":{},"ISSN":["0094-8276"],"subject":["General Earth and Planetary Sciences","Geophysics"],"container-title-short":"Geophys. Res. Lett.","published":{"date-parts":[[2002,5,15]]}} \ No newline at end of file diff --git a/test/testdata/970c142eb267d5734344dbd052796ca8221a9d81.json b/test/testdata/970c142eb267d5734344dbd052796ca8221a9d81.json new file mode 100644 index 00000000..ce05622f --- /dev/null +++ b/test/testdata/970c142eb267d5734344dbd052796ca8221a9d81.json @@ -0,0 +1,24 @@ +{ + "encoding": null, + "headers": { + "access-control-allow-headers": "X-Requested-With, Accept, Accept-Encoding, Accept-Charset, Accept-Language, Accept-Ranges, Cache-Control", + "access-control-allow-origin": "*", + "access-control-expose-headers": "Link", + "connection": "close", + "content-encoding": "gzip", + "content-length": "2187", + "content-type": "application/vnd.citationstyles.csl+json", + "date": "Sat, 11 Jun 2022 02:24:56 GMT", + "link": "; rel=\"canonical\", ; version=\"vor\"; rel=\"item\", ; version=\"tdm\"; rel=\"license\"", + "permissions-policy": "interest-cohort=()", + "server": "Jetty(9.4.40.v20210413)", + "vary": "Accept, Accept-Encoding", + "x-api-pool": "public", + "x-rate-limit-interval": "1s", + "x-rate-limit-limit": "50", + "x-ratelimit-interval": "1s", + "x-ratelimit-limit": "50" + }, + "status_code": 200, + "url": "https://api.crossref.org/v1/works/10.1029%2F2002GL014729/transform" +} \ No newline at end of file diff --git a/test/testdata/970c390b74690891434a59ec141cf87f86b33989.html b/test/testdata/970c390b74690891434a59ec141cf87f86b33989.html new file mode 100644 index 00000000..399fc550 --- /dev/null +++ b/test/testdata/970c390b74690891434a59ec141cf87f86b33989.html @@ -0,0 +1,264 @@ +UK allows working visas for Indian students | Pune News - Times of India
        UK allows working visas for Indian students
        This story is from November 25, 2001

        UK allows working visas for Indian students

        FacebookTwitterLinkedinEMail
        AA
        Text Size
        • Small
        • Medium
        • Large
        pune: the two-day british education fair being held in the city drew a fairly good crowd on saturday. while a similar event in ahmedabad attracted 1000 visitors in the very first three hours, in pune some 350 people had visited it till afternoon. representatives of 30 universities in the u.k. are pitching for students to study in the country. already this year, 7000 indian students enrolled to study in various british universities. changes in rules for students have undoubtedly enhanced britain's attractiveness as a study destination. the british government now allows indian students to convert their student visas into work visas and also allows them to work 20 hours a week during their offtime and on holidays. sunita kripalani, head of education counselling services, british high commission, says there has been a 40 per cent increase in the number of students going to the u.k. in 2001 over last year. "most of these students left the country after september 11," she said. the relaxed visa rules may have had much to do with the increasing number of indian students headed towards the u.k. but security fears have also played their part. says maxine davis, international director of the university of greenwich, "india is a big market for us and the latest happenings in the u.s. have provided us the opportunity to tap these resources. security is a matter of concern for every individual." he also said that a majority of universities welcome indian students because of their cultural background, commitment and competence in the english language. speaking in the same vein,tim hunt, director of international business development of watford-based west herts college, which specialises in advertising, media and hospitality, said the u.s. incident has a definite role to play in the influx of indian students to the u.k. most students seemed impressed with what was on offer. "i am quite impressed by their presentation. they have a lot to offer compared to the indian universities," said ashu mehrotra, who is pursuing his degree in environmental science. and nitin ghokale, who had made up his mind to go to the u.s. next year to pursue a specialised computer programing course said that soon after the u.s. incident, "i was in two minds of going ahead with my plans. but after visiting the fair, i'll do a rethink. these universities are offering the same courses and in a conducive atmosphere."
        FacebookTwitterLinkedinEMail
        Start a Conversation
        end of article
        \ No newline at end of file diff --git a/test/testdata/970c390b74690891434a59ec141cf87f86b33989.json b/test/testdata/970c390b74690891434a59ec141cf87f86b33989.json new file mode 100644 index 00000000..cd722104 --- /dev/null +++ b/test/testdata/970c390b74690891434a59ec141cf87f86b33989.json @@ -0,0 +1,23 @@ +{ + "encoding": "utf-8", + "headers": { + "Access-Control-Allow-Credentials": "false", + "Access-Control-Allow-Headers": "Origin,X-Requested-With,Content-Type,Accept", + "Access-Control-Allow-Methods": "GET,POST", + "Access-Control-Max-Age": "86400", + "Cache-Control": "max-age=0, no-cache, no-store", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "40852", + "Content-Type": "text/html; charset=utf-8", + "Date": "Wed, 21 Jul 2021 03:23:44 GMT", + "ETag": "W/\"3de5b-psU/VmrwnakCwgpTi98zhga2MRQ\"", + "Expires": "Wed, 21 Jul 2021 03:23:44 GMT", + "Pragma": "no-cache", + "Server": "nginx", + "Strict-Transport-Security": "max-age=86400", + "Vary": "Accept-Encoding" + }, + "status_code": 200, + "url": "https://timesofindia.indiatimes.com/city/pune/UK-allows-working-visas-for-Indian-students/articleshow/1163528927.cms" +} \ No newline at end of file diff --git a/test/testdata/98201bbc2eca52df38a818a6acb99d21d8b59149.html b/test/testdata/98201bbc2eca52df38a818a6acb99d21d8b59149.html new file mode 100644 index 00000000..338f72d8 --- /dev/null +++ b/test/testdata/98201bbc2eca52df38a818a6acb99d21d8b59149.html @@ -0,0 +1,635 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Electronic Frontier Foundation | Defending your rights in the digital world + + + + + + + + + + + + + Skip to main content + + + + + + +
        + +
        +

        +
        +
        + +
        + +
        + +
        +
        + The leading nonprofit defending digital privacy, free speech, and innovation. + +
        + + + +
        +

        The Latest

        +
        + + + +
        +
        + +
        +
        + +
        +
        + +
        +
        + +
        +
        + +
        +
        + +
        +
        +
        +
        +

        No Hunting Undocumented Immigrants with Stingrays

        +
        + + +
        +
        In the latest sign of mission creep in domestic deployment of battlefield-strength surveillance technology, U.S. Immigration and Customs Enforcement (ICE) earlier this year used a cell site simulator (CSS) to locate and arrest an undocumented immigrant, according to a report yesterday by The Detroit News . CSSs, often...
        + +
        +
        +
        + +
        +
        + +
        +
        + +
        +
        + + + + + + + + +
        +
        +
        +
        +
        + +
        +
        + + + + + + JavaScript license information
        + + + diff --git a/test/testdata/98201bbc2eca52df38a818a6acb99d21d8b59149.json b/test/testdata/98201bbc2eca52df38a818a6acb99d21d8b59149.json new file mode 100644 index 00000000..d7814833 --- /dev/null +++ b/test/testdata/98201bbc2eca52df38a818a6acb99d21d8b59149.json @@ -0,0 +1,34 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "417", + "Cache-Control": "public, max-age=1800", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Language": "en", + "Content-Length": "11930", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:55:05 GMT", + "Etag": "\"1495560025-1\"", + "Expires": "Sun, 19 Nov 1978 05:00:00 GMT", + "Fastly-Debug-Digest": "9ec820af22bafb88221c66a39795cfeb75f168d496d4f3f937e1385f127ca25a", + "Last-Modified": "Tue, 23 May 2017 17:20:25 GMT", + "Link": "; rel=\"image_src\",; rel=\"canonical\",; rel=\"shortlink\",; rel=\"publisher\"", + "Server": "nginx", + "Strict-Transport-Security": "max-age=31536000; includeSubdomains; preload", + "Vary": "Cookie,Accept-Encoding", + "Via": "1.1 varnish, 1.1 varnish, 1.1 varnish", + "X-Cache": "HIT, HIT", + "X-Cache-Hits": "2, 1", + "X-Content-Type-Options": "nosniff", + "X-Drupal-Cache": "HIT", + "X-Frame-Options": "SAMEORIGIN", + "X-Generator": "Drupal 7 (http://drupal.org)", + "X-Served-By": "cache-sjc3137-SJC, cache-iad2137-IAD", + "X-Timer": "S1495562106.662893,VS0,VE1", + "X-UA-Compatible": "IE=edge,chrome=1" + }, + "status_code": 200, + "url": "https://www.eff.org/" +} \ No newline at end of file diff --git a/test/testdata/9a5d30f4e0b4a0f1bb0a57364dd2b24f7eb23776.html b/test/testdata/9a5d30f4e0b4a0f1bb0a57364dd2b24f7eb23776.html new file mode 100644 index 00000000..76ad2340 --- /dev/null +++ b/test/testdata/9a5d30f4e0b4a0f1bb0a57364dd2b24f7eb23776.html @@ -0,0 +1,1461 @@ + + + + + + Sea otter return boosts ailing seagrass in California - BBC News + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
        + + +
        +
        +
        + + + + + + +
        + +
        + + + Science & Environment + + + + +
        + +
        + + Science & Environment + + + + +
        + +
        +

        Sea otter return boosts ailing seagrass in California

        + + + + + +
        +
        + + + sea otter ecology + + + + + +
        + Image caption + + A sea otter enjoys a crab in California, and helps seagrass in the process + +
        + +

        The return of sea otters to an estuary on the central Californian coast has significantly improved the health of seagrass, new research has found.

        Seagrass was deemed to be heading for extinction in this region before the otters returned.

        But scientists found that the animals triggered a chain reaction of events that boosted the water-dwelling plants.

        The research is published in the journal, PNAS.

        The urbanisation of California has led to a huge increase in nutrient pollution in coastal waters, from increasing use of nitrogen-rich fertilizers.

        This is said to be the reason for the dieback of seagrass, which has also been declining worldwide.

        This research suggests that the hunting to near-extinction of sea otters in the late 19th and early 20th Century may have exacerbated the problem, and conversely that their reintroduction is helping revive ailing seagrass populations, even in the face of hugely nutrient-rich water.

        Links in the chain

        The researchers assessed seagrass levels over the past 50 years in the Elkhorn Slough in Monterey Bay, and mapped their increases and declines.

        They looked at a variety of changes that may have affected the grass, but the only factor that really matched the changes in seagrass was sea otter numbers.

        They theorised that sea otters were eating the crabs which prey upon small invertebrates in the water.

        These invertebrates eat a type of algae which blooms when there are more nutrients in the soil. It grows on the leaves of the seagrass, shading them from sunlight and causing them to die back.

        This is quite a complex cascade of effects, so the researchers tested out their theory by comparing similar estuaries with and without sea otters, and by doing experiments in the lab, and in the field.

        These experiments, which included putting cages that sea otters either could or couldn't access, down on the seagrass, confirmed their hypothesis.

        + + + +
        + + +
        + +
        + Image caption + + Sea otters have been responsible for improving the health of the seagrass in these estuaries. + +
        + +

        Brent Hughes, lead author of the study, said: "This estuary is part of one of the most polluted systems in the entire world, but you can still get this healthy thriving habitat, and it's all because of the sea otters.

        "So it's almost like these sea otters are fighting the effects of poor water quality."

        Hughes described seagrass as "the canary in the coalmine" in terms of predicting levels of nutrient pollution in the water.

        Foundation species

        It also acts as a nursery habitat for many species of fish and it uses CO2 from sea water and the atmosphere, thus potentially helping with climate change.

        Not only that, but it acts as protection to the stability of the shoreline.

        Hughes said: "It's what we call a foundation species, like kelp forest, salt marsh or coral reef. The major problem from a global perspective is that seagrass is declining worldwide. And one of the major drivers of this decline has been nutrient inputs from anthropogenic sources, via agriculture or urban runoff."

        These findings are of particular interest at the moment, as a ban on sea otters moving along the coast to southern California was lifted last year. The ban was in place as there was a fear the sea otters would impinge on fisheries in the area.

        Hughes told BBC news: "That's important because there's a lot of these kind of degraded estuaries in southern California because of all the urban runoff from places like Los Angeles and San Diego.

        "Coastal managers will now have a better sense of what's going to happen when sea otters move in to their systems.

        "There's a huge potential benefit to sea otters returning to these estuaries, and in to these seagrass beds that might be threatened."

        +
        +
        + + + +
        +

        More on this story

        + +
        +

        Related Internet links

        +

        The BBC is not responsible for the content of external Internet sites

        +
        + + + + + + +
        + +
        + + + + + +
        + + + + +
        + +
        + + + +
        + + + + + + + + + + + + + + + + + diff --git a/test/testdata/9a5d30f4e0b4a0f1bb0a57364dd2b24f7eb23776.json b/test/testdata/9a5d30f4e0b4a0f1bb0a57364dd2b24f7eb23776.json new file mode 100644 index 00000000..a107dc6e --- /dev/null +++ b/test/testdata/9a5d30f4e0b4a0f1bb0a57364dd2b24f7eb23776.json @@ -0,0 +1,31 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "1", + "Cache-Control": "private, max-age=60, stale-while-revalidate", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Language": "en", + "Content-Length": "35305", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:52:57 GMT", + "Server": "Apache", + "Set-Cookie": "BBC-UID=6124a0ef56bb7b0b5e900c07661e1e355db0c132c17c0195a558a879311d393f0Mozilla%2F5.0%20%28Windows%20NT%2010.0%3B%20Win64%3B%20x64%3B%20rv%3A50.0%29%20Gecko%2F20100101%20Firefox%2F50.0; expires=Sat, 22 May 2021 17:52:57 GMT; path=/; domain=.bbc.com", + "Vary": "Accept-Encoding", + "Via": "1.1 varnish", + "X-Cache": "HIT", + "X-Cache-Action": "MISS", + "X-Cache-Age": "0", + "X-Cache-Hits": "1", + "X-Fastly-Cache-Status": "HIT-CLUSTER", + "X-LB-NoCache": "true", + "X-News-Cache-Id": "40708", + "X-News-Data-Centre": "telhc", + "X-PAL-Host": "pal181.back.live.telhc.local:80", + "X-Served-By": "cache-iad2124-IAD", + "X-Timer": "S1495561977.024664,VS0,VE1" + }, + "status_code": 200, + "url": "http://www.bbc.com/news/science-environment-23814524" +} \ No newline at end of file diff --git a/test/testdata/9b5f38900d6a3ce50d185c590a413af8c7e061b9.html b/test/testdata/9b5f38900d6a3ce50d185c590a413af8c7e061b9.html new file mode 100644 index 00000000..8f2689da --- /dev/null +++ b/test/testdata/9b5f38900d6a3ce50d185c590a413af8c7e061b9.html @@ -0,0 +1,182 @@ + + + + + + + + + +私とあの人の運命の出会い | 出会い系で知り合った二人の関係 + + + + + + + + + + + + + + +
        + + +
        +
        +
        + + +
        +
        +

        + 18歳独身女性ダンサーのセフレの作り方体験談 +

        +
        + +
        +

        私にはいま誰とも真面目に付き合う気がない。
        +人と付き合うことが面倒くさい。
        +それは仕事の人間関係も友達付き合いも、家族ともそう。
        +でも、それらは生きていく上では避けられないもので、
        +したくなくても、することになるもの。
        +唯一積極的に回避できる人間関係がなんなのか、面倒くさがりやの私は気がついてしまった。
        +それは、恋愛。
        +恋人関係だけは、望まなければ一生しなくてもいいものだ。

        +

        それに気づいてからは、誰とも付き合いたくなくなった。
        +でも年頃の女だ。性欲はある。
        +だからセフレがほしい。
        +セフレの作り方は考えるまでもなかった。
        +男というのは女以上に性欲に支配されやすいから女友達の誘いですら
        +断らない。
        +セフレの作り方は簡単だった。悩むまでもない。頭を使う必要性すらない。
        +男友達をそのままセフレにすればいい。
        +なんて合理的なんだろうか。
        +幸い、私は男友達は多い。女友達は少ないけど。

        +

        こんな価値観の私にも過去には彼氏がいた。
        +彼氏としたセックスとセフレとするセックスの違いが
        +わからない。

        +

        そう、わからないのだ。
        +そして、この、わからないという感覚こそが、
        +私がセフレを作る動機なのだろう。

        +

        彼氏とする性行為とセフレとする性行為の
        +違いがわかってしまう女はきっと、
        +セフレを持つことは、できない。

        +

        そしてたぶん、そういう女のほうが、
        +本当の愛、というものがなんなのかを知っている気がする。

        +

        一長一短だ。
        +私のように、わからない女は愛を失っても、なんとも思わないで済む。
        +でも、知っている女は喪失したときの傷や痛みも相当なものだ。
        +でもそういう人のほうが、きっと充実した恋愛ができるのだろう。

        +

        どっちが良いのかはわからない。
        +私はなにもわからない。

        +

        ただ確かなのは、この肉体で感じる快楽のみ。

        +
        + + +
        + +
        +
        +

        + 39歳既婚男性サラリーマンの出会い系体験談 +

        +
        + +
        +

        出会い系で遊ぶようになってもう6年程になりますが、やっぱりサクラや業者がうざったいなというのが
        +正直なところです。でもまあ最近ではある程度場数もこなしてきてポイント消費目当てのサクラや
        +業者は判断が付くようになってきました。
        +それと料体系が不明瞭なところでも痛い目をあったことがあるので消去法でやっていくと、
        +Jメールという出会い系サイトが主戦場となりました。Jメールの場合はポイントの前払い制なんで
        +まずその時点でリスクがないんですよね。後から予想外の経費がかかってしまってガックリ
        +なんてことにはならないわけです。
        +さらにはこちらのサイトですと私がやった実感の範囲で言えば、サクラや業者の割合が
        +極めて低いように思えるんですね。まあサクラや業者の特定はこちらからは論理的構造的に
        +不可能なので全て憶測でしかいえないんですが、この道6年の私の経験上の実感で言えば、
        +その意味でJメールは極めて優良なサイトの部類に入るのではと思っています。
        +今月もJメールからOLをゲットしましたしね。まあぶっちゃけ出会ってからはそのナンパ師の
        +技量が問われるところですが、少なくとも出会うまではどのサイトを使っているかがその
        +月のゲット率に大きく左右しているのは間違いないと思いますね。私もJメールを使う前は
        +今よりも半分以下の戦績でしたから、ナンパ師で出会いを探している人にはJメールはその
        +意味で教えたくないサイトの一つですね。私はこのような出会い系サイトをあと2つ知っています。

        +
        + + +
        + + + +
        +
        + + + +
        + +
        + + + + \ No newline at end of file diff --git a/test/testdata/9b5f38900d6a3ce50d185c590a413af8c7e061b9.json b/test/testdata/9b5f38900d6a3ce50d185c590a413af8c7e061b9.json new file mode 100644 index 00000000..c87dae9b --- /dev/null +++ b/test/testdata/9b5f38900d6a3ce50d185c590a413af8c7e061b9.json @@ -0,0 +1,15 @@ +{ + "encoding": "UTF-8", + "headers": { + "Connection": "Keep-Alive", + "Content-Type": "text/html; charset=UTF-8", + "Date": "Tue, 23 May 2017 18:00:22 GMT", + "Keep-Alive": "timeout=1, max=100", + "Server": "Apache", + "Transfer-Encoding": "chunked", + "X-Pad": "avoid browser bug", + "X-Pingback": "http://www.londondevelopmentcentre.org/xmlrpc.php" + }, + "status_code": 200, + "url": "http://www.londondevelopmentcentre.org/" +} \ No newline at end of file diff --git a/test/testdata/9d343541278598887f94cd65d9120a762a6803e7.html b/test/testdata/9d343541278598887f94cd65d9120a762a6803e7.html new file mode 100644 index 00000000..855a2ca9 --- /dev/null +++ b/test/testdata/9d343541278598887f94cd65d9120a762a6803e7.html @@ -0,0 +1,471 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +SMA reveals giant star cluster in the making + + + + + + + + + + + + + + + +
        + +
        +
        +
        + + + + + + + +
        +

        SMA reveals giant star cluster in the making

        + + + + + +
        +
        +
        +
        +
        +
        +
        +
          +
        • +
          +

          W49A might be one of the best-kept secrets in our galaxy. This star-forming region shines 100 times brighter than the Orion nebula, but is so obscured by dust that very little visible or infrared light escapes.

          +
          + +
          +

          The Smithsonian's Submillimeter Array (SMA) has peered through the dusty fog to provide the first clear view of this stellar nursery. The SMA revealed an active site of star formation being fed by streamers of infalling gas.

          +
          + +
          + + +
          + +
          +

          "We were amazed by all the features we saw in the SMA images," says lead author Roberto Galván-Madrid, who conducted this research at the Harvard-Smithsonian Center for Astrophysics (CfA) and the European Southern Observatory (ESO).

          +
          + +
          +

          W49A is located about 36,000 light-years from Earth, on the opposite side of the Milky Way. It represents a nearby example of the sort of vigorous star formation seen in so-called "starburst" galaxies, where stars form 100 times faster than in our galaxy.

          +
          + +
          +

          The heart of W49A holds a giant yet surprisingly compact star cluster. About 100,000 stars already exist within a space only 10 light-years on a side. In contrast, fewer than 10 stars lie within 10 light-years of our Sun. In a few million years, the giant star cluster in W49A will be almost as crowded as a globular cluster.

          +
          + +
          +

          The SMA also revealed an intricate network of filaments feeding gas into the center, much like tributaries feed water into mighty rivers on Earth. The gaseous filaments in W49A form three big streamers, which funnel star-building material inward at speeds of about 4,500 miles per hour (2 km/sec).

          +
          + +
          +

          "Move over, Mississippi!" quips co-author Qizhou Zhang of the CfA.

          +
          + +
          +

          Being denser than average will help the W49A star cluster to survive. Most star clusters in the galactic disk dissolve rapidly, their stars migrating away from each other under the influence of gravitational tides. This is why none of the Sun's sibling stars remain nearby. Since it is so compact, the cluster in W49A might remain intact for billions of years.

          +
          + +
          +

          The Submillimeter Array mapped the molecular gas within W49A in exquisite detail. It showed that central 30 light-years of W49A is several hundred times denser than the average molecular cloud in the Milky Way. In total, the nebula contains about 1 million suns' worth of gas, mostly molecular hydrogen.

          +
          + +
          +

          "We suspect that the organized architecture seen in W49A is rather common in massive stellar cluster-formation," adds co-author Hauyu Baobab Liu of the Academia Sinica Institute of Astronomy and Astrophysics (ASIAA) in Taiwan.

          +
          + +
          +

          The team expects to continue analyzing the SMA data for some time to come.

          +
          + +
          +

          "It's a mine of information," says Galván-Madrid.

          +
          + +
        • + +
        +
        + +
        + +
        +
        + +
        +
        +
        +
        +
        +

        Related Stories

        +
        + + + + + +
        +

        What Is Kratom and Is It Dangerous?

        + + + + + +
        +
        +
        + + + + + +
        +

        The best platform to create your e-commerce website

        + + + + + +
        +
        +
        + + + + + +
        +

        Is My Site Worth My Audience’s Time?

        + + + + + +
        +
        +
        +
        +
        + +
        + + + + + + + + + + + diff --git a/test/testdata/9d343541278598887f94cd65d9120a762a6803e7.json b/test/testdata/9d343541278598887f94cd65d9120a762a6803e7.json new file mode 100644 index 00000000..8c167091 --- /dev/null +++ b/test/testdata/9d343541278598887f94cd65d9120a762a6803e7.json @@ -0,0 +1,22 @@ +{ + "encoding": "UTF-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "0", + "Cache-Control": "max-age=60, public", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "8136", + "Content-Type": "text/html; charset=UTF-8", + "Date": "Tue, 23 May 2017 17:55:14 GMT", + "Vary": "Accept-Encoding", + "Via": "1.1 varnish", + "X-Cache": "MISS", + "X-Cache-Hits": "0", + "X-Powered-By": "PHP/5.5.9-1ubuntu4.20", + "X-Served-By": "cache-iad2642-IAD", + "X-Timer": "S1495562114.429966,VS0,VE412" + }, + "status_code": 200, + "url": "http://www.tgdaily.com/space-features/82906-sma-reveals-giant-star-cluster-in-the-making" +} \ No newline at end of file diff --git a/test/testdata/a38693590aed53ce2c21864cdd8b5d51760beb59.html b/test/testdata/a38693590aed53ce2c21864cdd8b5d51760beb59.html new file mode 100644 index 00000000..66deb291 --- /dev/null +++ b/test/testdata/a38693590aed53ce2c21864cdd8b5d51760beb59.html @@ -0,0 +1 @@ +{"result":{"total":999489,"groups":{"author":{"total":49,"items":[{"entity_type":"Author","author_title":"گروه عمومی دانشگاه علمی - کاربردی واحد 34 فرهنگ و","id":"Author-290481","url":"245677c8-e9b8-4748-9643-52f2dc27855f"}]},"printableBook":{"total":999425,"items":[{"book_subject":null,"book_parent_subject":["ادبیات"],"image":"https://pic.ketab.ir/DataBase/BookImages/86/86b02204.jpg","book_print_version":6,"book_cover_price":32000,"book_author":["سپهری ، سهراب"],"book_page_count":240,"url":"d2515c41-fd82-41f3-bfad-9262d3b43a3d","entity_type":"PrintableBook","book_title":"راز گل سرخ: نقد و گزیده شعرهای سهراب سپهری","book_issue_year":1386,"id":"Book-1348309","book_cover_type":"شومیز","book_publisher":"نگاه","book_volume_number":0}]},"publisher":{"total":15,"items":[{"image":"https://pic.ketab.ir/DataBase/Publishers/Arms/328375.jpg","entity_type":"Publisher","publisher_manager_fullname":" ","publisher_title":"سه سه تار","id":"Publisher-328375","url":"61cfafeb-cd3c-4fcd-8bb9-b5d1d17a8de2"}]}},"from":0},"facets":{"book_issue_year":{"1385":51867,"1396":42621,"1384":51198,"1395":41884,"1394":41789,"1383":40168,"1393":42214,"1382":35571,"1392":41755,"1381":32250,"1391":43021,"1390":49202,"1389":50914,"1400":38884,"1388":51792,"1399":36786,"1387":52592,"1398":44216,"1386":54869,"1397":41151},"book_parent_subject":{"آموزشی":90271,"فلسفه":40218,"ادبیات":155355,"کودک":147994,"کمک درسی":96741,"علوم طبیعی و ریاضیات":15935,"هنر":29855,"کمک درسی کودک":19779,"علوم اجتماعی":79941,"دانشگاهی":1,"دین":158934,"تاریخ و جغرافیا":42826,"علوم عملی":101666,"کلیات":1,"زبان":19908},"book_print_version_type":{"چاپ مجدد":517756,"چاپ اول":481669},"book_publisher":{"سمت":11120,"نشر نی":5496,"دانشگاه پیام نور":10680,"بین المللی گاج":11540,"موسسه فرهنگی مدرسه برهان":11065,"نشر چشمه":4571,"آیندگان هزار نکته وابسته به موسسه فرهنگی آیندگان هزار نکته":7227,"موسسه بوستان کتاب":6569,"خیلی سبز":4371,"قدیانی":11324,"پیدایش":4460,"مبتکران":14654,"شرکت انتشارات کانون فرهنگی آموزش":12307,"به نشر وابسته به آستان قدس رضوی":5664,"مدرسان شریف":12360,"شرکت انتشارات سوره مهر":4583,"امیرکبیر":6475,"موسسه چاپ و انتشارات دانشگاه تهران":5727,"نشر مرکز":5680,"افق":7275},"book_author":{"احمدی‌جزی ، کامران":858,"تریسی ، برایان":1290,"بازرگانی ، بهمن":1807,"محدثی ، جواد":965,"هیات ‌مولفان":1741,"شعبانی ، اسدالله":963,"فتاحی ، حسین":1437,"مطهری ، مرتضی":3450,"طباطبایی ، سیدمحمدحسین":1065,"موسوی ، سیدعلی":911,"قاسم‌نیا ، شکوه":1800,"سبحانی‌تبریزی ، جعفر":931,"مولوی ، جلال‌الدین‌محمدبن‌محمد":1323,"کوییلو ، پایولو":847,"انصاری ، حسین":835,"دپارتمان ا‌یندگان":2407,"عمیق ، مجید":959,"رحماندوست ، مصطفی":847,"فلاح‌زاده ، محمدحسین":921,"موحدی ، محمود":1694,"کشاورز ، ناصر":1949,"مکارم‌شیرازی ، ناصر":1996,"ال‌احمد ، جلال":1081,"حافظ ، شمس‌الدین‌محمد":2225,"حیدری‌ابهری ، غلامرضا":928,"حامی ، فرهاد":865,"قمی ، عباس":4996,"قرایتی ، محسن":1460,"کیانی ، مصطفی":868,"مجلسی ، محمدباقربن‌محمدتقی":1320,"قراچه‌داغی ، مهدی":2019,"فاضلی ، بنفشه":921,"جوادی‌املی ، عبدالله":1257,"وحیدی‌صدر ، مهدی":967,"اخلاصمندمنفرد ، علیرضا":1695,"صفایی‌دیبا ، علی‌اکبر":892,"نامی ، حسین":1269,"الهی‌قمشه‌ای ، مهدی":6641,"گروه مولفان":1236,"نیکوکار ، مسعود":939,"نجف‌خانی ، محبوبه":885,"گراس ، تونی":1205,"هیراتا ، شاگا":816,"استاین ، ار.ال.":1063,"فردوسی ، ابوالقاسم":939,"سعدی ، مصلح‌بن‌عبدالله":1310,"طالب‌تبار ، حمید":1050,"اناری ، شهاب":1341,"اعضای هیات علمی سنجش تکمیلی":1332,"محمدی‌ری‌شهری ، محمد":1249},"listModel":{"book_issue_year":[{"label":"1385","value":51867},{"label":"1396","value":42621},{"label":"1384","value":51198},{"label":"1395","value":41884},{"label":"1394","value":41789},{"label":"1383","value":40168},{"label":"1393","value":42214},{"label":"1382","value":35571},{"label":"1392","value":41755},{"label":"1381","value":32250},{"label":"1391","value":43021},{"label":"1390","value":49202},{"label":"1389","value":50914},{"label":"1400","value":38884},{"label":"1388","value":51792},{"label":"1399","value":36786},{"label":"1387","value":52592},{"label":"1398","value":44216},{"label":"1386","value":54869},{"label":"1397","value":41151}],"book_print_version_type":[{"label":"چاپ مجدد","value":517756},{"label":"چاپ اول","value":481669}],"book_parent_subject":[{"label":"آموزشی","value":90271},{"label":"فلسفه","value":40218},{"label":"ادبیات","value":155355},{"label":"کودک","value":147994},{"label":"کمک درسی","value":96741},{"label":"علوم طبیعی و ریاضیات","value":15935},{"label":"هنر","value":29855},{"label":"کمک درسی کودک","value":19779},{"label":"علوم اجتماعی","value":79941},{"label":"دانشگاهی","value":1},{"label":"دین","value":158934},{"label":"تاریخ و جغرافیا","value":42826},{"label":"علوم عملی","value":101666},{"label":"کلیات","value":1},{"label":"زبان","value":19908}],"book_publisher":[{"label":"سمت","value":11120},{"label":"نشر نی","value":5496},{"label":"دانشگاه پیام نور","value":10680},{"label":"بین المللی گاج","value":11540},{"label":"موسسه فرهنگی مدرسه برهان","value":11065},{"label":"نشر چشمه","value":4571},{"label":"آیندگان هزار نکته وابسته به موسسه فرهنگی آیندگان هزار نکته","value":7227},{"label":"موسسه بوستان کتاب","value":6569},{"label":"خیلی سبز","value":4371},{"label":"قدیانی","value":11324},{"label":"پیدایش","value":4460},{"label":"مبتکران","value":14654},{"label":"شرکت انتشارات کانون فرهنگی آموزش","value":12307},{"label":"به نشر وابسته به آستان قدس رضوی","value":5664},{"label":"مدرسان شریف","value":12360},{"label":"شرکت انتشارات سوره مهر","value":4583},{"label":"امیرکبیر","value":6475},{"label":"موسسه چاپ و انتشارات دانشگاه تهران","value":5727},{"label":"نشر مرکز","value":5680},{"label":"افق","value":7275}],"book_author":[{"label":"احمدی‌جزی ، کامران","value":858},{"label":"تریسی ، برایان","value":1290},{"label":"بازرگانی ، بهمن","value":1807},{"label":"محدثی ، جواد","value":965},{"label":"هیات ‌مولفان","value":1741},{"label":"شعبانی ، اسدالله","value":963},{"label":"فتاحی ، حسین","value":1437},{"label":"مطهری ، مرتضی","value":3450},{"label":"طباطبایی ، سیدمحمدحسین","value":1065},{"label":"موسوی ، سیدعلی","value":911},{"label":"قاسم‌نیا ، شکوه","value":1800},{"label":"سبحانی‌تبریزی ، جعفر","value":931},{"label":"مولوی ، جلال‌الدین‌محمدبن‌محمد","value":1323},{"label":"کوییلو ، پایولو","value":847},{"label":"انصاری ، حسین","value":835},{"label":"دپارتمان ا‌یندگان","value":2407},{"label":"عمیق ، مجید","value":959},{"label":"رحماندوست ، مصطفی","value":847},{"label":"فلاح‌زاده ، محمدحسین","value":921},{"label":"موحدی ، محمود","value":1694},{"label":"کشاورز ، ناصر","value":1949},{"label":"مکارم‌شیرازی ، ناصر","value":1996},{"label":"ال‌احمد ، جلال","value":1081},{"label":"حافظ ، شمس‌الدین‌محمد","value":2225},{"label":"حیدری‌ابهری ، غلامرضا","value":928},{"label":"حامی ، فرهاد","value":865},{"label":"قمی ، عباس","value":4996},{"label":"قرایتی ، محسن","value":1460},{"label":"کیانی ، مصطفی","value":868},{"label":"مجلسی ، محمدباقربن‌محمدتقی","value":1320},{"label":"قراچه‌داغی ، مهدی","value":2019},{"label":"فاضلی ، بنفشه","value":921},{"label":"جوادی‌املی ، عبدالله","value":1257},{"label":"وحیدی‌صدر ، مهدی","value":967},{"label":"اخلاصمندمنفرد ، علیرضا","value":1695},{"label":"صفایی‌دیبا ، علی‌اکبر","value":892},{"label":"نامی ، حسین","value":1269},{"label":"الهی‌قمشه‌ای ، مهدی","value":6641},{"label":"گروه مولفان","value":1236},{"label":"نیکوکار ، مسعود","value":939},{"label":"نجف‌خانی ، محبوبه","value":885},{"label":"گراس ، تونی","value":1205},{"label":"هیراتا ، شاگا","value":816},{"label":"استاین ، ار.ال.","value":1063},{"label":"فردوسی ، ابوالقاسم","value":939},{"label":"سعدی ، مصلح‌بن‌عبدالله","value":1310},{"label":"طالب‌تبار ، حمید","value":1050},{"label":"اناری ، شهاب","value":1341},{"label":"اعضای هیات علمی سنجش تکمیلی","value":1332},{"label":"محمدی‌ری‌شهری ، محمد","value":1249}]}},"spelling":null} \ No newline at end of file diff --git a/test/testdata/a38693590aed53ce2c21864cdd8b5d51760beb59.json b/test/testdata/a38693590aed53ce2c21864cdd8b5d51760beb59.json new file mode 100644 index 00000000..d9c3a224 --- /dev/null +++ b/test/testdata/a38693590aed53ce2c21864cdd8b5d51760beb59.json @@ -0,0 +1,21 @@ +{ + "encoding": "utf-8", + "headers": { + "AR-ATIME": "2.209", + "AR-CACHE": "BYPASS", + "AR-PoweredBy": "Arvan Cloud (arvancloud.com)", + "AR-Request-ID": "62bdc1cb1824dbf42cce82c9d3fd0264", + "AR-SID": "2012", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Type": "application/json; charset=utf-8", + "Date": "Sat, 27 Aug 2022 23:42:44 GMT", + "Keep-Alive": "timeout=65", + "Server": "ArvanCloud", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding, Accept-Encoding", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://msapi.ketab.ir/search/?query=964-6736-34-3&limit=1" +} \ No newline at end of file diff --git a/test/testdata/a49b8d5a96aca55c43ca4b5818e69080d92dfe0d.html b/test/testdata/a49b8d5a96aca55c43ca4b5818e69080d92dfe0d.html new file mode 100644 index 00000000..e44747dd --- /dev/null +++ b/test/testdata/a49b8d5a96aca55c43ca4b5818e69080d92dfe0d.html @@ -0,0 +1,438 @@ + + + + Wayback Machine + Internet Archive Wayback Machine + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
        + + +
        + + + + +
        +
        +

        + The Wayback Machine is an initiative of the + Internet Archive, + a 501(c)(3) non-profit, building a digital library of + Internet sites and other cultural artifacts in digital form. +
        Other projects include + Open Library & + archive-it.org. +

        +

        + Your use of the Wayback Machine is subject to the Internet Archive's + Terms of Use. +

        +
        +
        + + diff --git a/test/testdata/a49b8d5a96aca55c43ca4b5818e69080d92dfe0d.json b/test/testdata/a49b8d5a96aca55c43ca4b5818e69080d92dfe0d.json new file mode 100644 index 00000000..7f68ad10 --- /dev/null +++ b/test/testdata/a49b8d5a96aca55c43ca4b5818e69080d92dfe0d.json @@ -0,0 +1,20 @@ +{ + "encoding": "utf-8", + "headers": { + "Age": "6", + "Connection": "close", + "Content-Encoding": "gzip", + "Content-Length": "6014", + "Content-Type": "text/html; charset=utf-8", + "Date": "Wed, 24 May 2017 08:14:16 GMT", + "Server": "Tengine/2.1.0", + "Vary": "Accept-Encoding", + "X-Archive-Playback": "0", + "X-Cache": "HIT from google.com", + "X-Cache-Lookup": "HIT from google.com:86", + "X-Page-Cache": "HIT", + "X-location": "All" + }, + "status_code": 200, + "url": "http://web.archive.org/" +} \ No newline at end of file diff --git a/test/testdata/a522d7b31acb54155e4eeafd8c59e8246232cc6e.html b/test/testdata/a522d7b31acb54155e4eeafd8c59e8246232cc6e.html new file mode 100644 index 00000000..5ce10b70 --- /dev/null +++ b/test/testdata/a522d7b31acb54155e4eeafd8c59e8246232cc6e.html @@ -0,0 +1,35 @@ +
        + \ No newline at end of file diff --git a/test/testdata/a522d7b31acb54155e4eeafd8c59e8246232cc6e.json b/test/testdata/a522d7b31acb54155e4eeafd8c59e8246232cc6e.json new file mode 100644 index 00000000..92313e7e --- /dev/null +++ b/test/testdata/a522d7b31acb54155e4eeafd8c59e8246232cc6e.json @@ -0,0 +1,21 @@ +{ + "encoding": "utf-8", + "headers": { + "CF-Cache-Status": "DYNAMIC", + "CF-RAY": "74453d69fb208fd1-FRA", + "Cache-Control": "private, no-cache, no-store, max-age=0, must-revalidate", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Security-Policy": "frame-ancestors 'none'", + "Content-Type": "text/html; charset=utf-8", + "Date": "Fri, 02 Sep 2022 09:32:42 GMT", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=15724800; includeSubDomains", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding", + "X-Frame-Options": "SAMEORIGIN", + "X-Powered-By": "Next.js" + }, + "status_code": 200, + "url": "https://www.worldcat.org/title/24680975" +} \ No newline at end of file diff --git a/test/testdata/a5cd54054572628631ad49f5af470fc741690900.html b/test/testdata/a5cd54054572628631ad49f5af470fc741690900.html new file mode 100644 index 00000000..95a395a0 --- /dev/null +++ b/test/testdata/a5cd54054572628631ad49f5af470fc741690900.html @@ -0,0 +1 @@ +{"result":{"total":1532369,"groups":{"author":{"total":43,"items":[{"entity_type":"Author","author_title":"هشت ، بورکهارد","id":"Author-173267","url":"3f8685d8-37bb-4a21-a2b0-f6ffb302bb57"}]},"printableBook":{"total":1532286,"items":[{"book_subject":null,"book_parent_subject":["آموزشی"],"image":"https://pic.ketab.ir/DataBase/BookImages/79/79523022.jpg","book_print_version":1,"book_cover_price":10000,"book_author":["دیماتیو ، ام.رابین","کاویانی ، محمد"],"book_page_count":422,"url":"4cc231f9-35c2-4b60-a714-a0a11135e932","entity_type":"PrintableBook","book_title":"روانشناسی سلامت به ضمیمه نگرشی بر منابع اسلامی","book_issue_year":1379,"id":"Book-227129","book_cover_type":"شومیز","book_publisher":"سمت","book_volume_number":1}]},"publisher":{"total":40,"items":[{"image":"https://pic.ketab.ir/DataBase/Publishers/Arms/309074.jpg","entity_type":"Publisher","publisher_manager_fullname":" ","publisher_title":"هشت","id":"Publisher-309074","url":"a38ffbb0-5a52-4857-a6d1-27f5f3dfb340"}]}},"from":0},"facets":{"book_issue_year":{"1396":97903,"1385":51820,"1395":88801,"1384":51103,"1394":81046,"1383":40152,"1393":73237,"1382":35531,"1392":65743,"1391":63260,"1390":67758,"1401":38912,"1400":110301,"1389":64285,"1399":93722,"1388":59918,"1398":104050,"1387":56105,"1397":99594,"1386":55502},"book_parent_subject":{"آموزشی":139280,"فلسفه":65706,"ادبیات":252122,"کودک":219108,"کمک درسی":137888,"علوم طبیعی و ریاضیات":23815,"هنر":45307,"کمک درسی کودک":31623,"علوم اجتماعی":140914,"دانشگاهی":2,"دین":206986,"تاریخ و جغرافیا":67502,"علوم عملی":173552,"کلیات":2,"زبان":28479},"book_print_version_type":{"چاپ مجدد":673964,"چاپ اول":858322},"book_publisher":{"سمت":12585,"نشر نی":5688,"دانشگاه پیام نور":10679,"پرتقال(وابسته به موسسه انتشاراتی سرزمین بچه های خوشحال)":5849,"بین المللی گاج":17440,"موسسه فرهنگی مدرسه برهان":11062,"نشر چشمه":7742,"آیندگان هزار نکته وابسته به موسسه فرهنگی آیندگان هزار نکته":7281,"آموزشی تالیفی ارشدان":5817,"خیلی سبز":15916,"موسسه بوستان کتاب":6574,"قدیانی":13933,"مبتکران":14664,"شرکت انتشارات کانون فرهنگی آموزش":18792,"شرکت نشر قطره":6208,"مدرسان شریف":14274,"شرکت انتشارات سوره مهر":7122,"امیرکبیر":6567,"موسسه چاپ و انتشارات دانشگاه تهران":5734,"افق":8748},"book_author":{"احمدی‌جزی ، کامران":1039,"تریسی ، برایان":2179,"نصری ، کمیل":1197,"صادقی ، داریوش":1161,"بازرگانی ، بهمن":1803,"محدثی ، جواد":1012,"هیات ‌مولفان":1926,"شعبانی ، اسدالله":1212,"فتاحی ، حسین":1768,"مطهری ، مرتضی":3596,"طباطبایی ، سیدمحمدحسین":1181,"شیخی ، مژگان":1047,"قاسم‌نیا ، شکوه":2245,"سبحانی‌تبریزی ، جعفر":1040,"مولوی ، جلال‌الدین‌محمدبن‌محمد":1972,"هیات مولفان کانون فرهنگی اموزش":1645,"کوییلو ، پایولو":1156,"سپهری ، نیما":1033,"دپارتمان ا‌یندگان":2407,"عمیق ، مجید":1072,"موحدی ، محمود":1698,"کشاورز ، ناصر":2220,"مکارم‌شیرازی ، ناصر":2098,"ال‌احمد ، جلال":1668,"حافظ ، شمس‌الدین‌محمد":2624,"حیدری‌ابهری ، غلامرضا":1098,"حامی ، فرهاد":1151,"قمی ، عباس":5903,"قرایتی ، محسن":1637,"کیانی ، مصطفی":1110,"مجلسی ، محمدباقربن‌محمدتقی":1511,"قراچه‌داغی ، مهدی":2283,"عبدالمحمدی ، علیرضا":1146,"جوادی‌املی ، عبدالله":1359,"وحیدی‌صدر ، مهدی":1081,"اخلاصمندمنفرد ، علیرضا":1698,"نامی ، حسین":1435,"الهی‌قمشه‌ای ، مهدی":7680,"گروه مولفان":1727,"نجف‌خانی ، محبوبه":1227,"گراس ، تونی":1329,"فردوسی ، ابوالقاسم":1391,"استاین ، ار.ال.":1225,"سیاری ، مجید":1128,"سعدی ، مصلح‌بن‌عبدالله":2150,"طالب‌تبار ، حمید":1051,"اناری ، شهاب":1380,"اعضای هیات علمی سنجش تکمیلی":1332,"محمدی‌ری‌شهری ، محمد":1270,"میرزایی‌دلاویز ، محمود":1126},"listModel":{"book_issue_year":[{"label":"1396","value":97903},{"label":"1385","value":51820},{"label":"1395","value":88801},{"label":"1384","value":51103},{"label":"1394","value":81046},{"label":"1383","value":40152},{"label":"1393","value":73237},{"label":"1382","value":35531},{"label":"1392","value":65743},{"label":"1391","value":63260},{"label":"1390","value":67758},{"label":"1401","value":38912},{"label":"1400","value":110301},{"label":"1389","value":64285},{"label":"1399","value":93722},{"label":"1388","value":59918},{"label":"1398","value":104050},{"label":"1387","value":56105},{"label":"1397","value":99594},{"label":"1386","value":55502}],"book_print_version_type":[{"label":"چاپ مجدد","value":673964},{"label":"چاپ اول","value":858322}],"book_parent_subject":[{"label":"آموزشی","value":139280},{"label":"فلسفه","value":65706},{"label":"ادبیات","value":252122},{"label":"کودک","value":219108},{"label":"کمک درسی","value":137888},{"label":"علوم طبیعی و ریاضیات","value":23815},{"label":"هنر","value":45307},{"label":"کمک درسی کودک","value":31623},{"label":"علوم اجتماعی","value":140914},{"label":"دانشگاهی","value":2},{"label":"دین","value":206986},{"label":"تاریخ و جغرافیا","value":67502},{"label":"علوم عملی","value":173552},{"label":"کلیات","value":2},{"label":"زبان","value":28479}],"book_publisher":[{"label":"سمت","value":12585},{"label":"نشر نی","value":5688},{"label":"دانشگاه پیام نور","value":10679},{"label":"پرتقال(وابسته به موسسه انتشاراتی سرزمین بچه های خوشحال)","value":5849},{"label":"بین المللی گاج","value":17440},{"label":"موسسه فرهنگی مدرسه برهان","value":11062},{"label":"نشر چشمه","value":7742},{"label":"آیندگان هزار نکته وابسته به موسسه فرهنگی آیندگان هزار نکته","value":7281},{"label":"آموزشی تالیفی ارشدان","value":5817},{"label":"خیلی سبز","value":15916},{"label":"موسسه بوستان کتاب","value":6574},{"label":"قدیانی","value":13933},{"label":"مبتکران","value":14664},{"label":"شرکت انتشارات کانون فرهنگی آموزش","value":18792},{"label":"شرکت نشر قطره","value":6208},{"label":"مدرسان شریف","value":14274},{"label":"شرکت انتشارات سوره مهر","value":7122},{"label":"امیرکبیر","value":6567},{"label":"موسسه چاپ و انتشارات دانشگاه تهران","value":5734},{"label":"افق","value":8748}],"book_author":[{"label":"احمدی‌جزی ، کامران","value":1039},{"label":"تریسی ، برایان","value":2179},{"label":"نصری ، کمیل","value":1197},{"label":"صادقی ، داریوش","value":1161},{"label":"بازرگانی ، بهمن","value":1803},{"label":"محدثی ، جواد","value":1012},{"label":"هیات ‌مولفان","value":1926},{"label":"شعبانی ، اسدالله","value":1212},{"label":"فتاحی ، حسین","value":1768},{"label":"مطهری ، مرتضی","value":3596},{"label":"طباطبایی ، سیدمحمدحسین","value":1181},{"label":"شیخی ، مژگان","value":1047},{"label":"قاسم‌نیا ، شکوه","value":2245},{"label":"سبحانی‌تبریزی ، جعفر","value":1040},{"label":"مولوی ، جلال‌الدین‌محمدبن‌محمد","value":1972},{"label":"هیات مولفان کانون فرهنگی اموزش","value":1645},{"label":"کوییلو ، پایولو","value":1156},{"label":"سپهری ، نیما","value":1033},{"label":"دپارتمان ا‌یندگان","value":2407},{"label":"عمیق ، مجید","value":1072},{"label":"موحدی ، محمود","value":1698},{"label":"کشاورز ، ناصر","value":2220},{"label":"مکارم‌شیرازی ، ناصر","value":2098},{"label":"ال‌احمد ، جلال","value":1668},{"label":"حافظ ، شمس‌الدین‌محمد","value":2624},{"label":"حیدری‌ابهری ، غلامرضا","value":1098},{"label":"حامی ، فرهاد","value":1151},{"label":"قمی ، عباس","value":5903},{"label":"قرایتی ، محسن","value":1637},{"label":"کیانی ، مصطفی","value":1110},{"label":"مجلسی ، محمدباقربن‌محمدتقی","value":1511},{"label":"قراچه‌داغی ، مهدی","value":2283},{"label":"عبدالمحمدی ، علیرضا","value":1146},{"label":"جوادی‌املی ، عبدالله","value":1359},{"label":"وحیدی‌صدر ، مهدی","value":1081},{"label":"اخلاصمندمنفرد ، علیرضا","value":1698},{"label":"نامی ، حسین","value":1435},{"label":"الهی‌قمشه‌ای ، مهدی","value":7680},{"label":"گروه مولفان","value":1727},{"label":"نجف‌خانی ، محبوبه","value":1227},{"label":"گراس ، تونی","value":1329},{"label":"فردوسی ، ابوالقاسم","value":1391},{"label":"استاین ، ار.ال.","value":1225},{"label":"سیاری ، مجید","value":1128},{"label":"سعدی ، مصلح‌بن‌عبدالله","value":2150},{"label":"طالب‌تبار ، حمید","value":1051},{"label":"اناری ، شهاب","value":1380},{"label":"اعضای هیات علمی سنجش تکمیلی","value":1332},{"label":"محمدی‌ری‌شهری ، محمد","value":1270},{"label":"میرزایی‌دلاویز ، محمود","value":1126}]}},"spelling":null} \ No newline at end of file diff --git a/test/testdata/a5cd54054572628631ad49f5af470fc741690900.json b/test/testdata/a5cd54054572628631ad49f5af470fc741690900.json new file mode 100644 index 00000000..3592438e --- /dev/null +++ b/test/testdata/a5cd54054572628631ad49f5af470fc741690900.json @@ -0,0 +1,21 @@ +{ + "encoding": "utf-8", + "headers": { + "AR-ATIME": "3.227", + "AR-CACHE": "BYPASS", + "AR-PoweredBy": "Arvan Cloud (arvancloud.com)", + "AR-Request-ID": "7f27b70f1e6b93437cf6ef23c2b8b75c", + "AR-SID": "2012", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Type": "application/json; charset=utf-8", + "Date": "Sat, 27 Aug 2022 23:18:02 GMT", + "Keep-Alive": "timeout=65", + "Server": "ArvanCloud", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding, Accept-Encoding", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://msapi.ketab.ir/search/?query=978-964-459-398-7&limit=1" +} \ No newline at end of file diff --git a/test/testdata/a5d933df7d8d3a35d30995351f7574b12ff252ae.html b/test/testdata/a5d933df7d8d3a35d30995351f7574b12ff252ae.html new file mode 100644 index 00000000..781ed2e1 --- /dev/null +++ b/test/testdata/a5d933df7d8d3a35d30995351f7574b12ff252ae.html @@ -0,0 +1 @@ +[{"itemType":"book","title":"Dīwān-i H̱āqānī-i Širwānī Muṭābiq-i nusẖah-yi ẖaṭṭī-yi 763 hiǧrī","ISBN":["964-6736-71-8","978-964-6736-71-9"],"edition":"Čāp-i 1","place":"Tihrān","date":"1375 [1996]","numPages":"1 v. (762 Seiten)","oclc":"1176150182","url":"https://www.worldcat.org/oclc/1176150182","contributor":[["Ǧahāngīr","Manṣūr"],["Badīʿ al-Zamān","Furūzānfar"]],"author":[["Afḍal al-Dīn Badīl?-1198?","H̱āqānī-i Širwānī"]],"accessDate":"2022-01-08","source":["WorldCat"]}] \ No newline at end of file diff --git a/test/testdata/a5d933df7d8d3a35d30995351f7574b12ff252ae.json b/test/testdata/a5d933df7d8d3a35d30995351f7574b12ff252ae.json new file mode 100644 index 00000000..87077c2b --- /dev/null +++ b/test/testdata/a5d933df7d8d3a35d30995351f7574b12ff252ae.json @@ -0,0 +1,37 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "0", + "Connection": "keep-alive", + "NEL": "{ \"report_to\": \"wm_nel\", \"max_age\": 86400, \"failure_fraction\": 0.05, \"success_fraction\": 0.0}", + "Permissions-Policy": "interest-cohort=()", + "Report-To": "{ \"group\": \"wm_nel\", \"max_age\": 86400, \"endpoints\": [{ \"url\": \"https://intake-logging.wikimedia.org/v1/events?stream=w3c.reportingapi.network_error&schema_uri=/w3c/reportingapi/network_error/1.0.0\" }] }", + "Server-Timing": "cache;desc=\"pass\", host;desc=\"cp3060\"", + "Set-Cookie": "WMF-Last-Access=08-Jan-2022;Path=/;HttpOnly;secure;Expires=Wed, 09 Feb 2022 12:00:00 GMT, WMF-Last-Access-Global=08-Jan-2022;Path=/;Domain=.wikipedia.org;HttpOnly;secure;Expires=Wed, 09 Feb 2022 12:00:00 GMT, GeoIP=IR:09:Mashhad:36.30:59.59:v4; Path=/; secure; Domain=.wikipedia.org", + "Strict-Transport-Security": "max-age=106384710; includeSubDomains; preload", + "X-Cache": "cp3050 miss, cp3060 pass", + "X-Cache-Status": "pass", + "X-Client-IP": "31.14.145.3", + "access-control-allow-headers": "accept, content-type, content-length, cache-control, accept-language, api-user-agent, if-match, if-modified-since, if-none-match, dnt, accept-encoding", + "access-control-allow-methods": "GET,HEAD", + "access-control-allow-origin": "*", + "access-control-expose-headers": "etag", + "cache-control": "private, max-age=0, s-maxage=0, must-revalidate", + "content-length": "557", + "content-location": "https://en.wikipedia.org/api/rest_v1/data/citation/mediawiki/978-964-6736-71-9", + "content-security-policy": "default-src 'none'; frame-ancestors 'none'", + "content-type": "application/json; charset=utf-8", + "date": "Sat, 08 Jan 2022 14:34:58 GMT", + "referrer-policy": "origin-when-cross-origin", + "server": "restbase1022", + "vary": "Accept-Encoding", + "x-content-security-policy": "default-src 'none'; frame-ancestors 'none'", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-webkit-csp": "default-src 'none'; frame-ancestors 'none'", + "x-xss-protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://en.wikipedia.org/api/rest_v1/data/citation/mediawiki/978-964-6736-71-9" +} \ No newline at end of file diff --git a/test/testdata/aa5b23bdc9350cee0d1eac7feceeae008638c542.html b/test/testdata/aa5b23bdc9350cee0d1eac7feceeae008638c542.html new file mode 100644 index 00000000..400f6119 --- /dev/null +++ b/test/testdata/aa5b23bdc9350cee0d1eac7feceeae008638c542.html @@ -0,0 +1 @@ +{"indexed":{"date-parts":[[2022,5,16]],"date-time":"2022-05-16T14:27:11Z","timestamp":1652711231760},"reference-count":50,"publisher":"Springer Science and Business Media LLC","issue":"10","content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"published-print":{"date-parts":[[2017,10]]},"DOI":"10.1007\/jhep10(2017)157","type":"journal-article","created":{"date-parts":[[2017,10,24]],"date-time":"2017-10-24T16:05:23Z","timestamp":1508861123000},"update-policy":"http:\/\/dx.doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":29,"title":"Strange and charm HVP contributions to the muon (g \u2212 2) including QED corrections with twisted-mass fermions","prefix":"10.1007","volume":"2017","author":[{"given":"D.","family":"Giusti","sequence":"first","affiliation":[]},{"name":"on behalf of ETM collaboration","sequence":"first","affiliation":[]},{"given":"V.","family":"Lubicz","sequence":"additional","affiliation":[]},{"given":"G.","family":"Martinelli","sequence":"additional","affiliation":[]},{"given":"F.","family":"Sanfilippo","sequence":"additional","affiliation":[]},{"ORCID":"http:\/\/orcid.org\/0000-0002-5533-6746","authenticated-orcid":false,"given":"S.","family":"Simula","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2017,10,23]]},"reference":[{"key":"6916_CR1","doi-asserted-by":"crossref","unstructured":"Muon g-2 collaboration, G.W. Bennett et al., Final Report of the Muon E821 Anomalous Magnetic Moment Measurement at BNL, Phys. Rev. D 73 (2006) 072003 [\nhep-ex\/0602035\n\n] [\nINSPIRE\n\n].","DOI":"10.1103\/PhysRevD.73.072003"},{"key":"6916_CR2","doi-asserted-by":"crossref","unstructured":"Particle Data Group collaboration, C. Patrignani et al., Review of Particle Physics, Chin. Phys. C 40 (2016) 100001 [\nINSPIRE\n\n].","DOI":"10.1088\/1674-1137\/40\/10\/100001"},{"key":"6916_CR3","doi-asserted-by":"crossref","unstructured":"Muon g \u2212 2 collaboration, I. Logashenko et al., The Measurement of the Anomalous Magnetic Moment of the Muon at Fermilab, J. Phys. Chem. Ref. Data 44 (2015) 031211 [\nINSPIRE\n\n].","DOI":"10.1063\/1.4917553"},{"key":"6916_CR4","unstructured":"E34 collaboration, M. Otani, Design of the J-PARC MUSE H-line for the Muon g \u2212 2\/EDM Experiment at J-PARC (E34), JPS Conf. Proc. 8 (2015) 025010 [\nINSPIRE\n\n]."},{"key":"6916_CR5","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1016\/j.physrep.2009.04.003","volume":"477","author":"F Jegerlehner","year":"2009","unstructured":"F. Jegerlehner and A. Nyffeler, The Muon g \u2212 2, Phys. Rept. 477 (2009) 1 [\narXiv:0902.3360\n\n] [\nINSPIRE\n\n].","journal-title":"Phys. Rept."},{"key":"6916_CR6","doi-asserted-by":"crossref","unstructured":"M. Davier, A. Hoecker, B. Malaescu and Z. Zhang, Reevaluation of the Hadronic Contributions to the Muon g \u2212 2 and to \u03b1(M\n \n \n Z\n \n 2\n ), Eur. Phys. J. C 71 (2011) 1515 [Erratum ibid. C 72 (2012) 1874] [\narXiv:1010.4180\n\n] [\nINSPIRE\n\n].","DOI":"10.1140\/epjc\/s10052-010-1515-z"},{"key":"6916_CR7","doi-asserted-by":"crossref","unstructured":"K. Hagiwara, R. Liao, A.D. Martin, D. Nomura and T. Teubner, (g \u2212 2)\n \u03bc\n and \u03b1(M\n \n \n Z\n \n 2\n ) re-evaluated using new precise data, J. Phys. G 38 (2011) 085003 [\narXiv:1105.3149\n\n] [\nINSPIRE\n\n].","DOI":"10.1088\/0954-3899\/38\/8\/085003"},{"key":"6916_CR8","doi-asserted-by":"crossref","unstructured":"B.e. Lautrup, A. Peterman and E. de Rafael, Recent developments in the comparison between theory and experiments in quantum electrodynamics, Phys. Rept. 3 (1972) 193 [\nINSPIRE\n\n].","DOI":"10.1016\/0370-1573(72)90011-7"},{"key":"6916_CR9","doi-asserted-by":"crossref","unstructured":"E. de Rafael, Hadronic contributions to the muon g-2 and low-energy QCD, Phys. Lett. B 322 (1994) 239 [\nhep-ph\/9311316\n\n] [\nINSPIRE\n\n].","DOI":"10.1016\/0370-2693(94)91114-2"},{"key":"6916_CR10","doi-asserted-by":"crossref","unstructured":"T. Blum, Lattice calculation of the lowest order hadronic contribution to the muon anomalous magnetic moment, Phys. Rev. Lett. 91 (2003) 052001 [\nhep-lat\/0212018\n\n] [\nINSPIRE\n\n].","DOI":"10.1103\/PhysRevLett.91.052001"},{"key":"6916_CR11","volume":"D 85","author":"P Boyle","year":"2012","unstructured":"P. Boyle, L. Del Debbio, E. Kerrane and J. Zanotti, Lattice Determination of the Hadronic Contribution to the Muon g \u2212 2 using Dynamical Domain Wall Fermions, Phys. Rev. D 85 (2012) 074504 [\narXiv:1107.1497\n\n] [\nINSPIRE\n\n].","journal-title":"Phys. Rev."},{"key":"6916_CR12","doi-asserted-by":"crossref","first-page":"055","DOI":"10.1007\/JHEP03(2012)055","volume":"03","author":"M Della Morte","year":"2012","unstructured":"M. Della Morte, B. Jager, A. Juttner and H. Wittig, Towards a precise lattice determination of the leading hadronic contribution to (g \u2212 2)\n \u03bc\n , JHEP 03 (2012) 055 [\narXiv:1112.2894\n\n] [\nINSPIRE\n\n].","journal-title":"JHEP"},{"key":"6916_CR13","unstructured":"ETM collaboration, F. Burger, X. Feng, G. Hotzel, K. Jansen, M. Petschlies and D.B. Renner, Four-Flavour Leading-Order Hadronic Contribution To The Muon Anomalous Magnetic Moment, JHEP 02 (2014) 099 [\narXiv:1308.4327\n\n] [\nINSPIRE\n\n]."},{"key":"6916_CR14","doi-asserted-by":"crossref","unstructured":"HPQCD collaboration, B. Chakraborty et al., Strange and charm quark contributions to the anomalous magnetic moment of the muon, Phys. Rev. D 89 (2014) 114501 [\narXiv:1403.1778\n\n] [\nINSPIRE\n\n].","DOI":"10.1103\/PhysRevD.89.114501"},{"key":"6916_CR15","unstructured":"B. Chakraborty, C. Davies, P.G. de Oliveira, J. Koponen and G.P. Lepage, Lattice calculation of the HVP contribution to the anomalous magnetic moment of muon, \nPoS(LATTICE 2015)108\n\n [\narXiv:1511.05870\n\n] [\nINSPIRE\n\n]."},{"key":"6916_CR16","volume":"D 92","author":"G Bali","year":"2015","unstructured":"G. Bali and G. Endr\u00f6di, Hadronic vacuum polarization and muon g \u2212 2 from magnetic susceptibilities on the lattice, Phys. Rev. D 92 (2015) 054506 [\narXiv:1506.08638\n\n] [\nINSPIRE\n\n].","journal-title":"Phys. Rev."},{"key":"6916_CR17","volume":"D 93","author":"B Chakraborty","year":"2016","unstructured":"B. Chakraborty, C.T.H. Davies, J. Koponen, G.P. Lepage, M.J. Peardon and S.M. Ryan, Estimate of the hadronic vacuum polarization disconnected contribution to the anomalous magnetic moment of the muon from lattice QCD, Phys. Rev. D 93 (2016) 074509 [\narXiv:1512.03270\n\n] [\nINSPIRE\n\n].","journal-title":"Phys. Rev."},{"key":"6916_CR18","doi-asserted-by":"crossref","first-page":"232002","DOI":"10.1103\/PhysRevLett.116.232002","volume":"116","author":"T Blum","year":"2016","unstructured":"T. Blum et al., Calculation of the hadronic vacuum polarization disconnected contribution to the muon anomalous magnetic moment, Phys. Rev. Lett. 116 (2016) 232002 [\narXiv:1512.09054\n\n] [\nINSPIRE\n\n].","journal-title":"Phys. Rev. Lett."},{"key":"6916_CR19","unstructured":"RBC\/UKQCD collaboration, T. Blum et al., Lattice calculation of the leading strange quark-connected contribution to the muon g \u2212 2, JHEP 04 (2016) 063 [Erratum ibid. 05 (2017) 034] [\narXiv:1602.01767\n\n] [\nINSPIRE\n\n]."},{"key":"6916_CR20","volume":"D 96","author":"B Chakraborty","year":"2017","unstructured":"B. Chakraborty, C.T.H. Davies, P.G. de Oliviera, J. Koponen, G.P. Lepage and R.S. Van de Water, The hadronic vacuum polarization contribution to a\n \n \u03bc\n from full lattice QCD, Phys. Rev. D 96 (2017) 034516 [\narXiv:1601.03071\n\n] [\nINSPIRE\n\n].","journal-title":"Phys. Rev."},{"key":"6916_CR21","doi-asserted-by":"crossref","first-page":"020","DOI":"10.1007\/JHEP10(2017)020","volume":"10","author":"M Della Morte","year":"2017","unstructured":"M. Della Morte et al., The hadronic vacuum polarization contribution to the muon g \u2212 2 from lattice QCD, JHEP 10 (2017) 020 [\narXiv:1705.01775\n\n] [\nINSPIRE\n\n].","journal-title":"JHEP"},{"key":"6916_CR22","doi-asserted-by":"crossref","first-page":"222003","DOI":"10.1103\/PhysRevLett.115.222003","volume":"115","author":"J Green","year":"2015","unstructured":"J. Green, O. Gryniuk, G. von Hippel, H.B. Meyer and V. Pascalutsa, Lattice QCD calculation of hadronic light-by-light scattering, Phys. Rev. Lett. 115 (2015) 222003 [\narXiv:1507.01577\n\n] [\nINSPIRE\n\n].","journal-title":"Phys. Rev. Lett."},{"key":"6916_CR23","volume":"D 93","author":"T Blum","year":"2016","unstructured":"T. Blum, N. Christ, M. Hayakawa, T. Izubuchi, L. Jin and C. Lehner, Lattice Calculation of Hadronic Light-by-Light Contribution to the Muon Anomalous Magnetic Moment, Phys. Rev. D 93 (2016) 014503 [\narXiv:1510.07100\n\n] [\nINSPIRE\n\n].","journal-title":"Phys. Rev."},{"key":"6916_CR24","doi-asserted-by":"crossref","first-page":"074","DOI":"10.1007\/JHEP09(2015)074","volume":"09","author":"G Colangelo","year":"2015","unstructured":"G. Colangelo, M. Hoferichter, M. Procura and P. Stoffer, Dispersion relation for hadronic light-by-light scattering: theoretical foundations, JHEP 09 (2015) 074 [\narXiv:1506.01386\n\n] [\nINSPIRE\n\n].","journal-title":"JHEP"},{"key":"6916_CR25","doi-asserted-by":"crossref","first-page":"113","DOI":"10.1007\/JHEP09(2016)113","volume":"09","author":"J Bijnens","year":"2016","unstructured":"J. Bijnens and J. Relefors, Pion light-by-light contributions to the muon g \u2212 2, JHEP 09 (2016) 113 [\narXiv:1608.01454\n\n] [\nINSPIRE\n\n].","journal-title":"JHEP"},{"key":"6916_CR26","doi-asserted-by":"crossref","first-page":"161","DOI":"10.1007\/JHEP04(2017)161","volume":"04","author":"G Colangelo","year":"2017","unstructured":"G. Colangelo, M. Hoferichter, M. Procura and P. Stoffer, Dispersion relation for hadronic light-by-light scattering: two-pion contributions, JHEP 04 (2017) 161 [\narXiv:1702.07347\n\n] [\nINSPIRE\n\n].","journal-title":"JHEP"},{"key":"6916_CR27","doi-asserted-by":"crossref","first-page":"124","DOI":"10.1007\/JHEP04(2012)124","volume":"04","author":"GM Divitiis de","year":"2012","unstructured":"G.M. de Divitiis et al., Isospin breaking effects due to the up-down mass difference in Lattice QCD, JHEP 04 (2012) 124 [\narXiv:1110.6294\n\n] [\nINSPIRE\n\n].","journal-title":"JHEP"},{"key":"6916_CR28","doi-asserted-by":"crossref","unstructured":"RM123 collaboration, G.M. de Divitiis et al., Leading isospin breaking effects on the lattice, Phys. Rev. D 87 (2013) 114505 [\narXiv:1303.4896\n\n] [\nINSPIRE\n\n].","DOI":"10.1103\/PhysRevD.87.114505"},{"key":"6916_CR29","first-page":"114504","volume":"D 95","author":"D Giusti","year":"2017","unstructured":"D. Giusti et al., Leading isospin-breaking corrections to pion, kaon and charmed-meson masses with Twisted-Mass fermions, Phys. Rev. D 95 (2017) 114504 [\narXiv:1704.06561\n\n] [\nINSPIRE\n\n].","journal-title":"Phys. Rev."},{"key":"6916_CR30","unstructured":"V. Lubicz et al., Electromagnetic corrections to the leptonic decay rates of charged pseudoscalar mesons: lattice results, \nPoS(LATTICE2016)290\n\n [\narXiv:1610.09668\n\n] [\nINSPIRE\n\n]."},{"key":"6916_CR31","unstructured":"N. Tantalo, V. Lubicz, G. Martinelli, C.T. Sachrajda, F. Sanfilippo and S. Simula, Electromagnetic corrections to leptonic decay rates of charged pseudoscalar mesons: finite-volume effects, \narXiv:1612.00199\n\n [\nINSPIRE\n\n]."},{"key":"6916_CR32","doi-asserted-by":"crossref","first-page":"153","DOI":"10.1007\/JHEP09(2017)153","volume":"09","author":"P Boyle","year":"2017","unstructured":"P. Boyle et al., Isospin breaking corrections to meson masses and the hadronic vacuum polarization: a comparative study, JHEP 09 (2017) 153 [\narXiv:1706.05293\n\n] [\nINSPIRE\n\n].","journal-title":"JHEP"},{"key":"6916_CR33","doi-asserted-by":"crossref","first-page":"148","DOI":"10.1140\/epja\/i2011-11148-6","volume":"A 47","author":"D Bernecker","year":"2011","unstructured":"D. Bernecker and H.B. Meyer, Vector Correlators in Lattice QCD: Methods and applications, Eur. Phys. J. A 47 (2011) 148 [\narXiv:1107.4388\n\n] [\nINSPIRE\n\n].","journal-title":"Eur. Phys. J."},{"key":"6916_CR34","doi-asserted-by":"crossref","unstructured":"European Twisted Mass collaboration, N. Carrasco et al., Up, down, strange and charm quark masses with N\n \n f\n = 2 + 1 + 1 twisted mass lattice QCD, Nucl. Phys. B 887 (2014) 19 [\narXiv:1403.4504\n\n] [\nINSPIRE\n\n].","DOI":"10.1016\/j.nuclphysb.2014.07.025"},{"key":"6916_CR35","doi-asserted-by":"crossref","first-page":"141","DOI":"10.1016\/0550-3213(85)90606-6","volume":"B 258","author":"Y Iwasaki","year":"1985","unstructured":"Y. Iwasaki, Renormalization Group Analysis of Lattice Theories and Improved Lattice Action: Two-Dimensional Nonlinear O(N ) \u03c3-model, Nucl. Phys. B 258 (1985) 141 [\nINSPIRE\n\n].","journal-title":"Nucl. Phys."},{"key":"6916_CR36","doi-asserted-by":"crossref","unstructured":"Alpha collaboration, R. Frezzotti, P.A. Grassi, S. Sint and P. Weisz, Lattice QCD with a chirally twisted mass term, JHEP 08 (2001) 058 [\nhep-lat\/0101001\n\n] [\nINSPIRE\n\n].","DOI":"10.1088\/1126-6708\/2001\/08\/058"},{"key":"6916_CR37","doi-asserted-by":"crossref","unstructured":"R. Frezzotti and G.C. Rossi, Twisted mass lattice QCD with mass nondegenerate quarks, Nucl. Phys. Proc. Suppl. 128 (2004) 193 [\nhep-lat\/0311008\n\n] [\nINSPIRE\n\n].","DOI":"10.1016\/S0920-5632(03)02477-0"},{"key":"6916_CR38","doi-asserted-by":"crossref","unstructured":"R. Frezzotti and G.C. Rossi, Chirally improving Wilson fermions. 1. O(a) improvement, JHEP 08 (2004) 007 [\nhep-lat\/0306014\n\n] [\nINSPIRE\n\n].","DOI":"10.1088\/1126-6708\/2004\/08\/007"},{"key":"6916_CR39","doi-asserted-by":"crossref","unstructured":"R. Frezzotti and G.C. Rossi, Chirally improving Wilson fermions. 2. Four-quark operators, JHEP 10 (2004) 070 [\nhep-lat\/0407002\n\n] [\nINSPIRE\n\n].","DOI":"10.1088\/1126-6708\/2004\/10\/070"},{"key":"6916_CR40","doi-asserted-by":"crossref","first-page":"440","DOI":"10.1016\/0003-4916(78)90039-8","volume":"110","author":"K Osterwalder","year":"1978","unstructured":"K. Osterwalder and E. Seiler, Gauge Field Theories on the Lattice, Annals Phys. 110 (1978) 440 [\nINSPIRE\n\n].","journal-title":"Annals Phys."},{"key":"6916_CR41","doi-asserted-by":"crossref","first-page":"073","DOI":"10.1007\/JHEP03(2015)073","volume":"03","author":"F Burger","year":"2015","unstructured":"F. Burger, G. Hotzel, K. Jansen and M. Petschlies, The hadronic vacuum polarization and automatic \n\n\n \nO\n\na\n\n\n\n$$ \\mathcal{O}(a) $$\n improvement for twisted mass fermions, JHEP 03 (2015) 073 [\narXiv:1412.0546\n\n] [\nINSPIRE\n\n].","journal-title":"JHEP"},{"key":"6916_CR42","doi-asserted-by":"crossref","unstructured":"K.G. Chetyrkin, J.H. Kuhn and M. Steinhauser, Three loop polarization function and O(\u03b1\n \n \n S\n \n 2\n ) corrections to the production of heavy quarks, Nucl. Phys. B 482 (1996) 213 [\nhep-ph\/9606230\n\n] [\nINSPIRE\n\n].","DOI":"10.1016\/S0550-3213(96)00534-2"},{"key":"6916_CR43","unstructured":"S. Simula et al., QED corrections to meson decay rates in LQCD, PRACE project Pra-102693 [\nhttp:\/\/www.prace-ri.eu\/prace-10th-project-call\/#Fundamental\n\n]."},{"key":"6916_CR44","doi-asserted-by":"crossref","unstructured":"UKQCD collaboration, C. McNeile and C. Michael, Decay width of light quark hybrid meson from the lattice, Phys. Rev. D 73 (2006) 074506 [\nhep-lat\/0603007\n\n] [\nINSPIRE\n\n].","DOI":"10.1103\/PhysRevD.73.074506"},{"key":"6916_CR45","first-page":"114509","volume":"D 94","author":"C Alexandrou","year":"2016","unstructured":"C. Alexandrou, S. Bacchio, J. Finkenrath, A. Frommer, K. Kahl and M. Rottmann, Adaptive Aggregation-based Domain Decomposition Multigrid for Twisted Mass Fermions, Phys. Rev. D 94 (2016) 114509 [\narXiv:1610.02370\n\n] [\nINSPIRE\n\n].","journal-title":"Phys. Rev."},{"key":"6916_CR46","doi-asserted-by":"crossref","first-page":"433","DOI":"10.1016\/0370-2693(83)90987-5","volume":"B 123","author":"G Martinelli","year":"1983","unstructured":"G. Martinelli and Y.-C. Zhang, The Connection Between Local Operators on the Lattice and in the Continuum and Its Relation to Meson Decay Constants, Phys. Lett. B 123 (1983) 433 [\nINSPIRE\n\n].","journal-title":"Phys. Lett."},{"key":"6916_CR47","doi-asserted-by":"crossref","unstructured":"S. Aoki, K.-i. Nagai, Y. Taniguchi and A. Ukawa, Perturbative renormalization factors of bilinear quark operators for improved gluon and quark actions in lattice QCD, Phys. Rev. D 58 (1998) 074505 [\nhep-lat\/9802034\n\n] [\nINSPIRE\n\n].","DOI":"10.1103\/PhysRevD.58.074505"},{"key":"6916_CR48","doi-asserted-by":"crossref","first-page":"413","DOI":"10.1143\/PTP.120.413","volume":"120","author":"M Hayakawa","year":"2008","unstructured":"M. Hayakawa and S. Uno, QED in finite volume and finite size scaling effect on electromagnetic properties of hadrons, Prog. Theor. Phys. 120 (2008) 413 [\narXiv:0804.2044\n\n] [\nINSPIRE\n\n].","journal-title":"Prog. Theor. Phys."},{"key":"6916_CR49","volume":"D 95","author":"V Lubicz","year":"2017","unstructured":"V. Lubicz, G. Martinelli, C.T. Sachrajda, F. Sanfilippo, S. Simula and N. Tantalo, Finite-Volume QED Corrections to Decay Amplitudes in Lattice QCD, Phys. Rev. D 95 (2017) 034504 [\narXiv:1611.08497\n\n] [\nINSPIRE\n\n].","journal-title":"Phys. Rev."},{"key":"6916_CR50","volume":"D 90","author":"Z Davoudi","year":"2014","unstructured":"Z. Davoudi and M.J. Savage, Finite-Volume Electromagnetic Corrections to the Masses of Mesons, Baryons and Nuclei, Phys. Rev. D 90 (2014) 054503 [\narXiv:1402.6741\n\n] [\nINSPIRE\n\n].","journal-title":"Phys. Rev."}],"container-title":"Journal of High Energy Physics","original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/JHEP10(2017)157.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2017,12,3]],"date-time":"2017-12-03T04:34:58Z","timestamp":1512275698000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/JHEP10(2017)157"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017,10]]},"references-count":50,"journal-issue":{"issue":"10","published-print":{"date-parts":[[2017,10]]}},"alternative-id":["6916"],"URL":"http:\/\/dx.doi.org\/10.1007\/JHEP10(2017)157","relation":{},"ISSN":["1029-8479"],"subject":["Nuclear and High Energy Physics"],"container-title-short":"J. High Energ. Phys.","published":{"date-parts":[[2017,10]]},"article-number":"157"} \ No newline at end of file diff --git a/test/testdata/aa5b23bdc9350cee0d1eac7feceeae008638c542.json b/test/testdata/aa5b23bdc9350cee0d1eac7feceeae008638c542.json new file mode 100644 index 00000000..f3de75f3 --- /dev/null +++ b/test/testdata/aa5b23bdc9350cee0d1eac7feceeae008638c542.json @@ -0,0 +1,24 @@ +{ + "encoding": null, + "headers": { + "access-control-allow-headers": "X-Requested-With, Accept, Accept-Encoding, Accept-Charset, Accept-Language, Accept-Ranges, Cache-Control", + "access-control-allow-origin": "*", + "access-control-expose-headers": "Link", + "connection": "close", + "content-encoding": "gzip", + "content-length": "5606", + "content-type": "application/vnd.citationstyles.csl+json", + "date": "Thu, 09 Jun 2022 10:35:32 GMT", + "link": "; rel=\"canonical\", ; version=\"vor\"; type=\"application/pdf\"; rel=\"item\", ; title=\"S. Simula\"; rel=\"author\"", + "permissions-policy": "interest-cohort=()", + "server": "Jetty(9.4.40.v20210413)", + "vary": "Accept, Accept-Encoding", + "x-api-pool": "public", + "x-rate-limit-interval": "1s", + "x-rate-limit-limit": "50", + "x-ratelimit-interval": "1s", + "x-ratelimit-limit": "50" + }, + "status_code": 200, + "url": "https://api.crossref.org/v1/works/10.1007%2FJHEP10%282017%29157/transform" +} \ No newline at end of file diff --git a/test/testdata/abd5ad023a6787477b670d500d29638c83c8d09b.html b/test/testdata/abd5ad023a6787477b670d500d29638c83c8d09b.html new file mode 100644 index 00000000..3b7ab802 --- /dev/null +++ b/test/testdata/abd5ad023a6787477b670d500d29638c83c8d09b.html @@ -0,0 +1,116 @@ +{ + "header": { + "type": "esummary", + "version": "0.3" + }, + "result": { + "uids": [ + "11938998" + ], + "11938998": { + "uid": "11938998", + "pubdate": "1999 Mar 30", + "epubdate": "", + "source": "Wei Sheng Yan Jiu", + "authors": [ + { + "name": "Huang Y", + "authtype": "Author", + "clusterid": "" + }, + { + "name": "Lu J", + "authtype": "Author", + "clusterid": "" + }, + { + "name": "Shen Y", + "authtype": "Author", + "clusterid": "" + }, + { + "name": "Lu J", + "authtype": "Author", + "clusterid": "" + } + ], + "lastauthor": "Lu J", + "title": "[The protective effects of total flavonoids from Lycium Barbarum L. on lipid peroxidation of liver mitochondria and red blood cell in rats].", + "sorttitle": "protective effects of total flavonoids from lycium barbarum l on lipid peroxidation of liver mitochondria and red blood cell in rats", + "volume": "28", + "issue": "2", + "pages": "115-6", + "lang": [ + "chi" + ], + "nlmuniqueid": "9426367", + "issn": "1000-8020", + "essn": "", + "pubtype": [ + "Journal Article" + ], + "recordstatus": "PubMed - indexed for MEDLINE", + "pubstatus": "4", + "articleids": [ + { + "idtype": "pubmed", + "idtypen": 1, + "value": "11938998" + }, + { + "idtype": "rid", + "idtypen": 8, + "value": "11938998" + }, + { + "idtype": "eid", + "idtypen": 8, + "value": "11938998" + } + ], + "history": [ + { + "pubstatus": "pubmed", + "date": "2002/04/10 10:00" + }, + { + "pubstatus": "medline", + "date": "2003/08/22 05:00" + }, + { + "pubstatus": "entrez", + "date": "2002/04/10 10:00" + } + ], + "references": [ + ], + "attributes": [ + "Has Abstract" + ], + "pmcrefcount": 2, + "fulljournalname": "Wei sheng yan jiu = Journal of hygiene research", + "elocationid": "", + "doctype": "citation", + "srccontriblist": [ + ], + "booktitle": "", + "medium": "", + "edition": "", + "publisherlocation": "", + "publishername": "", + "srcdate": "", + "reportnumber": "", + "availablefromurl": "", + "locationlabel": "", + "doccontriblist": [ + ], + "docdate": "", + "bookname": "", + "chapter": "", + "sortpubdate": "1999/03/30 00:00", + "sortfirstauthor": "Huang Y", + "vernaculartitle": "" + } + } +} + diff --git a/test/testdata/abd5ad023a6787477b670d500d29638c83c8d09b.json b/test/testdata/abd5ad023a6787477b670d500d29638c83c8d09b.json new file mode 100644 index 00000000..8d4c0cf3 --- /dev/null +++ b/test/testdata/abd5ad023a6787477b670d500d29638c83c8d09b.json @@ -0,0 +1,26 @@ +{ + "encoding": "UTF-8", + "headers": { + "Access-Control-Allow-Origin": "*", + "Cache-Control": "private", + "Connection": "Keep-Alive", + "Content-Security-Policy": "upgrade-insecure-requests", + "Content-Type": "application/json; charset=UTF-8", + "Date": "Fri, 01 Mar 2019 07:23:11 GMT", + "Keep-Alive": "timeout=4, max=40", + "NCBI-PHID": "D0BD25D15D723FE50000126D8A50BFE6.1.1.m_1", + "NCBI-SID": "9B93DE88652CB826_D490SID", + "Server": "Finatra", + "Set-Cookie": "ncbi_sid=9B93DE88652CB826_D490SID; domain=.nih.gov; path=/; expires=Sun, 01 Mar 2020 07:23:11 GMT", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "Transfer-Encoding": "chunked", + "X-RateLimit-Limit": "3", + "X-RateLimit-Remaining": "3", + "X-UA-Compatible": "IE=Edge", + "X-XSS-Protection": "1; mode=block", + "content-encoding": "gzip", + "l5d-success-class": "1.0" + }, + "status_code": 200, + "url": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?api_key=dad47b304cafdc0790b32d335e3e3a403c08&retmode=json&tool=5j9.citer@github.com&email=dalba.wiki@gmail.com&db=pubmed&id=11938998" +} \ No newline at end of file diff --git a/test/testdata/acfee4001cfe0872b3ef785bc69d9cdb276e42f0.html b/test/testdata/acfee4001cfe0872b3ef785bc69d9cdb276e42f0.html new file mode 100644 index 00000000..8fc10df0 --- /dev/null +++ b/test/testdata/acfee4001cfe0872b3ef785bc69d9cdb276e42f0.html @@ -0,0 +1,1500 @@ + + + + + + + + + + + Live Science: The Most Interesting Articles, Mysteries & Discoveries + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
        + + + + + + + + +
        + + + + + + + + + +
        + +
        +
        + +
        +
        + +
        +
        + + +
        +
        +
        + +
        +
        +
        + + + + + + + + +
        +
        +
        + + + + + + + + + + + + + + + + +
        +
        + + +
        +
        + + +
        +
        +
        +
        +
        +
        + +
        + +
        +

        Follow Us

        + +
        + + + + + + + +
        +
        + +
        +
        + +
        +
        + +
        +

        Live Science Presents

        +

        + + Life's Little Mysteries + +

        +

        + + Our Amazing Planet + +

        +

        + + Expert Voices + +

        +
        +
        + +
        +
        + +
        +
        +
        +
        +
        +
        + + + +
        + +
        + + +
        + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/testdata/acfee4001cfe0872b3ef785bc69d9cdb276e42f0.json b/test/testdata/acfee4001cfe0872b3ef785bc69d9cdb276e42f0.json new file mode 100644 index 00000000..9bf27642 --- /dev/null +++ b/test/testdata/acfee4001cfe0872b3ef785bc69d9cdb276e42f0.json @@ -0,0 +1,20 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "max-age=0, no-cache", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "31383", + "Content-Type": "text/html; charset=UTF-8", + "Date": "Tue, 23 May 2017 17:54:15 GMT", + "Expires": "Tue, 23 May 2017 17:54:15 GMT", + "Pragma": "no-cache", + "Server": "nginx", + "Set-Cookie": "__uzma=59247747456e84.27619097; expires=Fri, 21-May-2027 17:54:15 GMT; Max-Age=315360000; path=/, __uzmd=1495562055; expires=Fri, 21-May-2027 17:54:15 GMT; Max-Age=315360000; path=/, __uzmc=915321032977; expires=Fri, 21-May-2027 17:54:15 GMT; Max-Age=315360000; path=/, __uzmb=1495562055; expires=Fri, 21-May-2027 17:54:15 GMT; Max-Age=315360000; path=/", + "Surrogate-Control": "content=\"ESI/1.0\"", + "Vary": "Accept-Encoding", + "X-Akamai-Transformed": "9 - 0 pmb=mRUM,1" + }, + "status_code": 200, + "url": "http://www.livescience.com/" +} \ No newline at end of file diff --git a/test/testdata/ae36a16a8a95ce5391a00d6ca01e887afe0ddfa7.html b/test/testdata/ae36a16a8a95ce5391a00d6ca01e887afe0ddfa7.html new file mode 100644 index 00000000..db028402 --- /dev/null +++ b/test/testdata/ae36a16a8a95ce5391a00d6ca01e887afe0ddfa7.html @@ -0,0 +1,1510 @@ + + + + + + + Right to Be Forgotten? Not That Easy - The New York Times + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
        + + + + + + + + + +
        + + +
        +
        + + + + +
        + +
        + + + + +
        +
        +
        + Photo +
        + + + +
        +
        + The court decision stems from a case brought by a Spaniard, Mario Costeja González, who was concerned about the prominence given by Google to a short newspaper notice from the 1990s about a house he owned being sold off to pay debts. + + Credit + Vincent West/Reuters +
        +
        + +

        LONDON — Eoin McKeogh knows how hard it can be to make the Internet forget.

        He started waging a court battle against the likes of Google, Facebook and Yahoo after a Dublin cabdriver posted a video in 2011 that showed someone who looked like him — but wasn’t — bailing on his cab fare. Mr. McKeogh, a university student who was in Japan at the time, was pilloried on the Internet after an anonymous user falsely named him as the fare dodger.

        While the original video was taken down long ago, Mr. McKeogh continues to fight in court to expunge the digital trail. He is among the thousands of Europeans trying to erase their online histories.

        In France, a mother recently sought to remove photos of her scantily clad teenage daughter from a website. In Romania, a woman tried to curtail online access to records of her divorce. In Britain, a former politician wanted to delete Google links to a book he viewed as defamatory toward him.

        Such efforts have accelerated after a landmark decision by the European high court this month that will require Google and other search providers to consider individuals’ requests to remove links that they say infringe on their privacy.

        Continue reading the main story +
        +
        +
        +
        +
        + +
        +
        +
        +

        In the first few days after the ruling, about 1,000 Europeans asked Google to take down links, with about half having criminal convictions and half not, according to people briefed on the requests. The requests included an actor seeking to expunge links to articles about an affair with an underage girl and a doctor seeking to take down negative reviews.

        +

        Search companies will face a considerable challenge in responding to the requests. Google alone handled more than 23 million requests in the last month to remove links to copyrighted material around the world. But much of those efforts are automated and address straightforward issues like taking down a link to a stolen movie.

        Dealing with individuals who bring complaints in Europe promises to be more complex because it would most likely require additional employees to grapple with less clear-cut decisions. Google now has a web form for Europeans to request that links be removed. The company also said it plans to create an advisory committee to “cultivate a public conversation about these issues.”

        While the ruling appears to newly enshrine a “right to be forgotten,” Europe has long taken an aggressive stance on individual rights in the digital age. Each nation in the European Union already has a data protection agency through which citizens can appeal for help in erasing their online histories.

        +

        The court decision stems from a case brought by a Spaniard, Mario Costeja González, who was concerned about the prominence given by Google to a short newspaper notice from the 1990s about a house he owned being sold off to pay debts. “I was never worried about my online image, I was worried about the impact on my work,” Mr. Costeja González, a lawyer, said in a brief interview. “I have always been in favor of freedom of expression.”

        +

        But the tech industry has portrayed the decision as a blow against the free flow of information on the web and a victory for those who want to cover up past misdeeds — including pedophiles, corrupt politicians and unscrupulous businesspeople.

        +

        “A simple way of understanding what happened here is that you have a collision between a right to be forgotten and a right to know. From Google’s perspective that’s a balance,” Eric Schmidt, Google’s executive chairman, said in recent comments on the decision. “Google believes, having looked at the decision, which is binding, that the balance that was struck was wrong.”

        Historically, many requests have been aimed at blocking wider access to what many would view as part of the public domain.

        + Photo +
        + + + +
        +
        + Eoin McKeogh is among the thousands of Europeans trying to delete embarrassing events from their online histories. + + Credit + Tom Honan. +
        +
        +

        Indaco Systems, a Romanian company, operates a website that publishes Romanian court proceedings, which are released by the government. The company has received hundreds of complaints this year from citizens who are concerned about public access to court filings that involve them. Many of the complaints are spurred by Google links leading to the case records.

        +

        Adrian Nicolaide, a lawyer for Indaco, said “the information is either public — and in this case anyone should have access to it — or it is not public, and the public should have no free access.”

        +

        “Google indexing official public information leads that information to a whole new level of publicity, but it does not infringe the very purpose of public information,” he added.

        +
        +

        The ruling also reflects the historically divergent views on privacy between the United States and Europe, and it comes alongside deep mistrust of American technology spurred by the revelations about the United States government’s mass surveillance practices.

        The court ruling “echoes what we identify as a social trend, which is the will of the individuals to master their online life,” said Isabelle Falque-Pierrotin, the chairwoman of the French data protection agency. Her agency is already taking in about 2,000 complaints a year from people who want Internet content or links taken down, she said.

        +

        She said the recent ruling was almost immediately cited in complaints coming into her agency.

        “It’s much too early to say it’s going to lead to an automatic increase, but I was surprised that within 24 hours some people who were complaining were mentioning this court ruling,” she said. “Lawyers are very efficient.”

        Once a contested item is online, however, the genie will not easily go back in the bottle.

        In Mr. McKeogh’s case, an Irish judge indicated the taxi video could still be found, and compelled the technology companies to take steps to remove “tags, threads and other means by which the material remains accessible and viewable.”

        +

        “All manner of nasty and seemingly idle minds got to work on the plaintiff, and as seems to happen with apparent impunity nowadays on social media sites, said whatever things first came into their vacant, idle and meddlesome heads,” Judge Michael Peart of Dublin wrote last year, when he granted Mr. McKeogh an injunction in a case.

        Mr. McKeogh’s lawyer declined requests for comment, citing the litigation. The case is now being considered by the Irish Supreme Court.

        +

        Judge Peart, in one of his rulings, noted the complexities of Mr. McKeogh’s quest. “This court does not have a magic wand,” he wrote. “The damage has already been done, and it is impossible to ‘unring’ the bell that has sounded so loudly.”

        +

        Europeans have a long history of trying to reclaim their privacy.

        Consider the case of Alexandre Dumas. In 1867, Dumas, the 65-year-old French author of “The Count of Monte Cristo,” posed for a series of what were seen as racy pictures with Adah Menken, a much younger American actress who was rumored to be his mistress. She posed in her underwear in some of the photos and cuddled with Dumas in others. A scandal followed when some of the pictures were published, and Dumas went to the French courts to try to get them back.

        “Privacy is deeply connected with the protection of personal honor in Europe,” said James Q. Whitman, a Yale law professor who wrote a detailed study contrasting European and American privacy policies.

        +

        “The European understanding is that public dissemination of embarrassing facts about one’s past could undermine one’s sense of honor and standing in society,” he added. “American privacy law isn’t really dedicated in the same way to protecting personal honor or social standing.”

        +

        In 1867, the French courts ruled that a “right to privacy” superseded the photographer’s property rights, and ordered the photos be sold back to Dumas. Still, some of them can be seen today. On the Internet.

        + +
        + Continue reading the main story +
        +
        +
        +
        + + + + + + +
        + + + + + + + + + +
        +
        +
        +
        +

        Go to Home Page »

        +

        + Site Index + + The New York Times + +

        + +
        + + + +
        + + +
        +
        + + + + + + + + + + + + + + + + diff --git a/test/testdata/ae36a16a8a95ce5391a00d6ca01e887afe0ddfa7.json b/test/testdata/ae36a16a8a95ce5391a00d6ca01e887afe0ddfa7.json new file mode 100644 index 00000000..90e416d8 --- /dev/null +++ b/test/testdata/ae36a16a8a95ce5391a00d6ca01e887afe0ddfa7.json @@ -0,0 +1,31 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "0", + "Cache-Control": "no-cache", + "Channels": "NytNow", + "Connection": "close", + "Content-Encoding": "gzip", + "Content-Length": "23044", + "Content-Security-Policy": "default-src data: 'unsafe-inline' 'unsafe-eval' https:; script-src data: 'unsafe-inline' 'unsafe-eval' https: blob:; style-src data: 'unsafe-inline' https:; img-src data: https: blob:; font-src data: https:; connect-src https: wss:; media-src https: blob:; object-src https:; child-src https: data: blob:; form-action https:; block-all-mixed-content;", + "Content-Type": "text/html; charset=utf-8", + "Cteonnt-Length": "101369", + "Date": "Tue, 23 May 2017 17:53:29 GMT", + "Server": "Apache", + "Set-Cookie": "nyt-a=7376628f7049f8d3269d2a6f31dfef6a0421376d1f2b38a86edcb084ea61779b; Expires=Wed, 23 May 2018 17:53:29 GMT; Path=/; Domain=.nytimes.com", + "Vary": "Accept-Encoding, Fastly-SSL", + "X-API-Version": "F-5-5", + "X-Age": "0", + "X-Cache": "MISS", + "X-Cache-Hits": "0", + "X-ESI": "1", + "X-Frame-Options": "DENY", + "X-Origin-Time": "2017-05-23 13:53:29 EDT", + "X-PageType": "article", + "X-Served-By": "cache-iad2640-IAD", + "X-Timer": "S1495562009.943518,VS0,VE310" + }, + "status_code": 200, + "url": "https://www.nytimes.com/2014/05/30/business/international/on-the-internet-the-right-to-forget-vs-the-right-to-know.html?hp&_r=1" +} \ No newline at end of file diff --git a/test/testdata/b1d0bdf1721231505195c290f8331f3f65093c14.html b/test/testdata/b1d0bdf1721231505195c290f8331f3f65093c14.html new file mode 100644 index 00000000..69df1b37 --- /dev/null +++ b/test/testdata/b1d0bdf1721231505195c290f8331f3f65093c14.html @@ -0,0 +1,2585 @@ + + + + + + + + + + InDaily | Adelaide News - Daily Independent News + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
        +
        + + +
        +

        + + InDaily + InDaily + +

        + + + + + + + + + +
        + Support independent Journalism + Donate + Subscribe +
        +
        + + + + + + +
        + +
        +
        + Support independent + journalism +
        + +
        +
        + +
        + + +
        +
        + +
        +
        + +
        + +
        + + + +
        +
        + +
        +
        +
        +
        +
        +
        + + + + News +
        + News +

        New SA border restrictions as Brisbane goes into lockdown

        +

        UPDATED | South Australia will impose border restrictions on travellers arriving from Greater Brisbane from tonight as the hotspot area enters a three-day lockdown, after a cleaner in a quarantine hotel was diagnosed with the highly contagious UK variant of COVID-19 – with another two cases of the strain also detected in Adelaide medi-hotels.

        + + + + +
        +
        +
        + +
        + +
        +
        + +
        +
        +
        + + + + + +
        +
        +
        + + +
        +
        +
        + +
        +
        +
        + + +
        +
        +
        + +
        +
        +
        + + + + +
        + +
        + + + + + + + + + +
        +
        +
        +
        + +
        + +
        + +
        + + +
        +
        + +
        + + +
        + +
        + + + +
        + +
        + +
        +
        + +
        + + + Real Estate + + + Real Estate + +

        11A Phillis St, Maylands

        +

        Custom built, freestanding Torrens titled home with superior finishes and fittings throughout. Prime, tightly held eastern suburbs location – only a short stroll to cafes, restaurants, shops and facilities of The Parade Norwood, Magill Road and less than 3km to the city.

        +
        + +
        +
        + +
        +
        + +
        + +
        +
        +
        + +
        +
        +
        + +
        + +
        + + + +
        + + + + + +
        +
        +
        + +
        +
        +
        + +
        + +
        + + + +
        + +
        + +
        +
        + + + +
        +
        + +
        + +
        +
        + +
        +

        InDaily must-reads

        +
        + +
        +
        + + + +
        +
        + +
        +
        +
        +

        Help our journalists uncover the facts

        +
        +

        In times like these InDaily provides valuable, local independent journalism in South Australia. As a news organisation it offers an alternative to The Advertiser, a different voice and a closer look at what is happening in our city and state for free. Any contribution to help fund our work is appreciated. Please click below to donate to InDaily. +

        +
        + Donate here +
        +
        + + + + + + +
        + + + +
        +
        + + +
        + + + + +
        +
        +
        +
        +
        + + The best local news sent straight to your inbox every workday at lunchtime. +
        +
        + + + +
        +
        +
        +
        +
        + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/testdata/b1d0bdf1721231505195c290f8331f3f65093c14.json b/test/testdata/b1d0bdf1721231505195c290f8331f3f65093c14.json new file mode 100644 index 00000000..dfb10205 --- /dev/null +++ b/test/testdata/b1d0bdf1721231505195c290f8331f3f65093c14.json @@ -0,0 +1,21 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "max-age=600, must-revalidate", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Type": "text/html; charset=UTF-8", + "Date": "Fri, 08 Jan 2021 12:55:28 GMT", + "Keep-Alive": "timeout=20", + "Link": "; rel=\"https://api.w.org/\", ; rel=\"alternate\"; type=\"application/json\", ; rel=shortlink", + "Server": "nginx", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding, Accept-Encoding, Accept-Encoding,Cookie", + "X-Cache": "HIT: 45", + "X-Cache-Group": "normal", + "X-Cacheable": "SHORT", + "X-Powered-By": "WP Engine" + }, + "status_code": 200, + "url": "https://indaily.com.au/" +} \ No newline at end of file diff --git a/test/testdata/b2e2c17a17b007058b190537c85d528283b336f0.html b/test/testdata/b2e2c17a17b007058b190537c85d528283b336f0.html new file mode 100644 index 00000000..87bca951 --- /dev/null +++ b/test/testdata/b2e2c17a17b007058b190537c85d528283b336f0.html @@ -0,0 +1,35 @@ +
        + \ No newline at end of file diff --git a/test/testdata/b2e2c17a17b007058b190537c85d528283b336f0.json b/test/testdata/b2e2c17a17b007058b190537c85d528283b336f0.json new file mode 100644 index 00000000..053a3386 --- /dev/null +++ b/test/testdata/b2e2c17a17b007058b190537c85d528283b336f0.json @@ -0,0 +1,21 @@ +{ + "encoding": "utf-8", + "headers": { + "CF-Cache-Status": "DYNAMIC", + "CF-RAY": "744569ab8be2bb95-FRA", + "Cache-Control": "private, no-cache, no-store, max-age=0, must-revalidate", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Security-Policy": "frame-ancestors 'none'", + "Content-Type": "text/html; charset=utf-8", + "Date": "Fri, 02 Sep 2022 10:02:54 GMT", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=15724800; includeSubDomains", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding", + "X-Frame-Options": "SAMEORIGIN", + "X-Powered-By": "Next.js" + }, + "status_code": 200, + "url": "https://www.worldcat.org/title/99999999999999" +} \ No newline at end of file diff --git a/test/testdata/b5b891f0f0f10f3dc3b0e316d23a60f02f162500.html b/test/testdata/b5b891f0f0f10f3dc3b0e316d23a60f02f162500.html new file mode 100644 index 00000000..6bd03230 --- /dev/null +++ b/test/testdata/b5b891f0f0f10f3dc3b0e316d23a60f02f162500.html @@ -0,0 +1,5545 @@ + + + + + + + + + + + + + + + + + + + + + +The Telegraph - Telegraph Online, Daily Telegraph, Sunday Telegraph - Telegraph + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
        +
        +
        + +
        +
        + +The Telegraph + +
        +
        + +
        +
        + +
        +
        + +
        +
        +
        +
        +
        +
        +
        +
        + +
        +
        + +
        +
        + +
        +
        + +
        +
        + +The Telegraph + +
        +
        +
        +
        + +
        +
        + +
        + +
        +
        +
        +
        +
        + + +
        +
        +
        +
        +
        +
        +
        + + +
        +
        +
        +
        + +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        + +

        Opinion

        +
        +
        +Hide +
        +
        +
        +
        +
        +
          +
        1. + +
        2. +
        3. + +
        4. +
        5. + +
        6. +
        7. + +
        8. +
        9. + +
        10. +
        11. + +
        12. +
        13. + +
        14. +
        + +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        + + + + + +Premium - Elections + + + + +
        +
        +
        +
        +

        ‘‘Theresa May isn't attacking pensioners. She's tackling welfarism’’

        Juliet Samuel
        +

        ‘‘ +The elephantine Tory poll lead has slipped a bit? Good. A scare can be a +tonic’’

        Tim Stanley
        +

        ‘‘Labour will be ungovernable after June 8. Its moderates must split or die’’

        Simon Heffer
        +

        ‘‘Honesty is the best policy – and it is only the Conservative manifesto that offers it’’

        William Hague
        +

        ‘‘Theresa May isn't attacking pensioners. She's tackling welfarism’’

        Juliet Samuel
        +
        +
        Expert political insight. Free for 30 days.
        + +
        +
        +
        +
        +
        +
        +
        +
        +
        + +
        +
        + +
        +
        +
        + + +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        + +
        +
        +
        + +
        +
        + +
        +
        +
        +
        +
        +
        +
        +
        +
        + +
        +
        +
        +
        +
        +
        +
        + + +
        +
        +
        +
        +
        +
        + +
        +
        +
        +
        +
        +
        +
        +
        +

        Most Read

        +
        +
        +
        +
        +
        +
        +
        +
        +
        + +
        +
        +
        +
        +
        +
        +
        +

        Recommended +

        +
        +
        +
          +
        1. + +
        2. +
        3. + +
        4. +
        5. + +
        6. +
        7. + +
        8. +
        9. + +
        10. +
        11. + +
        12. + + + + +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        + + +
        +
        +
        +
        +
        + +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +

        Money +

        +
        +
        +
          +
        1. + +
        2. +
        3. + +
        4. +
        5. + +
        6. +
        7. + +
        8. +
        9. + +
        10. +
        +
        +
        +
        +
        + +
        +
        +
        +
        +
        +
        + +
        +
        +
        +
        +
        + +
        +
        +
        +
        +
        +
        +
        +
        + +
        +
        +
        +
        +
        +

        Hotels +

        +
        +
        +
          + + + + + + + + +
        1. + +
        2. +
        3. + +
        4. +
        5. + +
        6. +
        7. + +
        8. +
        9. + +
        10. +
        11. + +
        12. +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +

        Culture +

        +
        +
        +
          + + + + + + + +
        1. + +
        2. +
        3. + +
        4. +
        5. + +
        6. +
        7. + +
        8. + + + + + + + + + + + + + + + +
        +
        +
        +
        +
        +
        +
        +
        + +
        +
        +
        +
        +
        +
        +
        +
        +
        + + +
        +
        +
        +
        +
        +
        + +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        + + +
        +
        Loading...
        +
        +
        Sponsored
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +

        Women +

        +
        +
        +
          +
        1. + +
        2. +
        3. + +
        4. +
        5. + +
        6. +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +

        Men +

        +
        +
        +
          +
        1. + +
        2. +
        3. + +
        4. +
        5. + +
        6. +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        + +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        + +
        +
          +
        1. + +
        2. +
        3. + +
        4. +
        5. + +
        6. +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        + +
        +
        +
        +
        + + +
        +
        +
        +
        +
        +
        +

        Cars +

        +
        +
        +
          +
        1. + +
        2. +
        3. + +
        4. +
        5. + +
        6. +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        + + +
        +
        +
        +
        +
        +
        +

        Brexit +

        +
        + +
        +
        +
        +
        +
        +
        + +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +
        +

        Sponsored

        +
        +
        +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + +
          +
          +
          +
          + + + + + + + + + + + + + diff --git a/test/testdata/b5b891f0f0f10f3dc3b0e316d23a60f02f162500.json b/test/testdata/b5b891f0f0f10f3dc3b0e316d23a60f02f162500.json new file mode 100644 index 00000000..41637eff --- /dev/null +++ b/test/testdata/b5b891f0f0f10f3dc3b0e316d23a60f02f162500.json @@ -0,0 +1,17 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "max-age=1", + "Connection": "keep-alive, Transfer-Encoding", + "Content-Encoding": "gzip", + "Content-Security-Policy": "frame-ancestors 'self' www.stumbleupon.com stumbleupon.com;", + "Content-Type": "text/html; charset=UTF-8", + "Date": "Tue, 23 May 2017 17:53:16 GMT", + "Server": "nginx", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding", + "X-Mod-Pagespeed": "Powered By mod_pagespeed" + }, + "status_code": 200, + "url": "http://www.telegraph.co.uk/" +} \ No newline at end of file diff --git a/test/testdata/b65411c1a9455bddc6c31d9139fba3b59ffbddbb.html b/test/testdata/b65411c1a9455bddc6c31d9139fba3b59ffbddbb.html new file mode 100644 index 00000000..031e070b --- /dev/null +++ b/test/testdata/b65411c1a9455bddc6c31d9139fba3b59ffbddbb.html @@ -0,0 +1,672 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + بررسی فضایل قرآنی در دعای ابوحمزه ثمالی - پایگاه مجلات تخصصی نور + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Skip to main content + + + + + + +
          + + + + +
          + + + +
          +
          + + + + + + + +
          +
          + + + + + +
          +
          + فهرست مقالات +
          + +

          + بررسی فضایل قرآنی در دعای ابوحمزه ثمالی +

          +
          +
          + + + + + +
          +
          + +
          +

          + نویسنده: + +

          +
          +

          + + + (22 صفحه - از 103 تا 124) +

          +
          + +
          + + + +
          +
          + + + + + + +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + +
          +
          +
          + +
          +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +

          + +

          +
          +
          +
          +
          + + + + +
          + +
          + +
          +
          +
            +
          • +
            + دانلود HTML +
            + +
          • +
          • +
            + دانلود PDF +
            + +
          • +
          +
          +
          + +
          + +
          + +

          + برای مشاهده محتوای مقاله لازم است وارد پایگاه شوید. در صورتی که عضو نیستید از قسمت عضویت اقدام فرمایید. +

          +
          +
          +
          + + + + cloob + + + + + + + +
          +
          + + +
          +
          +
          + + + + + + + +
          + + + + + + + + + + + + + + + + + + + +
          + + + + + + + + + + + +
          + + +
          + + + +
          + + +
          + + diff --git a/test/testdata/b65411c1a9455bddc6c31d9139fba3b59ffbddbb.json b/test/testdata/b65411c1a9455bddc6c31d9139fba3b59ffbddbb.json new file mode 100644 index 00000000..e772c3e7 --- /dev/null +++ b/test/testdata/b65411c1a9455bddc6c31d9139fba3b59ffbddbb.json @@ -0,0 +1,15 @@ +{ + "encoding": "utf-8", + "headers": { + "Cache-Control": "private", + "Content-Encoding": "gzip", + "Content-Length": "10875", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:51:57 GMT", + "Set-Cookie": "__RequestVerificationToken=k9WRxDxfEhe4qUmhNL_6sa1F6fdtg-1zaAw-XVghePBjCDe0MqEcW8YxXygKWDO7X4c8p8WfjREpPOI7otDnrTmmiLK3Ghvo1zIjAjOl6yA1; path=/; HttpOnly, .ASPXBrowserOverride=Mozilla%2f4.0+(compatible%3b+MSIE+6.0%3b+Windows+CE%3b+IEMobile+8.12%3b+MSIEMobile+6.0); expires=Tue, 30-May-2017 17:51:57 GMT; path=/", + "X-Download-Options": "noopen", + "X-Frame-Options": "SAMEORIGIN" + }, + "status_code": 200, + "url": "http://www.noormags.ir/view/fa/articlepage/692447/%d8%a8%d8%b1%d8%b1%d8%b3%db%8c-%d9%81%d8%b6%d8%a7%db%8c%d9%84-%d9%82%d8%b1%d8%a2%d9%86%db%8c-%d8%af%d8%b1-%d8%af%d8%b9%d8%a7%db%8c-%d8%a7%d8%a8%d9%88%d8%ad%d9%85%d8%b2%d9%87-%d8%ab%d9%85%d8%a7%d9%84%db%8c?sta=%D8%AF%D8%B9%D8%A7%DB%8C%20%D8%A7%D8%A8%D9%88%D8%AD%D9%85%D8%B2%D9%87%20%D8%AB%D9%85%D8%A7%D9%84%DB%8C" +} \ No newline at end of file diff --git a/test/testdata/b66d034bd6b396c02941dcec4f46b7868ec12949.html b/test/testdata/b66d034bd6b396c02941dcec4f46b7868ec12949.html new file mode 100644 index 00000000..ec8a078e --- /dev/null +++ b/test/testdata/b66d034bd6b396c02941dcec4f46b7868ec12949.html @@ -0,0 +1,959 @@ + + + + + + + The Investment column: TT Group | The Independent + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + + + + +
            + +
          1. + + +
          2. + +
          3. + +
          4. +
          + + +
          +
          +
          +
          +
          +
          +
          + +
          + + +
          + +
          + + + +
          + + + +
          + + +

          The Investment column: TT Group

          + + +
            +
          • +
          • + +
          • +
          +
          +
          + +
          Click to follow
          The Independent Online
          +
          +
          + + + + +
          +
          +
          + TT GROUP is a barometer for UK manufacturing and despite the optimism of the Confederation of British Industry, the immediate situation is dire.

          The group's shares have lagged those of its peers since a failed bid for Hall engineering. A negative trading statement knocked a further 15 per cent off the shares yesterday, prompting executive chairman John Newman to spend almost pounds 800,000 buying into the stock. Has he snapped up a bargain?

          +

          TT supplies fastners and climatic controls for Rover Group. In common with Rover, it is complaining about the strength of sterling. The exchange rate has also eroded margins in its glass bottle manufacturing operation, which has been hit by European imports and the growing popularity of plastic.

          +

          The investment argument for TT depends on the prospects for the pound- euro rate. Meanwhile, the group is looking to make acquisitions to enable it to offer a wider range of products to its customers.

          +

          TT warns demand will be flat this year. But in the businesses where margins are suffering, TT still achieves returns on capital of 40 per cent. Housebroker DKB shaved pretax profits forecasts for this year down almost 30 per cent to pounds 40m. On expectations of earnings of 17.3p per share this year, the shares, which closed down 24.5p at 133p, trade on a forward p/e of 7.8. That's an unjustified discount to their peers given the possibility of a turnaround.

          +
          + +
          + +
          +
          + +
          +
          +
          +
          + + + +
          +
          +
          +
          +
          +
          +
          +
          + +
          +
          + +
          +

          Comments

          +
          + +
          + +
          + +
          + +
          +
          + +
          +
          + + + +
          + +
          +
          +
          + + +
          + + + +
          + + + + + + + + + +
          + + + + + + + + + + +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/testdata/b66d034bd6b396c02941dcec4f46b7868ec12949.json b/test/testdata/b66d034bd6b396c02941dcec4f46b7868ec12949.json new file mode 100644 index 00000000..e6003584 --- /dev/null +++ b/test/testdata/b66d034bd6b396c02941dcec4f46b7868ec12949.json @@ -0,0 +1,34 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "0", + "Cache-control": "no-cache, no-store, max-age=0, must-revalidate", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Language": "en", + "Content-Length": "14382", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:54:07 GMT", + "Etag": "\"1495562046-1\"", + "Expires": "Sun, 19 Nov 1978 05:00:00 GMT", + "Last-modified": "Wed, 21 Sep 2011 16:49:58 GMT", + "Link": "; rel=\"canonical\",; rel=\"shortcut icon\",; rel=\"apple-touch-icon\",; rel=\"apple-touch-icon_72x72\",; rel=\"apple-touch-icon_76x76\",; rel=\"apple-touch-icon_114x114\",; rel=\"apple-touch-icon_120x120\",; rel=\"apple-touch-icon_144x144\",; rel=\"apple-touch-icon_152x152\",; rel=\"apple-touch-icon_180x180\",; rel=\"android-icon_36x36\",; rel=\"android-icon_48x48\",; rel=\"android-icon_72x72\",; rel=\"android-icon_96x96\",; rel=\"android-icon_114x114\",; rel=\"android-icon_192x192\",; rel=\"ms-icon_70x70\",; rel=\"ms-icon_144x144\",; rel=\"ms-icon_150x150\",; rel=\"ms-icon_310x310\"", + "Server": "nginx", + "Set-Cookie": "Locale=US", + "Vary": "Accept-Encoding, Locale, ines_tg", + "Via": "1.1 varnish-v4, 1.1 varnish", + "X-AH-Environment": "prod", + "X-Cache": "MISS, MISS", + "X-Cache-Hits": "0", + "X-Content-Type-Options": "nosniff", + "X-Drupal-Cache": "MISS", + "X-Frame-Options": "SAMEORIGIN", + "X-Generator": "Drupal 7 (http://drupal.org)", + "X-Request-ID": "v-d0ade038-3fe0-11e7-83f9-22000b0a13de", + "X-Served-By": "cache-iad2633-IAD", + "X-Timer": "S1495562046.399582,VS0,VE836" + }, + "status_code": 200, + "url": "http://www.independent.co.uk/news/business/the-investment-column-tt-group-1103208.html" +} \ No newline at end of file diff --git a/test/testdata/b7fb32973954a902dea7574e5dd79bf4299b5a9f.html b/test/testdata/b7fb32973954a902dea7574e5dd79bf4299b5a9f.html new file mode 100644 index 00000000..1179ff69 --- /dev/null +++ b/test/testdata/b7fb32973954a902dea7574e5dd79bf4299b5a9f.html @@ -0,0 +1,628 @@ + + + + + + + + + + + + + + + + + + Google from Protecting the Vote: How Internet Platforms Are Addressing Election and Voter Suppression-Related Misinformation and Disinformation on JSTOR + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + + + + +
          + +
          + + + + + + + + + + + +
          +
          + + + + + + Have library access? + + Log in through your library + + + + + + + +
          + + +
          + + + + + + +
          +
          + + +
          + +
          + +
          +
          + +
          +
          + +
          +
          + + + + + + + + + + + + + + + + + + + + + + + + +
          + +
          + + + + + + + + + + + + + + + + + + + + + + + + + +
          + +
          + + + + + + + + + diff --git a/test/testdata/b7fb32973954a902dea7574e5dd79bf4299b5a9f.json b/test/testdata/b7fb32973954a902dea7574e5dd79bf4299b5a9f.json new file mode 100644 index 00000000..fd60e4c6 --- /dev/null +++ b/test/testdata/b7fb32973954a902dea7574e5dd79bf4299b5a9f.json @@ -0,0 +1,21 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Type": "text/html; charset=utf-8", + "Date": "Fri, 04 Jun 2021 11:13:20 GMT", + "Server": "Apache/2.4.29 (Ubuntu)", + "Set-Cookie": "AccessSession=H4sIAAAAAAAAAK1Sy27bMBC85ysEncOAS4ribm-yDTdFj8mtKIrlK1GhJIYlBWiD_Hv1sg3ncSvAC2dmB8MdvlxkWV6HPPuS5YVVxrDTpUuxcOQdOgpWFjqQZ51CfjmK_aKu2QtmBjnDvxe42q6wrFagtmDR4LbaSL2WFssNbCpVmlm9X9Q2WOaAEp2mwjlk7QgDWo-BkgSY1X2_yB0HijIpEag0omDnBHqNAsAkG8FbVcZ5hPvufhxJ3LRxQp65mV2gVArBEBFSMVH1bnLXcAXFcOwV2MUlPNSP5zZt-3QOdP78zt537Qj9yF6G-2m9IKWkyXfAmsPOD0BcABr2oDUZkYiHJ5KSApGjUBQSAvqkAx9muj-7OA193T_1u6P1EV5xW_szbnhO_VD_jduG70ZJt-_jwLxefpBVv82q3mWNKG2IXiRtnSi8NYI9KiE94JDeO8ny_2fNfs5FdMcypYFTmQ1_RvSfENx1-6mxZQfr9fyTbw5J1rfz76sfuenDr1OLNzPx7fa6-l7lY7iL13-NZ1r4UgMAAA; Path=/; SameSite=Lax; Secure, AccessSessionSignature=4d41748bf4ea622a15be5bddce5207e167f44f30c79cb8eb18e9ff6b146dcf24; Path=/; SameSite=Lax; Secure, AccessSessionTimedSignature=366b26f6d8a1fbfebe86bf6450035cc5ce58a825abdc3b7ccbc8209466fc6746; Path=/; SameSite=Lax; Secure, UUID=bad9e0f2-d965-4abb-8c38-115f7e1c726e; expires=Mon, 03 Jun 2024 11:13:20 GMT; Max-Age=94608000; Path=/; SameSite=None; Secure, csrftoken=PikBkxtjgrI7TMfXsTLqATSfz1uD9YUNHikCGaYP8Ff1OEaSWPYneQaudGVow8rU; expires=Fri, 03 Jun 2022 11:13:20 GMT; Max-Age=31449600; Path=/; SameSite=Lax; Secure, ReferringRequestId=excelsior:c0501db5496d7b1f50a9a6a0d4dcbb36; Path=/; SameSite=Lax; Secure, _pxhd=2787b83c569675677186be3beb6a32464475d097b94fcf07fcbaf8144d2c895d:ddfb0710-c525-11eb-8228-dd6017db91b4; Expires=Fri, 01 Jan 2021 00:00:00 GMT; path=/;", + "Vary": "Cookie,Accept-Encoding,Fastly-SSL,Origin,X-Requested-Host", + "Via": "1.1 varnish", + "X-Cache": "MISS", + "X-Cache-Hits": "0", + "X-JSTOR-Restarts": "2", + "X-Served-By": "cache-fra19149-FRA", + "transfer-encoding": "chunked" + }, + "status_code": 200, + "url": "https://www.jstor.org/stable/resrep26363.7?Search=yes&resultItemClick=true&searchText=google&searchUri=%2Faction%2FdoBasicSearch%3FQuery%3Dgoogle%26acc%3Doff%26wc%3Don%26fc%3Doff%26group%3Dnone%26refreqid%3Dsearch%253A2e627536469ca8786b576957a9797d56&ab_segments=0%2Fbasic_search_gsv2%2Fcontrol&refreqid=fastly-default%3Af90c911269c590baf37330b9d16ae1cd&seq=1#metadata_info_tab_contents" +} \ No newline at end of file diff --git a/test/testdata/b821fac310c23e1bb0317ae480b317ca5c0ed751.html b/test/testdata/b821fac310c23e1bb0317ae480b317ca5c0ed751.html new file mode 100644 index 00000000..1c3db6ab --- /dev/null +++ b/test/testdata/b821fac310c23e1bb0317ae480b317ca5c0ed751.html @@ -0,0 +1,1495 @@ + + + + + + + $2 Billion for Clippers? In Time, It May Be a Steal for Steve Ballmer - The New York Times + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + + + + + + + + +
          + + +
          +
          + + + + +
          + +
          + + + + +
          +
          +
          + Photo +
          + + + +
          +
          + Steve Ballmer, left, and N.B.A. Commissioner Adam Silver at a Los Angeles Clippers playoff game on May 11. + + Credit + Noel Vasquez/GC Images +
          +
          + +

          Men with vast fortunes who have bought professional teams as toys are not uncommon. Whether they considered the franchises sound investments is almost beside the point.

          The latest major player in the business is Steve Ballmer, the former Microsoft chief executive, who agreed on Thursday to pay $2 billion for the Los Angeles Clippers — nearly four times as much as the previous record price for an N.B.A. franchise.

          On Friday, the league said that in light of the sale, it would withdraw its pending charges against the current owner, Donald Sterling.

          Could the team possibly be worth that much, fans immediately wondered — or is Mr. Ballmer, whose net worth is estimated at $19 billion, simply indulgent?

          Continue reading the main story +
          +
          +
          +
          +
          + +
          +
          +
          +

          In time, the $2 billion bid that seems shocking today may be viewed as a relative bargain.

          There was a time when $10 million was considered an outrageous sum to buy the Yankees. It was 1973, the Bronx was crumbling, and George Steinbrenner’s purchase (for about $50 million in today’s dollars) made jaws drop. Jerry Jones’s $140 million outlay for the Dallas Cowboys in 1989 had the same effect, as did the 2012 sale of the Los Angeles Dodgers for $2.15 billion.

          +

          Mr. Ballmer, like Mr. Steinbrenner, Mr. Jones and others before him, may be betting that sports will continue to be a growth industry, bringing expanding revenue from broadcast rights, ticket sales and sponsorship deals.

          “Everyone looking at this is looking at the future of the N.B.A. and the upcoming TV deals,” said Sal Galatioto, the president of Galatioto Sports Partners. He added, “It does significantly boost the price of large-market N.B.A. teams, and all N.B.A. teams.”

          The Clippers are the 13th-most valuable franchise in the N.B.A., according to calculations by Forbes, which estimated that the team generated $128 million in revenue last year. Nearly 40 percent of that came from fees for television rights — the driving force for major sports deals these days.

          +

          The deal for the Dodgers, for example, was largely predicated on the prospect that the new owners — investors from Guggenheim Partners — could set up a local sports network. They did that last year, in an $8 billion, 25-year arrangement with Time Warner Cable.

          +

          The Clippers’ local cable television contract, for $18 million a year, is nearing renewal, and projections suggest that the team could get as much as $60 million a year. Mr. Ballmer would also benefit from the N.B.A.'s next round of national television deals, which begins in the 2016-17 season. The teams are expected to receive substantially more than the $30 million a year each one currently gets.

          + Photo +
          + + + +
          +
          + T-shirts on seats at Staples Center in Los Angeles before an opening-round playoff game between the Clippers and the Warriors. + + Credit + Mark J. Terrill/Associated Press +
          +
          +

          The Clippers are a scarce asset, another factor that might have enticed Mr. Ballmer. Clubs in big-market cities like New York and Los Angeles, and cornerstone franchises like the New England Patriots and the Dallas Cowboys, are in position to generate significantly higher bids because there are so few available.

          +

          The Clippers are not a cornerstone team, but they are in Los Angeles. And after decades as a doormat, they are on the upswing, with stars like Chris Paul and Blake Griffin. Their more glamorous local rivals, the Lakers, are down.

          Assuming his purchase is approved by the N.B.A., Mr. Ballmer, 58, is likely to enjoy significant personal financial benefits. When an investor purchases a sports team, he can attribute a large part of the purchase price to the player contracts he is acquiring. As those contracts expire, their depreciation can offset income.

          +
          +

          Given how low interest rates are, he could finance part of the purchase of the team relatively inexpensively, and he could later bring in minority shareholders to recoup some of his purchase.

          +

          Still, there are no guarantees that the Clippers will make a profit for Mr. Ballmer. Rob Tilliss, who runs Inner Circle Sports, which advised one of the other bidders, said that there were reasons to be optimistic about the Clippers’ financial outlook but that the $2 billion offer was based on wishful assumptions that would all have to come true for Mr. Ballmer to get his money back.

          +

          “If you believe in the growth of the league, you believe in the TV rights renewals and you want to be the big guy in L.A., it makes sense,” Mr. Tilliss said. “But I can’t make the economic argument for you.”

          The bid was three and a half times the amount the team was recently valued at by Forbes. It was also 20 percent more than the next closest offer. Some analysts said Mr. Ballmer might have purposely submitted a bid that would far surpass those of the other contenders — including a group with Oprah Winfrey — so he could swiftly end the auction and win the support of Rochelle Sterling, who co-owns the Clippers with her husband.

          +

          Mr. Ballmer took a similar approach when he ran Microsoft, paying what some analysts thought were obscene amounts for companies. In 2011, for example, Microsoft paid $8.5 billion for Skype, more than tripling what eBay had paid for the company several years earlier.

          The team Mr. Ballmer is hoping to buy has never made it past the second round of the N.B.A. playoffs and does not have its own arena, a significant financial handicap.

          +

          Mr. Ballmer made one previous attempt at buying a stake in an N.B.A. team. Last year, his group’s bid to buy the Sacramento Kings failed; the group had planned to move them to Seattle. Now he stands to play the role of savior and could lobby from within the league to expand to Seattle.

          +

          “This comes on the heels of Ballmer going through a wretched fight for the Kings, and he’s leaving Microsoft and wondering about what to do next,” said Marc Ganis, who advises owners and potential owners. “Now he’ll be a hero for stepping up to take over a franchise that the nation wants taken away from Donald Sterling. He will ride in on his shiny steed.”

          + +
          + Continue reading the main story +
          +
          +
          +
          + + + + + + +
          + + + + + + + + + +
          +
          +
          +
          +

          Go to Home Page »

          +

          + Site Index + + The New York Times + +

          + +
          + + + +
          + + +
          +
          + + + + + + + + + + + + + + + + diff --git a/test/testdata/b821fac310c23e1bb0317ae480b317ca5c0ed751.json b/test/testdata/b821fac310c23e1bb0317ae480b317ca5c0ed751.json new file mode 100644 index 00000000..1b313d12 --- /dev/null +++ b/test/testdata/b821fac310c23e1bb0317ae480b317ca5c0ed751.json @@ -0,0 +1,31 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "0", + "Cache-Control": "no-cache", + "Channels": "NytNow", + "Connection": "close", + "Content-Encoding": "gzip", + "Content-Length": "21922", + "Content-Security-Policy": "default-src data: 'unsafe-inline' 'unsafe-eval' https:; script-src data: 'unsafe-inline' 'unsafe-eval' https: blob:; style-src data: 'unsafe-inline' https:; img-src data: https: blob:; font-src data: https:; connect-src https: wss:; media-src https: blob:; object-src https:; child-src https: data: blob:; form-action https:; block-all-mixed-content;", + "Content-Type": "text/html; charset=utf-8", + "Cteonnt-Length": "98386", + "Date": "Tue, 23 May 2017 17:53:32 GMT", + "Server": "Apache", + "Set-Cookie": "nyt-a=00fd2f841ed60bca239c76764336fd5d4d697c5bd4747fc39598f66d69da0cd6; Expires=Wed, 23 May 2018 17:53:32 GMT; Path=/; Domain=.nytimes.com", + "Vary": "Accept-Encoding, Fastly-SSL", + "X-API-Version": "F-5-5", + "X-Age": "0", + "X-Cache": "MISS", + "X-Cache-Hits": "0", + "X-ESI": "1", + "X-Frame-Options": "DENY", + "X-Origin-Time": "2017-05-23 13:53:32 EDT", + "X-PageType": "article", + "X-Served-By": "cache-iad2620-IAD", + "X-Timer": "S1495562012.460456,VS0,VE473" + }, + "status_code": 200, + "url": "https://www.nytimes.com/2014/05/31/sports/basketball/steven-a-ballmers-2-billion-play-for-clippers-is-a-big-bet-on-the-nba.html?hp&_r=0" +} \ No newline at end of file diff --git a/test/testdata/b92f7b63b5dffb6446be8c2b81edf79fcc9333e1.html b/test/testdata/b92f7b63b5dffb6446be8c2b81edf79fcc9333e1.html new file mode 100644 index 00000000..1a395fab --- /dev/null +++ b/test/testdata/b92f7b63b5dffb6446be8c2b81edf79fcc9333e1.html @@ -0,0 +1,19 @@ + +TY - JOUR +T1 - تحلیل منافع بهره وری ناشی از اصلاحات صنعت برق استرالیا: چارچوب های روش شناختی +T2 - مطالعات اقتصاد انرژی +JF - مطالعات اقتصاد انرژی +Y1 - 1383/// + +LA - fa +UR - http://www.noormags.ir/view/fa/articlepage/105489 +SP - 55 +EP - 55 +SN - +VL - 3 +IS - 1 + +ID - 105489 +AU - فتح‌الله‌زاده‌اقدم,‌رضا + +ER - diff --git a/test/testdata/b92f7b63b5dffb6446be8c2b81edf79fcc9333e1.json b/test/testdata/b92f7b63b5dffb6446be8c2b81edf79fcc9333e1.json new file mode 100644 index 00000000..419b4672 --- /dev/null +++ b/test/testdata/b92f7b63b5dffb6446be8c2b81edf79fcc9333e1.json @@ -0,0 +1,15 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "private", + "Content-Disposition": "attachment; filename=\"noormags-105489.ris\"", + "Content-Length": "467", + "Content-Type": "application/x-Research-Info-Systems; charset=UTF-8", + "Date": "Tue, 23 May 2017 17:51:56 GMT", + "Set-Cookie": "CRCIS_SessionId=41lxxqyybob3daxytiempmg0; path=/, .ASPXBrowserOverride=Mozilla%2f4.0+(compatible%3b+MSIE+6.0%3b+Windows+CE%3b+IEMobile+8.12%3b+MSIEMobile+6.0); expires=Tue, 30-May-2017 17:51:56 GMT; path=/", + "X-Download-Options": "noopen", + "X-Frame-Options": "SAMEORIGIN" + }, + "status_code": 200, + "url": "http://www.noormags.ir/view/fa/citation/ris/105489" +} \ No newline at end of file diff --git a/test/testdata/b97779a7fd47ca9d7bd5706c087514e8b8dde133.html b/test/testdata/b97779a7fd47ca9d7bd5706c087514e8b8dde133.html new file mode 100644 index 00000000..049361cb --- /dev/null +++ b/test/testdata/b97779a7fd47ca9d7bd5706c087514e8b8dde133.html @@ -0,0 +1,2964 @@ + + + + + + + + + + + + + +Marine 'collapse' linked to whale decline - Telegraph + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          +
          + + + + + + + + + + + + + + + + + + + + +
          + + + + + + + + + + +
          + + +
          +
          + +
          Telegraph.co.uk
          +
          + +
          +
          +
          +

          + Tuesday 23 May 2017

          +
          + +
          +
          + +
          +
          +
          +
          + + + +
          +
          + +
          +
            +
          +
          +
          +
          +
          + + + +
          +
          +
          Advertisement
          +
          + +
          + + + + + + +
          +
          +
          +
            +
          1. Home»
          2. +
          3. News»
          4. +
          5. Science»
          6. +
          7. Science News
          +
          +
          +
          + + +
          +
          + +
          +
          + + + +
          + +

          Marine 'collapse' linked to whale decline

          + + + +
          +
          +
          +
          + +
          + +

          A "domino effect" that links the collapse of seal, sea lion and sea otter populations to the demise of great whales has been identified by scientists.

          Overfishing of whales in the North Pacific triggered "one of the longest and most complex ecological chain reactions ever," accoriding to the study published in the Proceedings of the National Academy of Sciences.

          Lead author, Dr Alan Springer of the University of Alaska, Fairbanks, and his co-authors said the trigger was the capture of hundreds of thousands of great whales from the North Pacific from 1946 to 1979.

          This forced killer whales to seek alternative sources of food: beginning with harbour seals (populations collapsed early 70s - early 80s) then fur seals (mid 70s - mid 80s), sea lions (late 70s - 90s) and, finally, sea otters (90s - today). By the late 1990s, low numbers of sea otters allowed an explosion of sea urchins, which destroyed kelp forests.

          Co-author Dr Jim Estes of the University of California, Santa Cruz, said: "In principle, we think that when any species is exploited to excess - be it pollock, halibut or whales - it may trigger a broad and devastating 'domino effect."'

          +
          +
          + +
          +
          + +
          +
          + +
          + + +
          +
          + +
          +
          + + +
          +

          In Science News

          +
          +
          +
          + +
          + A combination photograph shows the beginning (top L) to the end (top L to bottom R) of a total solar eclipse as seen from the beach of Ternate island, Indonesia +   +
          + +

          + Total eclipse, in pictures +

          + +
          +
          +
          +
          + +
          + +   +
          + +

          + Scott Kelly returns to Earth +

          + +
          +
          + +
          +
          + +
          + An astonishing image of a pregnant pony uterus has been selected as the overall winner for the 2015 Wellcome Image Awards. The photograph was taken by Michael Frank, and is of an historic specimen from the Lanyon Anatomy Museum of the Royal Veterinary College in London. It shows the preserved uterus of a New Forest pony, approximately five months into the pregnancy +   +
          + +

          + Wellcome Image Awards +

          + +
          +
          +
          +
          + +
          + +   +
          + +

          + The first space 'selfie' +

          + +
          +
          +
          +
          + +
          + +   +
          + +

          + Named after Sir David +

          + +
          +
          +
          +
          + +
          + Space selfie by European Space Agency astronaut Alexander Gerst +   +
          + +

          + Pictures of the year- part 4 +

          + +
          +
          +
          +
          +
          +
          + +
          + +
          + + + + +
          +
          +
          + +

          + Top news galleries +

          +
          + + + + + + + + + + + + + + + + +
          +
          +
          + + + + + + + + + + +
          + +
          + + + +
          +
          +
          + +
          +
          +
          +
          + +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          + +
          Advertisement
          +
          +
          + +
          +
          + +
          +
          +
          + +

          + Latest Video» +

          +
          + + +
          + +
          + + + + + + + +
          +
          + +
          +
          Scientist in lab + +
          + + Sponsored +

          + When media meets medicine +

          + +
          +
          + +
          +
          +
          +
          +
          +
          + + + +
          + +

          + More from the web +

          +
          + + +
          + +
          +
          +
          +
          + +
          Advertisement
          +
          +
          + +
          + +
          +
          + +
          Advertisement
          +
          +
          +
          + +
          +
          + + + + + + + +
          +
          + + + +
          + +

          + More from the web +

          +
          + + +
          + +
          +
          +
          +
          +
          +
          +
          + + + +
          + +

          + More from the web +

          +
          + + +
          + +
          +
          +
          + + +
          +
          + +
          +
          + +
          +
          + +
          +
          +
          +
          + + + +
          + +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          +
          +
          +
          +
          + +
          + +
          + + +
          + +
          +
          + + + + + + +
          + + + + + + +
          + + + + + + +
          + + + + +
          +
          + +
          +

          © Copyright of Telegraph Media Group Limited 2017

          +

          Terms and Conditions

          +

          Today's News

          +

          Archive

          +

          Style Book

          +

          Weather Forecast

          +
          +
          +
          + +
          + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + +
          + + + + + diff --git a/test/testdata/b97779a7fd47ca9d7bd5706c087514e8b8dde133.json b/test/testdata/b97779a7fd47ca9d7bd5706c087514e8b8dde133.json new file mode 100644 index 00000000..56da1c8c --- /dev/null +++ b/test/testdata/b97779a7fd47ca9d7bd5706c087514e8b8dde133.json @@ -0,0 +1,18 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "max-age=604800", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Language": "en-GB", + "Content-Length": "22007", + "Content-Type": "text/html;charset=UTF-8", + "Date": "Tue, 23 May 2017 17:53:19 GMT", + "ETag": "3313298-1495561998883", + "Server": "nginx", + "Vary": "Accept-Encoding", + "X-UA-Compatible": "IE=Edge" + }, + "status_code": 200, + "url": "http://www.telegraph.co.uk/news/science/science-news/3313298/Marine-collapse-linked-to-whale-decline.html" +} \ No newline at end of file diff --git a/test/testdata/b98e740c2474b9276735f64587a830673eae9e1c.html b/test/testdata/b98e740c2474b9276735f64587a830673eae9e1c.html new file mode 100644 index 00000000..06e0a2ef --- /dev/null +++ b/test/testdata/b98e740c2474b9276735f64587a830673eae9e1c.html @@ -0,0 +1,1415 @@ + + + + + + + + + + + + + + + + + + + + + + MIT News | Massachusetts Institute of Technology + + + + + + + + + + + + + + + + + + +
          +
          + +
          + + +
          + + +
          +
          + + +
          +

          In the Media

          +
          +
          + Read More +
          +
          +
          +
          +
          + + + +
          + + + diff --git a/test/testdata/b98e740c2474b9276735f64587a830673eae9e1c.json b/test/testdata/b98e740c2474b9276735f64587a830673eae9e1c.json new file mode 100644 index 00000000..8ae4715e --- /dev/null +++ b/test/testdata/b98e740c2474b9276735f64587a830673eae9e1c.json @@ -0,0 +1,21 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Cache-Control": "public, max-age=587", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Language": "en", + "Content-Length": "20975", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:54:56 GMT", + "Server": "Apache", + "Vary": "Accept-Encoding", + "X-Content-Type-Options": "nosniff, nosniff", + "X-Frame-Options": "SAMEORIGIN", + "X-Varnish": "1897441211 1897441135", + "X-Varnish-Cache": "HIT" + }, + "status_code": 200, + "url": "http://news.mit.edu/" +} \ No newline at end of file diff --git a/test/testdata/bb0c17a5757184f4e9bdd99b4662a46d23a45e47.html b/test/testdata/bb0c17a5757184f4e9bdd99b4662a46d23a45e47.html new file mode 100644 index 00000000..ad24acbb --- /dev/null +++ b/test/testdata/bb0c17a5757184f4e9bdd99b4662a46d23a45e47.html @@ -0,0 +1,2880 @@ + + + + + + + + + + + + + +The sperm whale works in extraordinary ways - Telegraph + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          +
          + + + + + + + + + + + + + + + + + + + + +
          + + + + + + + + + + +
          + + +
          +
          + +
          Telegraph.co.uk
          +
          + +
          +
          +
          +

          + Tuesday 23 May 2017

          +
          + +
          +
          + +
          +
          +
          +
          + + + +
          +
          + +
          +
            +
          +
          +
          +
          +
          + + +
          +
          +
          Advertisement
          +
          + +
          + + + + + + +
          +
          +
          +
            +
          1. Home»
          2. +
          3. News
          +
          +
          +
          + + +
          +
          + +
          +
          + + + +
          + +

          The sperm whale works in extraordinary ways

          +

          +As a new exhibition about sperm whales opens, biologist Hal Whitehead + describes the fascinating mechanisms of this creature + +

          + + + +
          +
          +
          +
          +
          + +
          + +
           
          +
          Image 1 of 3
          +
          +
          +
          +
          +
          + +
          + +
           
          +
          Image 1 of 3
          +
          +
          +
          +
          +
          + +
          + +
           
          +
          Image 1 of 3
          +
          +
          +
          +
          +
          +
          + + +
          +
          + + +
          + +
          +
          +
          + +
          +
          +
          +
          +
          + + +
          + +

          From Herman Melville to JMW Turner, writers and artists have long been fascinated by the sperm whale. Scientists, too, have been drawn to the creature, whether trying to understand the mechanisms behind its deep dives, the strange sounds it makes or its impact on the oceans.

          But for me, there is one question that overshadows all the others: is there any other being that bundles together so many extremes? Sperm whales are the largest-toothed animals in the world, have the longest intestines, the biggest brains and the largest noses. Their dives may be the deepest and longest of any mammal – and even with their numbers drastically reduced by whaling, they still take as much food out of the ocean each year as all of mankind’s fisheries put together. They live in the deepest oceans, ranging from the equator to the edges of the ice caps (the females live mostly in the tropics and the males, which are three times larger, at the poles). And such is their impact on the planet that the iron in the faeces of Antarctica’s sperm whales fertilises enough phytoplankton to slow the impact of global warming.

          This Friday, the impact of these marvellous creatures will be celebrated in the Peninsula Arts Whale Festival, a three-day festival exploring the art, writing and films that sperm whales have inspired, from Melville’s Moby Dick to Turner’s The Whale Ship. Indeed, ever since 1712, when a whaling ship from Nantucket became the first commercial vessel to slaughter a sperm whale, we have been closely linked: sperm whales were the mainstay of the oil industry from about 1750 to 1860, and became a major target at the peak of modern whaling in the 1960s.

          Sperm whaling was an important force behind our exploration of the oceans, and thus of the movement of animals, diseases and genes around the world. But the whalers’ focus was on the whale’s most extraordinary feature: its nose. About a quarter of the whale’s body is taken up by its massive snout, properly called the spermaceti organ, which is filled with a very fine grade of oil. This spermaceti was the oil that drove the Yankee whalers, lit the homes of the wealthy and lubricated the Industrial Revolution.

          But what do sperm whales themselves use their nose for? For a long while, scientists speculated that it might be an aid to diving or, after Moby Dick, a battering ram. The function of the spermaceti organ was finally nailed down about 10 years ago by the meticulous studies of the Danish scientists Bertel Møhl, Peter Madsen and their colleagues. They discovered that the spermaceti organ is actually the basis of the most powerful sonar system in the natural world, which gives the sperm whale a major advantage in the hunt for the elusive resources of the deep oceans. That, in turn, enables them to maintain their vast bodies and huge brains, which account for their ecological success.

          +

          Yet just as fascinating as the whales’ bodies are their minds. The fact that they evolved this all-conquering nose meant that sperm whales were now largely competing with each other for all that deep-water squid. This competition, as well as the threat of predators, in the form of killer whales, has turned sperm whales into intensely social animals. As Thomas Beale, a surgeon on a 19th-century whaling ship, noted: “The females are very remarkable for attachment to their young, which they may be frequently seen urging and assisting to escape from danger with the most unceasing care and fondness. They are also not less remarkable for their strong feeling of sociality or attachment to one another; and this is carried to so great an extent, as that one female of a herd being attacked and wounded, her faithful companions will remain around her to the last moment, or until they are wounded themselves.”

          It is this aspect of the sperm whales’ existence, in fact, that has been the focus of my research over the past 28 years. I go to sea in a 40ft sailing boat, listening for the powerful clicks coming from the spermaceti organs and using them as a beacon to locate the groups. We can identify the sperm whales individually by photographing their tails – raised into the air at the start of their deep foraging dives – as each has a distinctive pattern of marks.

          When the early whalers saw the huge, mature males surrounded by attentive females, they assumed they were watching a harem. But actually, it is the women who are the constant presence in each others’ lives, sticking together in family-based units of about 10. These groups of mothers and daughters stay together for their entire lives, travelling over thousands of miles, defending themselves communally against killer whales, and babysitting and suckling each others’ offspring while the other mothers are making deep dives.

          The males, by contrast, swim down from the cold waters to mate – but when we followed the female groups, we saw that they would join them for a matter of hours at the most. Sometimes the females seemed more than indifferent, literally turning their tails to the incoming giant. Other males were clearly more welcome, with the females gathering closely around him, touching and stroking. Female preference seems to have a large role in the mating process.

          The sperm whales’ social structure has other levels. Most striking are the clans, which we discovered from my colleague Luke Rendell’s analysis of “codas”, the patterned series of clicks that sperm whales use for communication. The females off the Galapagos Islands belong to two principal clans. There is the “regular” clan who go “click-click-click-click-click” and make convoluted tracks close to the islands, and the “plus-one” clan – “click-click-click-pause-click” – who generally move in straighter lines further offshore. Although social units from the two are often in the same general area, they only group with others from their own clan.

          Studying the whales’ genes, we found no differences to explain these behaviour patterns – so were forced to conclude that the members of the clans behave differently because they have different cultures, learning particular ways of life from the elders of their unit as they grow up. In other words, sperm whales live in a multicultural society.

          Culture’s profound role in the life of the sperm whale has important implications for how we see and treat the whales. Off the Galapagos, when the water is warm, the seas become less productive. Sperm whales, like most ocean life, suffer. But females of the “plus-one” clan feed better in these difficult conditions. This cultural diversity may be crucial for survival as the effects of our own human culture change the oceanic environment.

          Some of the remaining mysteries of the sperm whale will soon become clear: extraordinarily sensitive tags placed on the whales using suction cups, for a few hours at a time, are beginning to reveal the whales’ life at depth, how they dive and then feed far beneath the surface. But their brains and cultures present a greater challenge. We need to probe their learning, their thoughts and values – and, in the process, give humanity a glimpse of what Moby Dick’s motivation really was.

          Professor Hal Whitehead, of Dalhousie University in Halifax, Nova Scotia, will give Saturday’s keynote lecture as part of the Peninsula Arts Whale Festival, which runs from Friday to Sunday at the University of Plymouth, tel: 01752 585 050, www.peninsula-arts.co.uk

          +
          + +
          +
          + +
          +
          + +
          + + +
          +
          + +
          +
          + + +
          +

          In News

          +
          + + +
          +
          + +
          + General Election 2010; as it happened +   +
          + +

          + General Election 2015 +

          +
          +
          +
          +
          +
          +
          + +
          + +
          + + + + +
          +
          +
          + +

          + Top news galleries +

          +
          + + + + + + + + + + + +
          +
          +
          + + + + + + + + + + +
          + +
          + + + +How we moderate + +
          +
          telegraphuk
          +
          + + + blog comments powered by Disqus +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          + +
          Advertisement
          +
          +
          + +
          +
          + +
          +
          +
          + +

          + Latest video» +

          +
          + + +
          + +
          + + + + + + + +
          +
          + +
          +
          Scientist in lab + +
          + + Sponsored +

          + When media meets medicine +

          + +
          +
          + +
          +
          + +
          + +
          Featured Current Accounts
          Bank Account NameOffer More details
          First Direct 1st Current Account Exclusive £125 offer if you switch through MoneySuperMarket Apply
          TSB Classic Plus 3% AER on balances up to £1,500 Apply
          Natwest Reward Account 3% rewards on selected household bills Apply
          +
          + + +
          +
          +
          +
          +
          + + + +
          + +

          + More from the web +

          +
          + + +
          + +
          +
          +
          +
          + +
          Advertisement
          +
          +
          + +
          + +
          +
          + +
          Advertisement
          +
          +
          +
          + +
          +
          + + + + + + + +
          +
          + + + +
          + +

          + More from the web +

          +
          + + +
          + +
          +
          +
          +
          +
          +
          +
          + + + +
          + +

          + More from the web +

          +
          + + +
          + +
          +
          +
          + + +
          +
          + +
          +
          + +
          +
          + +
          +
          +
          +
          + + + +
          + +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          +
          +
          +
          +
          + +
          + +
          + + +
          + +
          +
          + + + + + + +
          + + + + + + +
          + + + + + + +
          + + + + +
          +
          + +
          +

          © Copyright of Telegraph Media Group Limited 2017

          +

          Terms and Conditions

          +

          Today's News

          +

          Archive

          +

          Style Book

          +

          Weather Forecast

          +
          +
          +
          + +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + +
          + + + + + diff --git a/test/testdata/bb0c17a5757184f4e9bdd99b4662a46d23a45e47.json b/test/testdata/bb0c17a5757184f4e9bdd99b4662a46d23a45e47.json new file mode 100644 index 00000000..f01ed277 --- /dev/null +++ b/test/testdata/bb0c17a5757184f4e9bdd99b4662a46d23a45e47.json @@ -0,0 +1,18 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "max-age=604800", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Language": "en-GB", + "Content-Length": "24595", + "Content-Type": "text/html;charset=UTF-8", + "Date": "Tue, 23 May 2017 17:53:21 GMT", + "ETag": "8323909-1495562000822", + "Server": "nginx", + "Vary": "Accept-Encoding", + "X-UA-Compatible": "IE=Edge" + }, + "status_code": 200, + "url": "http://www.telegraph.co.uk/news/8323909/The-sperm-whale-works-in-extraordinary-ways.html" +} \ No newline at end of file diff --git a/test/testdata/bcad6c54a769b0994f10315ba9a4ba23b54367f9.html b/test/testdata/bcad6c54a769b0994f10315ba9a4ba23b54367f9.html new file mode 100644 index 00000000..70062a6d --- /dev/null +++ b/test/testdata/bcad6c54a769b0994f10315ba9a4ba23b54367f9.html @@ -0,0 +1,1498 @@ + + + +BBC NEWS | Programmes | Newsnight Home | Malaria advice 'risks lives' + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +[an error occurred while processing this directive] + + + + + + + + +
          Help
          BBC TwoNewsnight
          + + + + + + + + + + + + + + + + + + + + + + +
          + + + + +
          + + + + +
          + Newsnight +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + FAQs +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + + + + + + + + + + + +
          + + + + +
          + + + +Last Updated: Thursday, 13 July 2006, 17:50 GMT 18:50 UK + +
          + + + + + + + + + + +
          + + + + + + + + + + + + + + +
          +
          +
          + Malaria advice 'risks lives' +
          +
          +
          + + + + + + + +
          + + + + + + +
          +
          + + + + By Meirion Jones + + + +
          + + BBC Newsnight + + +
          +

          + + + + + + +
          + + + + +

          + + + + + + + +
          + + +
          + Homeopath +
          + + + +
          Newsnight secretly filmed some homeopaths' advice
          +
          + + +
          +
          +
          + +
          + + + + + + + + + + + +
          + +
          + + + + + + + +Some high street homeopaths claim they can prevent malaria, a Newsnight investigation has found. + +

          + +Secret filming revealed homeopaths were claiming their preparations could be used instead of anti-malarial drugs to protect travellers in high risk areas such as sub-saharan Africa. + +

          + +Dr Ron Behrens of the London School of Hygiene and Tropical Medicine told Newsnight, "Sub Saharan Africa is a high risk of malaria. If they got it and they weren't immediately diagnosed and treated they could die and that claim would actually put their lives at risk." + +

          + +Two million Britons are expected to visit malarial areas this year - including many young backpackers setting off the next few weeks; 2,000 will come back with malaria. +

          + +'Not effective' +

          + + + + + + + + + +
          + + + + +
          +
          + + People may even die of malaria if they follow this advice +
          + + + + +
          + + +
          +
          Peter Fisher, director Royal London Homeopathic Hospital
          + + +
          + +
          + + + + + + +Dr Behrens has treated patients who fell for the homeopaths claims "We've certainly had patients admitted to our unit with the malignant form of malaria who have been taking homeopathic remedies and without a doubt the reason that they were taking them and not effective drugs was the reason they had malaria." +

          + +The Royal London Homeopathic Hospital is run by doctors who are also homeopaths and who treat conditions such as hay fever and rheumatism. They are also furious that some homeopaths are making these false claims about malaria. +

          + +The hospital's Director Peter Fisher told Newsnight "I'm very angry about it because people are going to get malaria - there is absolutely no reason to think that homeopathy works to prevent malaria and you won't find that in any textbook or journal of homeopathy so people will get malaria, people may even die of malaria if they follow this advice." + +

          + +Many of those who contract malaria while on holiday are people who failed to take their medication because they dislike the side-effects. How many relied on homeopathic remedies is not known but they are reluctant to admit that to doctors when they return for fear of looking foolish. + +

          +Undercover +

          + + + + +
          +
          + Mosquito +
          Every year 20 people die in the UK after contracting malaria abroad
          +
          +
          + + + + + + +The London School of Hygiene and Tropical Medicine was so concerned that it got together with the scientific pressure group Sense about Science to organise a survey of 10 homeopathic practices. +

          +They sent an undercover researcher in to say she was about to go in to a malaria infested country. They all recommended doses of homeopathic remedies - 99.99% water with an almost undetectable trace of effective remedies such as quinine. None of them directed the researcher to a GP or Travel Clinic. +

          + +Newsnight followed up their research with a hidden camera. A researcher went to Nelsons Pharmacy off Oxford Street in London, which claims to be Britain's biggest manufacturer of homeopathic remedies - and that was all they recommended for malaria. +

          + +High risk area +

          +Even when the researcher said she planned to go to Malawi - a high risk area - Nelsons only suggested the addition of garlic, oil of citronella and vitamins rather than a trip to the doctors. +

          +The Nelsons adviser told the researcher that the homeopathic compounds would protect her. "They make it so your energy doesn't have a malaria-shaped hole in it so the malarial mosquitos won't come along and fill that in." + +

          + +Nelsons subsequently said that this was against their policy which is to tell patients about the advice "to take prophylactic drugs, as well as providing information about the homeopathic remedies that are available" and they have reiterated that advice to their homeopaths. +

          + +Helios in Covent Garden, London told Newsnight's researcher she only needed their homeopathic compounds to protect her, saying "Yes you don't need to take anything else." +

          +Helios told us they still defend their advice. "Many people have researched anti-malarial drugs," they said, "and are concerned about the side effects. We give advice on traditional homeopathic remedies." +

          +Superdrug +

          + + + + + + +
          +
          + Larium +
          Prescribed drugs like Larium can help prevent malaria
          +
          +
          + + + + + +We also made an appointment to see a homeopath at Superdrug in the Strand in London. Our researcher told the Superdrug staff member at the pharmacy counter she wanted the homeopathic appointment "to ask about malaria". +

          + +The Superdrug employee replied, "I see you want a homeopathic kind of solution." We were told we would pay Superdrug directly 58 for the first appointment but Superdrug have subsequently pointed out that she is an independent homeopath who runs a private clinic once a week in their store. They also told us that, "It is Superdrug policy to offer traditionally recognized medicines." + +

          + +Around 20 people died of malaria last year after returning to the UK - mainly because they failed to take adequate prophylactic protection. It is a small number compared to the million people who died worldwide of malaria but the Chief Medical Officer told Newsnight tonight that the British cases are avoidable if proper prophylactics are taken. He warned against relying on homeopathic remedies for malaria. +

          +


          + + + + + + +

          +Statement by one homeopathy practice featured in the film: +

          +"The Vale Practice is a complimentary therapy centre that does exactly that - compliment the medical model, this however can only be done after a thorough consultation. In this instance a consultation was refused and direct and leading questions were put to the homeopath which can be taken out of context. The fact of the matter is that prophylactic drugs do have side effects and there may be alternatives to consider, the ultimate decision always rests with the patient, and as a practice The Vale Practice are there to advise and support in both instances." + + + + + + + + +

          + + +

          + + + + +
          + + + + + +
          + + + + + + + +
          + + + + + + + + + + + + + + + + + +
          + Watch Newsnight on BBC iPlayer + +
          + + + + + + + +
          + + + + + + + + + +
          + + + + + + + + + + + +
          + + + WATCH HIGHLIGHTS + +
          + + + + + + + + + +
          + + + + + + + Angela Merkel + +Merkel opposes Greece euro exit
          + + +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + + + + + + + + + + Mark Thompson + +A look back at career of BBC boss Mark Thompson
          + + +
          + + + + + + +
          + + + + + + + + + The head of the Catholic Church in England and Wales, Vincent Nichols + +Archbishop: 'Gay relationships are friendships'
          + + +
          + + + + + + +
          + + + + + + + + + Child in East Timor + +The lost children of East Timor
          + + +
          + + + + + + +
          + + + + + + + + + Bishop Nazir Ali + +Bishop Nazir Ali on the need for morality
          + + +
          + + + + + + +
          + + + + + + + + + + + + + + + + + + + +
          + + NEWSNIGHT BLOGS + + +
          + + + + + + + +
          + + + + + + + + + + + + + + + + +
          + + + + + + + + + + + + + + + + +
          + + + + + + + + + + + + + + + + +
          + + + + + + + + + + + + + + + + +
          + + + + + + + + + + + + + + + + +
          + + + + + + + + + + + + + + + + + + + + + + + + +
          + + + + + + +
          + + + + + + + + + + + +
          + RELATED INTERNET LINKS + +
          + + + + + + + + +
          + The BBC is not responsible for the content of external internet sites +
          + + + + + + + + +
          + + + + + + + + + + + +
          + +
          + +
          +
          + + + +
          + +
          + + + + + + + +
          + + +
          + + + + + + + +
          + + + FEATURES, VIEWS, ANALYSIS + +
          + + + + + + + +
          + + + + + + +
          + +
          + + + + + + + +
          + Horses sculpture in memory of Genghis Khan, Ordos, Inner Mongolia + + + + + Ghost town + +
          + + + + + +
          + +
          + + + + Has China's housing bubble burst? + + + +
          + + + + + + + + + + + + + +
          + +
          + +
          + + + + + + + +
          + Afo - the world's oldest clove tree + + + + + The guerilla plant + +
          + + + + + +
          + +
          + + + + How the world's oldest clove tree defied an empire + + + +
          + + + + + + + + + + + + + +
          + +
          + +
          + + + + + + + +
          + Sergei Polunin + + + + + Walking away + +
          + + + + + +
          + +
          + + + + Why Royal Ballet principal Sergei Polunin quit + + + +
          + + + + + + + + + + + + + +
          + +
          + +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + +
          + +
          +
          + +
          + +
          + +
          +
            +
          • MMIX
          • + + + +
          • Back to top ^^
          • +
          + +
          + + + + +banner +watch listen + + + + + + +bbc sport + + + + + + +Americas +Africa +Europe +Middle East +South Asia +Asia Pacific + + +
          + + + + + +
          + + + \ No newline at end of file diff --git a/test/testdata/bcad6c54a769b0994f10315ba9a4ba23b54367f9.json b/test/testdata/bcad6c54a769b0994f10315ba9a4ba23b54367f9.json new file mode 100644 index 00000000..fa2dc124 --- /dev/null +++ b/test/testdata/bcad6c54a769b0994f10315ba9a4ba23b54367f9.json @@ -0,0 +1,17 @@ +{ + "encoding": "ISO-8859-1", + "headers": { + "Cache-Control": "max-age=0", + "Connection": "Keep-Alive", + "Content-Type": "text/html", + "Date": "Tue, 23 May 2017 17:52:58 GMT", + "Expires": "Tue, 23 May 2017 17:52:58 GMT", + "Keep-Alive": "timeout=5, max=730", + "Server": "Apache", + "Set-Cookie": "BBC-UID=b5e9d274a7b61f8ad51b19dea1e5f75dae4b53aa20108550a5a639ced60f93580Mozilla%2f5%2e0%20%28Windows%20NT%2010%2e0%3b%20Win64%3b%20x64%3b%20rv%3a50%2e0%29%20Gecko%2f20100101%20Firefox%2f50%2e0; expires=Wed, 23-May-18 17:52:58 GMT; path=/; domain=bbc.co.uk;, BBC-UID=b5e9d274a7b61f8ad51b19dea1e5f75dae4b53aa20108550a5a639ced60f93580Mozilla%2f5%2e0%20%28Windows%20NT%2010%2e0%3b%20Win64%3b%20x64%3b%20rv%3a50%2e0%29%20Gecko%2f20100101%20Firefox%2f50%2e0; expires=Wed, 23-May-18 17:52:58 GMT; path=/; domain=bbc.co.uk;", + "Transfer-Encoding": "chunked", + "Vary": "X-CDN" + }, + "status_code": 200, + "url": "http://news.bbc.co.uk/2/hi/programmes/newsnight/5178122.stm" +} \ No newline at end of file diff --git a/test/testdata/bd625fc35a78d02f30972ee97928a09b07d645fd.html b/test/testdata/bd625fc35a78d02f30972ee97928a09b07d645fd.html new file mode 100644 index 00000000..0f613b7c --- /dev/null +++ b/test/testdata/bd625fc35a78d02f30972ee97928a09b07d645fd.html @@ -0,0 +1,10 @@ +TY - BOOK +T1 - Physics for Scientists and Engineers, Volume 1, Chapters 1-22 +A1 - Serway, R.A. +A1 - Jewett, J.W. +SN - 9781439048382 +T3 - Physics for Scientists and Engineers +UR - https://books.google.com/books?id=6upvonUt0O8C +Y1 - 2009 +PB - Cengage Learning +ER - diff --git a/test/testdata/bd625fc35a78d02f30972ee97928a09b07d645fd.json b/test/testdata/bd625fc35a78d02f30972ee97928a09b07d645fd.json new file mode 100644 index 00000000..39874716 --- /dev/null +++ b/test/testdata/bd625fc35a78d02f30972ee97928a09b07d645fd.json @@ -0,0 +1,21 @@ +{ + "encoding": null, + "headers": { + "Alt-Svc": "h3-29=\":443\"; ma=2592000,h3-27=\":443\"; ma=2592000,h3-25=\":443\"; ma=2592000,h3-T050=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"", + "Cache-Control": "private, max-age=0", + "Content-Disposition": "attachment; filename=Physics_for_Scientists_and_Engineers_Vol.ris", + "Content-Length": "284", + "Content-Type": "application/x-research-info-systems", + "Date": "Sat, 11 Jul 2020 09:16:27 GMT", + "Expires": "Sat, 11 Jul 2020 09:16:27 GMT", + "P3P": "CP=\"This is not a P3P policy! See g.co/p3phelp for more info.\"", + "Server": "OFE/0.1", + "Set-Cookie": "NID=204=TP3EIbIif056oh2e9nOCOCJEQqGa07TN75x7shjkzvf-jzqXh0NwbDZkjoESrZYAo3EgP2WddlbF6bBK-CqCm2IkA58Mxbt43YFQMAGhprzGfvTuEpR0KS9dN9HuOzs-HDagr-0ytUXK8uNOqPk5XTvwQEI4I6tXdP21V7ETTZM; expires=Sun, 10-Jan-2021 09:16:27 GMT; path=/; domain=.google.com; Secure; HttpOnly; SameSite=none", + "Strict-Transport-Security": "max-age=604800", + "X-Content-Type-Options": "nosniff", + "X-Frame-Options": "SAMEORIGIN", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://books.google.com/books/download/?id=6upvonUt0O8C&output=ris" +} \ No newline at end of file diff --git a/test/testdata/bd7aa388347c38d4822e36054dc93d429e453a24.html b/test/testdata/bd7aa388347c38d4822e36054dc93d429e453a24.html new file mode 100644 index 00000000..c1ab97b2 --- /dev/null +++ b/test/testdata/bd7aa388347c38d4822e36054dc93d429e453a24.html @@ -0,0 +1,160 @@ + + + + +arXiv.org e-Print archive + + + + + + + + + + + + + + + +
          +

          Open access to 1,268,850 e-prints in Physics, Mathematics, Computer Science, Quantitative Biology, Quantitative Finance and Statistics

          +
          + Subject search and browse: + + + + +
          + +

          +20 Apr 2017: Applied Physics subject area added to arXiv
          +10 Mar 2017: New members join arXiv Member Advisory Board
          +06 Mar 2017: arXiv Scientific Director Search
          +10 Feb 2017: Attention Submitters: our TeX processing system has been updated
          +See cumulative "What's New" pages. +Read robots beware before attempting any automated download +

          +

          Physics

          + +

          Mathematics

          + +

          Computer Science

          + +

          Quantitative Biology

          + +

          Quantitative Finance

          + +

          Statistics

          + + +
          +

          About arXiv

          + + +
          + + + diff --git a/test/testdata/bd7aa388347c38d4822e36054dc93d429e453a24.json b/test/testdata/bd7aa388347c38d4822e36054dc93d429e453a24.json new file mode 100644 index 00000000..65a03006 --- /dev/null +++ b/test/testdata/bd7aa388347c38d4822e36054dc93d429e453a24.json @@ -0,0 +1,17 @@ +{ + "encoding": "utf-8", + "headers": { + "Connection": "Keep-Alive", + "Content-Encoding": "gzip", + "Content-Length": "4891", + "Content-Type": "text/html; charset=utf-8", + "Date": "Thu, 01 Jun 2017 05:16:38 GMT", + "Keep-Alive": "timeout=8, max=100", + "Server": "Apache", + "Set-Cookie": "browser=151.246.252.117.1496294198871567; path=/; max-age=946080000; domain=.arxiv.org", + "Strict-Transport-Security": "max-age=31536000", + "Vary": "Accept-Encoding,User-Agent" + }, + "status_code": 200, + "url": "https://arxiv.org/" +} \ No newline at end of file diff --git a/test/testdata/c1a110eef1a4b0af3a17d2da0a15bf1ba7fab08b.html b/test/testdata/c1a110eef1a4b0af3a17d2da0a15bf1ba7fab08b.html new file mode 100644 index 00000000..cad75858 --- /dev/null +++ b/test/testdata/c1a110eef1a4b0af3a17d2da0a15bf1ba7fab08b.html @@ -0,0 +1,2022 @@ + + + + + + + + + + + + + + + Al Jazeera: Live News | Bold Perspectives | Exclusive Films + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + + + + + + + +
          +
          +
          +
          + +
          +
          +
          +
          + + + +
          +
          +
          + +
          +
          +
          +
          +
          +
          + + + + + + + + + + + +
          +
          +
          + + +
          + +
          +
          +
          truetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetruetrue + More News +
          + +
          +
          +
          body : Layout 1 : Cell 1 : Layout 1 : Cell 3 : Layout 1 : Cell 4 : Layout 2 : Cell 6 : Layout 1 : Cell 9
          + + + + +
          + + + +
          +
          + +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +

          +

          +

          +

          +
          +
          +
          + + + + + +
          +
          +
          +
          + +
          + +
          + +
          +
          +
          + + + +
          + +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + diff --git a/test/testdata/c1a110eef1a4b0af3a17d2da0a15bf1ba7fab08b.json b/test/testdata/c1a110eef1a4b0af3a17d2da0a15bf1ba7fab08b.json new file mode 100644 index 00000000..efb0dbdc --- /dev/null +++ b/test/testdata/c1a110eef1a4b0af3a17d2da0a15bf1ba7fab08b.json @@ -0,0 +1,20 @@ +{ + "encoding": "ISO-8859-1", + "headers": { + "Access-Control-Allow-Origin": "http://live.aljazeera.com", + "Cache-Control": "public, max-age=60", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "27215", + "Content-Type": "text/html", + "Date": "Tue, 30 May 2017 04:57:45 GMT", + "Expires": "Tue, 30 May 2017 04:58:45 GMT", + "Server": "Microsoft-IIS/10.0", + "X-Method": "GET", + "X-Powered-By": "VSH-Z-2", + "X-WR-MODIFICATION": "Content-Length", + "publisher": "Al Jazeera (ORYX CMS) - ZUB" + }, + "status_code": 200, + "url": "http://www.aljazeera.com/" +} \ No newline at end of file diff --git a/test/testdata/c20a426ccf6694797b8d93d6d60e54a76af6c699.html b/test/testdata/c20a426ccf6694797b8d93d6d60e54a76af6c699.html new file mode 100644 index 00000000..e6d953e3 --- /dev/null +++ b/test/testdata/c20a426ccf6694797b8d93d6d60e54a76af6c699.html @@ -0,0 +1,724 @@ + + + + + + تصویر کتاب المعجم الموضوعي لإحادیث الإمام المهدي (عجل الله فرجه الشریف) - جلد 1 - صفحه 1 - کورانی، علی + + + + + + + + + + + + + + + + + + +
          +
          + + + + +
          +
          + + + +
          + + +
          +
          +
          + + + +
          +
          +
          + + +
          + +
          +
          + + + + + + + + + +
          + +
          +
          +
          +
          + + +
          +
          +
          +
          + + +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + + + + + + + + +
          +
          +
          + +
          + + + + +
          +
          + +
          + +
          + +
          + +
          + + +
          +
          + + + + + diff --git a/test/testdata/c20a426ccf6694797b8d93d6d60e54a76af6c699.json b/test/testdata/c20a426ccf6694797b8d93d6d60e54a76af6c699.json new file mode 100644 index 00000000..306c3c8a --- /dev/null +++ b/test/testdata/c20a426ccf6694797b8d93d6d60e54a76af6c699.json @@ -0,0 +1,17 @@ +{ + "encoding": "utf-8", + "headers": { + "Cache-Control": "private", + "Content-Encoding": "gzip", + "Content-Length": "15444", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:51:52 GMT", + "Server": "Microsoft-IIS/7.5", + "Set-Cookie": "ASP.NET_SessionId=1ytp4zmvcsiakee2eaxjbn1h; path=/; HttpOnly", + "Vary": "Accept-Encoding", + "X-AspNet-Version": "4.0.30319", + "X-Powered-By": "ASP.NET" + }, + "status_code": 200, + "url": "http://www.noorlib.ir/View/fa/Book/BookView/Image/18454" +} \ No newline at end of file diff --git a/test/testdata/c39fb3a31f1d10d5c8dc2b8e6fcc3ed71d48212a.html b/test/testdata/c39fb3a31f1d10d5c8dc2b8e6fcc3ed71d48212a.html new file mode 100644 index 00000000..6ce265c5 --- /dev/null +++ b/test/testdata/c39fb3a31f1d10d5c8dc2b8e6fcc3ed71d48212a.html @@ -0,0 +1 @@ +{"indexed":{"date-parts":[[2022,5,14]],"date-time":"2022-05-14T13:53:54Z","timestamp":1652536434299},"reference-count":0,"publisher":"American Psychological Association (APA)","issue":"9","content-domain":{"domain":[],"crossmark-restriction":false},"published-print":{"date-parts":[[1939]]},"DOI":"10.1037\/h0063404","type":"journal-article","created":{"date-parts":[[2006,6,8]],"date-time":"2006-06-08T07:22:35Z","timestamp":1149751355000},"page":"641-656","source":"Crossref","is-referenced-by-count":162,"title":"Studies in retention.","prefix":"10.1037","volume":"30","author":[{"given":"H. F.","family":"Spitzer","sequence":"first","affiliation":[]}],"member":"15","container-title":"Journal of Educational Psychology","original-title":[],"language":"en","deposited":{"date-parts":[[2011,8,13]],"date-time":"2011-08-13T09:03:25Z","timestamp":1313226205000},"score":1,"resource":{"primary":{"URL":"http:\/\/content.apa.org\/journals\/edu\/30\/9\/641"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[1939]]},"references-count":0,"journal-issue":{"issue":"9","published-print":{"date-parts":[[1939]]}},"alternative-id":["1940-02338-001"],"URL":"http:\/\/dx.doi.org\/10.1037\/h0063404","relation":{},"ISSN":["0022-0663"],"subject":["Developmental and Educational Psychology","Education"],"container-title-short":"Journal of Educational Psychology","published":{"date-parts":[[1939]]}} \ No newline at end of file diff --git a/test/testdata/c39fb3a31f1d10d5c8dc2b8e6fcc3ed71d48212a.json b/test/testdata/c39fb3a31f1d10d5c8dc2b8e6fcc3ed71d48212a.json new file mode 100644 index 00000000..69f8975e --- /dev/null +++ b/test/testdata/c39fb3a31f1d10d5c8dc2b8e6fcc3ed71d48212a.json @@ -0,0 +1,24 @@ +{ + "encoding": null, + "headers": { + "access-control-allow-headers": "X-Requested-With, Accept, Accept-Encoding, Accept-Charset, Accept-Language, Accept-Ranges, Cache-Control", + "access-control-allow-origin": "*", + "access-control-expose-headers": "Link", + "connection": "close", + "content-encoding": "gzip", + "content-length": "703", + "content-type": "application/vnd.citationstyles.csl+json", + "date": "Thu, 09 Jun 2022 10:37:20 GMT", + "link": "; rel=\"canonical\"", + "permissions-policy": "interest-cohort=()", + "server": "Jetty(9.4.40.v20210413)", + "vary": "Accept, Accept-Encoding", + "x-api-pool": "public", + "x-rate-limit-interval": "1s", + "x-rate-limit-limit": "50", + "x-ratelimit-interval": "1s", + "x-ratelimit-limit": "50" + }, + "status_code": 200, + "url": "https://api.crossref.org/v1/works/10.1037%2Fh0063404/transform" +} \ No newline at end of file diff --git a/test/testdata/c8b5fafdb17cdce774489ce1e74dd56341091b82.html b/test/testdata/c8b5fafdb17cdce774489ce1e74dd56341091b82.html new file mode 100644 index 00000000..5d8a2c4c --- /dev/null +++ b/test/testdata/c8b5fafdb17cdce774489ce1e74dd56341091b82.html @@ -0,0 +1,1198 @@ + + + + + + + + + + 'Revolutionary' Physics: Do Sterile Neutrinos Lurk in the Universe? + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + + + + + + + +
          + + + + + + + + + +
          + +
          +
          + +
          +
          + +
          +
          + + +
          +
          +
            +
          • +
          • +
          • +
          • +
          • +
          +
          +
          + + +
          +
          +
          +
          +
          + + + + + +

          'Revolutionary' Physics: Do Sterile Neutrinos Lurk in the Universe?

          + + + + +
          +
          + +
          +
            +
          • + +
          • +
          • + +
          • +
          • + + + + + +
          • +
          • + +
          • +
          • + +
          • +
          • MORE
          • +
          +
          + + + +
          +
          +
          +
          + 'Revolutionary' Physics: Do Sterile Neutrinos Lurk in the Universe? +
          + +
          +
          The detector for the MicroBooNe is gently lowered into place.
          + Credit: Fermilab +
          +
          +
          +

          + A completely new subatomic particle — one so reclusive and strange that it passes undetected through ordinary matter — could be lurking in the universe.

          +

          + If so, a detector set to turn on later this year could find the first convincing evidence for the particle, called a sterile neutrino. The new experiment, whose 30-ton detector was recently lowered into place at Fermi National Accelerator Laboratory in Illinois, will look for traces of this elusive particle transforming into another type of neutrino.

          +

          + Unlike the Higgs boson, the particle thought to explain why other particles have mass and which most physicists predicted should exist for decades, sterile neutrinos would be in the realm of completely unknown physics that only some physicists believe exist, said Bonnie Fleming, the experiment's spokeswoman and a physicist at Yale University. "It would be completely revolutionary," Fleming said. [Wacky Physics: The Coolest Little Particles in Nature]

          +

          + Ghostly particles

          +

          + Neutrinos are miniscule, nearly massless subatomic particles that form during nuclear reactions in the hearts of stars, supernovae and other explosive cosmic events. Though trillions of neutrinos pass through our bodies every second, they almost never interact with other matter, giving them the nickname "ghost particles."

          +

          + The known neutrinos come in three different types, or flavors — electron, muon and tau — and in the last 15 to 20 years, scientists have learned that those flavors oscillate, or change into one another, with a certain frequency. (During collisions, electron neutrinos can also turn into electrons, muon neutrinos can transform into muons, and tau neutrinos can turn into tau leptons, particles that are similar to electrons.

          +

          + But a few hints suggest there could be a totally new type of neutrino out there. For instance, experiments in the 1990s to detect neutrinos from the sun found possible evidence that electron neutrinos were disappearing. Another experiment designed to probe neutrino oscillation found extra electron neutrinos appearing. One explanation for these anomalies is that the neutrinos were morphing into an intermediate particle called a sterile neutrino.

          +

          + If such sterile neutrinos exist, they would interact only with matter through the incredible weak force of gravity, making direct detection impossible, Fleming told Live Science.

          +

          + Hunting sterile neutrinos

          +

          + So starting late this year or early in 2015, Fleming and her colleagues will look for indirect evidence of sterile neutrinos. The experiment, called MicroBooNE, will shoot a beam of pure muon-flavored neutrinos 0.3 miles (0.5 kilometers) through a 30-ton metal tank filled with argon. Though most of these ghost particles will travel through the argon unchanged, some will occasionally change flavor to an electron neutrino, tau neutrino — or possibly a sterile neutrino. 

          +
          + The 30-ton argon detector has been under construction for two years. + +
          +
          +
          The 30-ton argon detector has been under construction for two years.
          + Credit: Fermilab +
          +
          +

          +

          + Some fraction of these neutrinos will then go on to collide with the nuclei of argon atoms in the detector.

          +

          + "They will shatter that nucleus, and parts of that nucleus will go everywhere," said Matt Strassler, a physicist at Harvard University who was not involved in the study. As part of the collision, electron neutrinos will sometimes morph into electrons, Strassler added.

          +

          + The detector then identifies where, when and what type of particles were created by analyzing the trail left by ionized, or charged, particles after the collision.

          +

          + Because the researchers know how often electron neutrinos should convert into electrons during such collisions, any deviation from expectations could be a sign that a muon neutrino morphed into an intermediate sterile neutrino, then into an electron neutrino, and finally into an electron.

          +

          + Longshot physics

          +

          + Though the discovery of a sterile neutrino is a possibility, it's not likely, Strassler said.

          +

          + MicroBooNE is working to clarify tantalizing hints in data from a precursor experiment called MiniBooNE, but there's a good chance that MiniBooNE's "dirty measurement" is picking up other processes instead, Strassler said.

          +

          + Even if the new experiment uncovers something strange, there's no guarantee sterile neutrinos caused the signal, rather than some other completely different interaction, he said.

          +

          + "There's a very small — not zero — chance that they're actually going to uncover one of the great secrets of the universe," Strassler told Live Science.

          +

          + Follow Tia Ghose on Twitter and Google+. Follow Live Science @livescience, Facebook & Google+. Original article on Live Science.

          + +
          +
          + + +
          + +
          +
          + +
          +
          + +
          + +
          + +
          +
          Author Bio
          +
          + Tia Ghose + +
          +
          Tia Ghose, Senior Writer
          +
          +

          + Tia has interned at Science News, Wired.com, and the Milwaukee Journal Sentinel and has written for the Center for Investigative Reporting, Scientific American, and ScienceNow. She has a master's degree in bioengineering from the University of Washington and a graduate certificate in science writing from the University of California Santa Cruz.

          + +
          + +
          +
          +
          + +
          + +
          +
          +
          +
          +
          + +
          +
          +
          Follow Us
          + +
          + + + + +
          +
          + +
          +
          + +
          + +
          +
          +
          + +
          +
          +
          + + +
          +
          +
          +
          + + +
          + +
          + + +
          + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/testdata/c8b5fafdb17cdce774489ce1e74dd56341091b82.json b/test/testdata/c8b5fafdb17cdce774489ce1e74dd56341091b82.json new file mode 100644 index 00000000..3621816e --- /dev/null +++ b/test/testdata/c8b5fafdb17cdce774489ce1e74dd56341091b82.json @@ -0,0 +1,19 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "max-age=0, no-cache", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "30237", + "Content-Type": "text/html; charset=UTF-8", + "Date": "Tue, 23 May 2017 17:54:15 GMT", + "Expires": "Tue, 23 May 2017 17:54:15 GMT", + "Pragma": "no-cache", + "Server": "nginx", + "Set-Cookie": "__uzma=59247747035b40.04506070; expires=Fri, 21-May-2027 17:54:15 GMT; Max-Age=315360000; path=/, __uzmd=1495562055; expires=Fri, 21-May-2027 17:54:15 GMT; Max-Age=315360000; path=/, __uzmc=857111084218; expires=Fri, 21-May-2027 17:54:15 GMT; Max-Age=315360000; path=/, __uzmb=1495562055; expires=Fri, 21-May-2027 17:54:15 GMT; Max-Age=315360000; path=/", + "Surrogate-Control": "content=\"ESI/1.0\"", + "Vary": "Accept-Encoding" + }, + "status_code": 200, + "url": "http://www.livescience.com/46619-sterile-neutrino-experiment-beginning.html?cmpid=514645_20140702_27078936" +} \ No newline at end of file diff --git a/test/testdata/cbeee66c347615f111e571862101ec1bf3e40ed9.html b/test/testdata/cbeee66c347615f111e571862101ec1bf3e40ed9.html new file mode 100644 index 00000000..d2590d24 --- /dev/null +++ b/test/testdata/cbeee66c347615f111e571862101ec1bf3e40ed9.html @@ -0,0 +1,18 @@ + +TY - JOUR +T1 - زندگی نامه علمی دکتر کاووس حسن لی +T2 - شعر +JF - شعر +Y1 - 1387/// + +LA - fa +UR - https://www.noormags.ir/view/fa/articlepage/454096 +SP - 17 +EP - 19 +SN - +VL - 62 +IS - 15 + +ID - 454096 + +ER - diff --git a/test/testdata/cbeee66c347615f111e571862101ec1bf3e40ed9.json b/test/testdata/cbeee66c347615f111e571862101ec1bf3e40ed9.json new file mode 100644 index 00000000..bf8c838d --- /dev/null +++ b/test/testdata/cbeee66c347615f111e571862101ec1bf3e40ed9.json @@ -0,0 +1,16 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "private", + "Content-Disposition": "attachment; filename=\"noormags-454096.ris\"", + "Content-Length": "269", + "Content-Type": "application/x-Research-Info-Systems; charset=UTF-8", + "Date": "Fri, 13 Apr 2018 07:58:11 GMT", + "Set-Cookie": "CRCIS_SessionId=aiz4qxh2qloclwwqrp05ovqx; path=/; secure; HttpOnly", + "Strict-Transport-Security": "max-age=3153600", + "X-Download-Options": "noopen", + "X-Frame-Options": "SAMEORIGIN" + }, + "status_code": 200, + "url": "https://www.noormags.ir/view/fa/citation/ris/454096" +} \ No newline at end of file diff --git a/test/testdata/cfd583f43da034a0150306993e4a55be6de97a0c.html b/test/testdata/cfd583f43da034a0150306993e4a55be6de97a0c.html new file mode 100644 index 00000000..09e20c74 --- /dev/null +++ b/test/testdata/cfd583f43da034a0150306993e4a55be6de97a0c.html @@ -0,0 +1,1122 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + PubMed + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Skip to main page content + + +
          + + + +
          + + +
          +
          +
          + + + + + +
          +

          Home Page

          +
          +
          + + + + + + + +
          + PubMed® comprises more than 34 million citations for biomedical literature from MEDLINE, life science journals, and online books. Citations may include links to full text content from PubMed Central and publisher web sites. +
          +
          +
          + + + +
          + + + + + +
          +
          +

          Download

          + + + + + + + +
          + +
          +
          +

          Explore

          + + + + + +
          + +
          + + + +
          + +
          + + +
          + + + + + +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/testdata/cfd583f43da034a0150306993e4a55be6de97a0c.json b/test/testdata/cfd583f43da034a0150306993e4a55be6de97a0c.json new file mode 100644 index 00000000..b05d3ccf --- /dev/null +++ b/test/testdata/cfd583f43da034a0150306993e4a55be6de97a0c.json @@ -0,0 +1,23 @@ +{ + "encoding": "utf-8", + "headers": { + "Alt-Svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "Cache-Control": "private", + "Content-Type": "text/html; charset=utf-8", + "Date": "Sat, 27 Aug 2022 12:11:04 GMT", + "Referrer-Policy": "same-origin", + "Server": "nginx", + "Set-Cookie": "pm-csrf=cU8p0Vp6TpbyjW685GqICByEpxhHQ4jHeLOZCtz3dbnIrwtXmr6puQADD0R1Olqo; expires=Sat, 26 Aug 2023 12:11:04 GMT; HttpOnly; Max-Age=31449600; Path=/; SameSite=Lax; Secure, pm-sessionid=rngwb8xtr4b3re0fwx8fea4ki0s0gzh7; expires=Sat, 27 Aug 2022 20:11:04 GMT; HttpOnly; Max-Age=28800; Path=/; Secure, ncbi_sid=CDBB677A309EB7D3_18024SID; Domain=.nih.gov; expires=Sun, 27 Aug 2023 12:11:04 GMT; HttpOnly; Max-Age=31536000; Path=/; Secure", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "Transfer-Encoding": "chunked", + "Vary": "Origin", + "Via": "1.1 google", + "X-Content-Type-Options": "nosniff", + "X-Frame-Options": "DENY", + "X-UA-Compatible": "IE=Edge", + "X-XSS-Protection": "1; mode=block", + "content-encoding": "gzip" + }, + "status_code": 200, + "url": "https://pubmed.ncbi.nlm.nih.gov/" +} \ No newline at end of file diff --git a/test/testdata/d0f92165648711ec5143b20e5f2fa5445087e206.html b/test/testdata/d0f92165648711ec5143b20e5f2fa5445087e206.html new file mode 100644 index 00000000..b873b818 --- /dev/null +++ b/test/testdata/d0f92165648711ec5143b20e5f2fa5445087e206.html @@ -0,0 +1,2065 @@ + + + + + + + + + +Man Undergoes Extensive Plastic Surgery To Look Like Justin Bieber, Spends $100,000 In 5 Years | HuffPost + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + + + + +
          + + +
          +
          +
          +
          +
          + +
          +
          + +
          + + +
          +
          +
          + +
          + + +
          + + + + + + +
          + + + + +
          + + +
          + + + + +CELEBRITY + + + +
          +10/19/2013 01:00 pm ET +| + +Updated +Oct 21, 2013 + +
          + + + + + + +
          +

          Man Undergoes Extensive Plastic Surgery To Look Like Justin Bieber, Spends $100,000 In 5 Years

          +
          + + + + + + +
          +
          +
          + + +
          +
          + +
          + + + +
          + + + + + + +
          +

          Not only is songwriter Toby Sheldon a 33-year-old Justin Bieber fan, his Bieber-devotion would blow teeny-bopping Beliebers out of the water.

          +

          That's because Sheldon went so far as to spend almost $100,000 on five years worth of plastic surgery to make him look like his idol, according to the British tabloid Closer.

          +

          This is what Sheldon looks like after all that plastic surgery he had done to resemble the 19-year-old Canadian crooner:

          +

          + + +

          On top of Botox injections and hair transplants, Sheldon had costly "smile surgery" done to make his smile look just like Bieber's, according to multiple reports.

          +

          "It's Justin's smile that gives him his youthful look. So I had my upper lip lifted [and] my bottom lip plumped out," the musician told Closer.

          +

          (h/t Reddit)

          + + + + + + + + +
          + + + + + + + + +
          +

          ALSO ON HUFFPOST:

          +
          + +
          +
          +
          +
          +
          +
          Justin Bieber through the years
          +
          +
          +
          + +
          + + + +
          +
          + + + + + + +
          +
          + + + + + + + + +
          +
          + +
          + +
          + + + + + + + + + + + + + + +
          + + + + + + + + + + + + +
          + +
          + + +
          + + + + + + +
          + + +
          +
          +
          +
          +
          +

          CONVERSATIONS

          +
          + + +
          +
          + +
          + + +
          + + +
          + + + + + + + + + + + + + + + + +
          + + +
          + + + + + + + + + + + + + + + +
          +
          + + + + + + + + + + + + + + + + + +
          + + + + + + + + + +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + +
          + + diff --git a/test/testdata/d0f92165648711ec5143b20e5f2fa5445087e206.json b/test/testdata/d0f92165648711ec5143b20e5f2fa5445087e206.json new file mode 100644 index 00000000..29930baa --- /dev/null +++ b/test/testdata/d0f92165648711ec5143b20e5f2fa5445087e206.json @@ -0,0 +1,32 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "249", + "Cache-Control": "max-age=300, public, must_revalidate=false", + "Content-Encoding": "gzip", + "Content-Length": "48579", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:59:41 GMT", + "Last-Modified": "Tue, 23 May 2017 17:55:32 GMT", + "Server": "ECD (iad/1986)", + "Vary": "Accept-Encoding", + "X-Cache": "HIT", + "X-Content-Type-Options": "nosniff", + "X-EC-Lua": "19365-geo", + "X-Frame-Options": "ALLOWALL", + "X-GEO-URL-AU": "http://www.huffingtonpost.com.au/entry/plastic-surgery-justin-bieber-100k_n_4128563", + "X-GEO-URL-BR": "http://www.huffpostbrasil.com/entry/plastic-surgery-justin-bieber-100k_n_4128563", + "X-GEO-URL-IN": "http://www.huffingtonpost.in/entry/plastic-surgery-justin-bieber-100k_n_4128563", + "X-GEO-URL-MX": "http://www.huffingtonpost.com.mx/entry/plastic-surgery-justin-bieber-100k_n_4128563", + "X-GEO-URL-ZA": "http://www.huffingtonpost.co.za/entry/plastic-surgery-justin-bieber-100k_n_4128563", + "X-HP-Trace-ID": "O01eycCn", + "X-HP-Trace-Project": "HPMW/production/70604bb", + "X-Mobile-URL": "http://m.huffpost.com/us/entry/4128563?utm_hp_ref=mostpopular", + "X-Request-Id": "554897a0-6399-48e8-9af9-33dccdee3e10", + "X-Runtime": "0.071822", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "http://www.huffingtonpost.com/2013/10/19/plastic-surgery-justin-bieber-100k_n_4128563.html?utm_hp_ref=mostpopular" +} \ No newline at end of file diff --git a/test/testdata/d3d78cb6c1e06d19e38192affa52aba0b52bc721.html b/test/testdata/d3d78cb6c1e06d19e38192affa52aba0b52bc721.html new file mode 100644 index 00000000..a7e9bc0f --- /dev/null +++ b/test/testdata/d3d78cb6c1e06d19e38192affa52aba0b52bc721.html @@ -0,0 +1,3656 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Man Undergoes Extensive Plastic Surgery To Look Like Justin Bieber, Spends $100,000 In 5 Years + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + + + + + +
          + + + + +
          +
          + + + +
          + + + + + + + + + + + + + + + + + + + + + +
          + + + + + +
          + + + +
          + + iOS app + Android app + + More +
          + +
          +
          + +
          + + + + + +
          + + + +
          +
          + +
          + +
          + +
          + +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + +
          +
          + + +
          + + +
          +
          + + + +
          + + +
          + +
          + + + +
          + + + + + + + + + + +
          + +
          + + +
          +
          +
          + + + + + + + + + + + + + + + + + + + +
          + + + +
          + + + + + + + + + + + + + +
          +
          +

          + Man Undergoes Extensive Plastic Surgery To Look Like Justin Bieber, Spends $100,000 In 5 Years +

          + +
          + +

          + The Huffington Post +  |  + + + + + + + Posted:   |  Updated: 10/21/2013 11:18 am EDT + + + + + + +

          + + +
          + +
          +
          + +
          + +
          + +
          +
          + + + +
          + + + + + + + + + + + + + + + + + + +
          + + + +

          Not only is songwriter Toby Sheldon a 33-year-old Justin Bieber fan, his Bieber-devotion would blow teeny-bopping Beliebers out of the water.

          + +

          That's because Sheldon went so far as to spend almost $100,000 on five years worth of plastic surgery to make him look like his idol, according to the British tabloid Closer.

          + +

          This is what Sheldon looks like after all that plastic surgery he had done to resemble the 19-year-old Canadian crooner:

          + +

          + + + + +

          On top of Botox injections and hair transplants, Sheldon had costly "smile surgery" done to make his smile look just like Bieber's, according to multiple reports.

          + +

          "It's Justin's smile that gives him his youthful look. So I had my upper lip lifted [and] my bottom lip plumped out," the musician told Closer.

          + +

          (h/t Reddit)

          + + +
          +

          Also on HuffPost:

          +
          +
          Loading Slideshow...
          +
          • Justin Bieber performs on the street August 20, 2007 in Stratford, Canada. (Photo by Irving Shuter/Getty Images)

            Justin Bieber performs on the street August 20, 2007 in Stratford, Canada. (Photo by Irving Shuter/Getty Images)

          • Justin Bieber Visits The Nintendo World Store - September 1, 2009

            Justin Bieber visits the Nintendo World Store on September 1, 2009 in New York City.

          • Justin Bieber Performs On ABC's "Good Morning America"

            NEW YORK - NOVEMBER 15: Musician Justin Bieber performs on ABC's 'Good Morning America' at ABC News' Good Morning America Times Square Studio on November 15, 2009 in New York City. (Photo by Neilson Barnard/Getty Images)

          • Z100's Jingle Ball 2009 - Press Room

            NEW YORK - DECEMBER 11: Singer Justin Bieber attends Z100's Jingle Ball 2009 at Madison Square Garden on December 11, 2009 in New York City. (Photo by Jason Kempin/Getty Images)

          • KIIS FM's Wango Tango 2010 - Arrivals

            LOS ANGELES, CA - MAY 15: Justin Bieber arrives at KIIS FM's Wango Tango 2010 at the Staples Center on May 15, 2010 in Los Angeles, California. (Photo by Angela Weiss/Getty Images)

          • KIIS-FM Presents Justin Bieber At Nokia Plaza L.A. LIVE

            LOS ANGELES, CA - FEBRUARY 13: Singer Justin Bieber poses after his free performance presented by KIIS-FM at Nokia Plaza L.A. Live on February 13, 2010 in Los Angeles, California. (Photo by Angela Weiss/Getty Images)

          • Justin Bieber Signs Copies Of "First Step 2 Forever" - November 26, 2010

            NEW YORK - NOVEMBER 26: Justin Bieber promotes his new book 'First Step 2 Forever' at Barnes & Noble, 5th Avenue on November 26, 2010 in New York City. (Photo by Andy Kropa/Getty Images)

          • 68th Annual Golden Globe Awards - Arrivals

            BEVERLY HILLS, CA - JANUARY 16: Singer Justin Bieber arrives at the 68th Annual Golden Globe Awards held at The Beverly Hilton hotel on January 16, 2011 in Beverly Hills, California. (Photo by Frazer Harrison/Getty Images)

          • Justin Bieber: Never Say Never - Inside Arrivals

            LONDON, ENGLAND - FEBRUARY 16: Justin Bieber attends the 'Justin Bieber: Never Say Never' premiere at the O2 Cineworld on February 16, 2011 in London, England. (Photo by Jon Furniss/WireImage)

          • 2011 Billboard Music Awards - Press Room

            LAS VEGAS, NV - MAY 22: Singer Justin Bieber poses in the press room with the Digital Artist of the Year award during the 2011 Billboard Music Awards at the MGM Grand Garden Arena May 22, 2011 in Las Vegas, Nevada. (Photo by Isaac Brekken/Getty Images)

          • 2011 CMT Music Awards - Audience And Backstage

            NASHVILLE, TN - JUNE 08: Singer Justin Bieber attends the 2011 CMT Music Awards at the Bridgestone Arena on June 8, 2011 in Nashville, Tennessee. (Photo by Kevin Mazur/WireImage)

          • BET Awards '11 - Arrivals

            LOS ANGELES, CA - JUNE 26: Musician Justin Bieber arrives at the BET Awards '11 held at the Shrine Auditorium on June 26, 2011 in Los Angeles, California. (Photo by Jason Merritt/Getty Images)

          • 2011 Teen Choice Awards - Arrivals

            UNIVERSAL CITY, CA - AUGUST 07: Singer Justin Bieber arrives at the 2011 Teen Choice Awards held at Gibson Amphitheatre on August 7, 2011 in Universal City, California. (Photo by Jon Kopaloff/FilmMagic)

          • 2011 MTV Video Music Awards - Arrivals

            LOS ANGELES, CA - AUGUST 28: Justin Bieber arrives at the 2011 MTV Video Music Awards at the Nokia Theatre L.A. Live on August 28, 2011 in Los Angeles, CA. (Photo by Gregg DeGuire/FilmMagic)

          • The 53rd Annual GRAMMY Awards - Arrivals

            LOS ANGELES, CA - FEBRUARY 13: Singer Justin Bieber arrives at The 53rd Annual GRAMMY Awards held at Staples Center on February 13, 2011 in Los Angeles, California. (Photo by Jason Merritt/Getty Images)

          • Justin Bieber Lights The Empire State Building

            NEW YORK, NY - NOVEMBER 18: Justin Bieber poses on the observation deck at The Empire State Building on November 18, 2011 in New York City. (Photo by Cindy Ord/Getty Images)

          • "Justin Bieber: Never Say Never" - New York Premiere

            NEW YORK, NY - FEBRUARY 02: Justin Bieber attends the New York premiere of 'Justin Bieber: Never Say Never' at Regal E-Walk 13 on February 2, 2011 in New York City. (Photo by Michael Loccisano/Getty Images)

          • Dolce & Gabbana Celebrates Fashion's Night Out

            NEW YORK, NY - SEPTEMBER 08: Justin Bieber attends the Dolce & Gabbana Boutique on September 8, 2011 in New York City. (Photo by Eugene Gologursky/Getty Images for Dolce & Gabbana)

          • The 40th American Music Awards - Arrivals

            LOS ANGELES, CA - NOVEMBER 18: Singer Justin Bieber attends the 40th American Music Awards held at Nokia Theatre L.A. Live on November 18, 2012 in Los Angeles, California. (Photo by Jason Merritt/Getty Images)

          • 2012 Teen Choice Awards - Arrivals

            UNIVERSAL CITY, CA - JULY 22: Justin Bieber arrives at the 2012 Teen Choice Awards at Gibson Amphitheatre on July 22, 2012 in Universal City, California. (Photo by Steve Granitz/WireImage)

          • Teen Choice Awards 2012 - Show

            UNIVERSAL CITY, CA - JULY 22: Singer Justin Bieber accepts the Male Summer Music Star award onstage during the 2012 Teen Choice Awards at Gibson Amphitheatre on July 22, 2012 in Universal City, California. (Photo by Kevin Winter/Getty Images)

          • 2012 MuchMusic Video Awards - Show

            TORONTO, ON - JUNE 17: Justin Bieber perfoms at the 2012 MuchMusic Video Awards at MuchMusic HQ on June 17, 2012 in Toronto, Canada. (Photo by George Pimentel/WireImage)

          • 2012 Teen Choice Awards - Red Carpet

            UNIVERSAL CITY, CA - JULY 22: Singer Justin Bieber arrives at the 2012 Teen Choice Awards at Gibson Amphitheatre on July 22, 2012 in Universal City, California. (Photo by Kevin Mazur/WireImage)

          • 2012 Billboard Music Awards - Arrivals

            LAS VEGAS, NV - MAY 20: Justin Bieber arrives at the 2012 Billboard Music Awards at MGM Grand on May 20, 2012 in Las Vegas, Nevada. (Photo by Gregg DeGuire/WireImage)

          • 2012 MuchMusic Video Awards - Arrivals

            TORONTO, ON - JUNE 17: Justin Bieber and guest arrives at the 2012 MuchMusic Video Awards at MuchMusic HQ on June 17, 2012 in Toronto, Canada. (Photo by George Pimentel/WireImage)

          • 2012 Nickelodeon's Kids' Choice Awards - Show

            LOS ANGELES, CA - MARCH 31: Singer Justin Bieber onstage at the 2012 Nickelodeon's Kids' Choice Awards at Galen Center on March 31, 2012 in Los Angeles, California. (Photo by Jeff Kravitz/FilmMagic)

          • NRJ Music Awards 2012 - Red Carpet Arrivals

            CANNES, FRANCE - JANUARY 28: Justin Bieber poses as he arrives at NRJ Music Awards 2012 at Palais des Festivals on January 28, 2012 in Cannes, France. (Photo by Pascal Le Segretain/Getty Images)

          • Justin Bieber performs at the O2

            DUBLIN, IRELAND - FEBRUARY 17: Justin Bieber performs at the O2 on February 17, 2013 in Dublin, Ireland. (Photo by Phillip Massey/WireImage)

          • Celebrity Sightings In London - February 19, 2013

            LONDON, UNITED KINGDOM - FEBRUARY 19: Justin Bieber sighting at Beat Club on February 19, 2013 in London, England. (Photo by Alan Chapman/FilmMagic)

          + + + +
          + + + + + +
          +
          + + + +
          +
          +
          + +
          +
          +
          + +
          +
          +
          +
          + FOLLOW CELEBRITY
          +
          + +
          + + +
          +
          + + +
          + + +
          +
          + + + +
          + + +
          +

          From our partners

          +
          +
          + + + + + + + + + + + + + + +
          +
          + +
          +
          +
          + + +
          + + +
          + + + +
          +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + +
          + + + +
          +
          + + + +
          + + + + + + + +
          +
          + +
           
          + +
          + + + + + + + + +  + +
          + +
          +
          +
          +
          + + + + + + + + + + +
          + + +
          + + + + + + + + +
          + +
          + + + + + + + + + + + + + + + + +
          +
          + +
          + +
          + + + +
          + + + + + +
          +
          + +
          +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          +
          +
          + + + + + + + + + + + + + + + + + diff --git a/test/testdata/d3d78cb6c1e06d19e38192affa52aba0b52bc721.json b/test/testdata/d3d78cb6c1e06d19e38192affa52aba0b52bc721.json new file mode 100644 index 00000000..d352847e --- /dev/null +++ b/test/testdata/d3d78cb6c1e06d19e38192affa52aba0b52bc721.json @@ -0,0 +1,28 @@ +{ + "encoding": "utf-8", + "headers": { + "Connection": "close", + "Content-Encoding": "gzip", + "Content-Length": "46851", + "Content-Type": "text/html; charset=utf-8", + "Date": "Wed, 24 May 2017 08:14:20 GMT", + "Link": "; rel=\"original\", ; rel=\"timemap\"; type=\"application/link-format\", ; rel=\"timegate\", ; rel=\"first memento\"; datetime=\"Sun, 20 Oct 2013 06:56:21 GMT\", ; rel=\"prev memento\"; datetime=\"Sun, 20 Oct 2013 06:56:21 GMT\", ; rel=\"memento\"; datetime=\"Mon, 21 Oct 2013 23:04:44 GMT\", ; rel=\"next memento\"; datetime=\"Thu, 24 Oct 2013 16:48:47 GMT\", ; rel=\"last memento\"; datetime=\"Mon, 06 Mar 2017 11:33:12 GMT\"", + "Memento-Datetime": "Mon, 21 Oct 2013 23:04:44 GMT", + "Server": "Tengine/2.1.0", + "Vary": "Accept-Encoding", + "X-Archive-Guessed-Charset": "UTF-8", + "X-Archive-Orig-cache-control": "max-age=32", + "X-Archive-Orig-connection": "close", + "X-Archive-Orig-content-length": "186928", + "X-Archive-Orig-date": "Mon, 21 Oct 2013 23:04:44 GMT", + "X-Archive-Orig-p3p": "CP='NO P3P'", + "X-Archive-Orig-server": "Apache", + "X-Archive-Playback": "0", + "X-Cache": "MISS from google.com", + "X-Cache-Lookup": "MISS from google.com:86", + "X-Page-Cache": "HIT", + "X-location": "All" + }, + "status_code": 200, + "url": "http://web.archive.org/web/20131021230444/http://www.huffingtonpost.com/2013/10/19/plastic-surgery-justin-bieber-100k_n_4128563.html?utm_hp_ref=mostpopular" +} \ No newline at end of file diff --git a/test/testdata/d3dde2e9b421666deb05a8a9553549ddbd91a413.html b/test/testdata/d3dde2e9b421666deb05a8a9553549ddbd91a413.html new file mode 100644 index 00000000..c8895e05 --- /dev/null +++ b/test/testdata/d3dde2e9b421666deb05a8a9553549ddbd91a413.html @@ -0,0 +1,200 @@ + + + + +[1608.05006] Automaticity in Computation and Student Success in Introductory Physical + Science Courses + + + + + + + + + + + + + + + + + + + + + + + +
          + + + +
          +
          + +
          +Full-text links: +

          Download:

          + + +
          + +
          +

          Current browse context:

          +
          physics.ed-ph
          + +

          Change to browse by:

          + +
          +
          +

          References & Citations

          + +
          +
          +
          +

          Bookmark

          (what is this?) +
          +CiteULike logo +BibSonomy logo +Mendeley logo +del.icio.us logo +Digg logo +Reddit logo +ScienceWISE logo + +
          +
          + +
          +
          +

          Physics > Physics Education

          +
          +

          Title: +Automaticity in Computation and Student Success in Introductory Physical Science Courses

          + + +
          +Abstract: Between 1984 and 2011, the percentage of US bachelor degrees awarded in +physics declined by 25%, in chemistry declined by 33%, and overall in physical +sciences and engineering fell 40%. Data suggest that these declines are +correlated to a deemphasis in most states of practicing computation skills in +mathematics. Analysis of state standards put into place between 1990 and 2010 +find that most states directed teachers to deemphasize both memorization and +student practice in computational problem solving. Available state test score +data show a significant decline in student computation skills. In recent +international testing, scores for US 16 to 24 year olds in numeracy finished +last among 22 tested nations in the OECD. Recent studies in cognitive science +have found that to solve well-structured problems in the sciences, students +must first memorize fundamental facts and procedures in mathematics and science +until they can be recalled with automaticity, then practice applying those +skills in a variety of distinctive contexts. Actions are suggested to improve +US STEM graduation rates by aligning US math and science curricula with the +recommendations of cognitive science. +
          + +
          + + + + + + + + + + + + + + + + + +
          Comments: +26 pages, 5 figures
          Subjects: +Physics Education (physics.ed-ph)
          +Cite as: +arXiv:1608.05006 [physics.ed-ph]
           (or arXiv:1608.05006v2 [physics.ed-ph] for this version)
          +
          +
          +

          Submission history

          +From: JudithAnn Hartman [view email] +
          +[v1] Wed, 17 Aug 2016 16:07:57 GMT (730kb)
          +[v2] Tue, 27 Sep 2016 13:54:01 GMT (430kb)
          +
          + +
          +
          +
          + + + diff --git a/test/testdata/d3dde2e9b421666deb05a8a9553549ddbd91a413.json b/test/testdata/d3dde2e9b421666deb05a8a9553549ddbd91a413.json new file mode 100644 index 00000000..15befa44 --- /dev/null +++ b/test/testdata/d3dde2e9b421666deb05a8a9553549ddbd91a413.json @@ -0,0 +1,20 @@ +{ + "encoding": "utf-8", + "headers": { + "Connection": "Keep-Alive", + "Content-Encoding": "gzip", + "Content-Length": "3980", + "Content-Type": "text/html; charset=utf-8", + "Date": "Thu, 01 Jun 2017 05:16:38 GMT", + "ETag": "\"Wed, 28 Sep 2016 00:06:36 GMT\"", + "Expires": "Fri, 02 Jun 2017 00:00:00 GMT", + "Keep-Alive": "timeout=8, max=100", + "Last-Modified": "Wed, 28 Sep 2016 00:06:36 GMT", + "Server": "Apache", + "Set-Cookie": "browser=151.246.252.117.1496294198851786; path=/; max-age=946080000; domain=.arxiv.org", + "Strict-Transport-Security": "max-age=31536000", + "Vary": "Accept-Encoding,User-Agent" + }, + "status_code": 200, + "url": "https://arxiv.org/abs/1608.05006?utm_medium=email&utm_source=other&utm_campaign=opencourse.GdeNrll1EeSROyIACtiVvg.announcements%257Eopencourse.GdeNrll1EeSROyIACtiVvg.4xDVKzx5EeeJjRJrkGD1dA" +} \ No newline at end of file diff --git a/test/testdata/d54cb7f41c44b9401cfe38bafce2558e96bc967b.html b/test/testdata/d54cb7f41c44b9401cfe38bafce2558e96bc967b.html new file mode 100644 index 00000000..fc0c91de --- /dev/null +++ b/test/testdata/d54cb7f41c44b9401cfe38bafce2558e96bc967b.html @@ -0,0 +1,1699 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Traffic lights: There’s a better way | MIT News + + + + + + + + + + + + + + + + + + +
          +
          + +
          + + +
          + +
          +
          + + + +
          + +
          +
          + + + + + + + + + +
          +
          + +
          + +
          +
          • +
            +
            +

            Photo: Jose-Luis Olivares/MIT

            Full Screen
            +
          • +
          • +
            +
            This figure shows two maps with colored lines that represent the main roads in Lausanne, Switzerland. The three colors represent how long it takes to commute: red is the longest commute, yellow is average, and green is the shortest commute. The left map, with conventional traffic light programming, has many red lines that represent long commutes. The right map, which uses the researcher's improved system, has many green lines that represent short commutes.
+
            +

            This figure shows two maps with colored lines that represent the main roads in Lausanne, Switzerland. The three colors represent how long it takes to commute: red is the longest commute, yellow is average, and green is the shortest commute. The left map, with conventional traffic light programming, has many red lines that represent long commutes. The right map, which uses the researcher's improved system, has many green lines that represent short commutes. +

            Image courtesy of the researchers

            Full Screen
            +
          • +
          + + + + + +
          +
          + +
          +
          + +
          + +

          Traffic lights: There’s a better way

          + + +

          MIT researchers develop an improved system for timing of urban lights to minimize commuting times. + +

          +
          +
          +
          + +
          + +
          + +
          +
          +

          + +
          + +

          +
          + + +
          + +
          +
          +

          Press Contact

          +

          Andrew Carleen
          Email: expertrequests@mit.edu
          Phone: 617-253-1682
          MIT News Office

          +
          +
          +

          Media Resources

          +

          2 images for download

          +

          Access Media

          +

          + Media can only be downloaded from the desktop version of this website. +

          +
          +
          + + + +
          +
          +
          + + +
          + + +

          Anyone who has ever driven a city street and been frustrated by having to stop again and again for red lights has probably thought that there must be a better way. Now, researchers at MIT have developed a means of computing optimal timings for city stoplights that can significantly reduce drivers’ average travel times.

          + +

          Existing software for timing traffic signals has several limitations, says Carolina Osorio, an assistant professor of civil and environmental engineering at MIT. She is lead author of a forthcoming paper in the journal Transportation Science that describes the new system, based on a study of traffic in Lausanne, Switzerland.

          + +

          “Usually in practice, when you want to time traffic lights, traditionally it’s been done in a local way,” Osorio says. “You define one intersection, or maybe a set of intersections along an arterial, and you fine-tune or optimize the traffic lights there. What is less done, and is more difficult to do, is when you look at a broader scale, in this case the city of Lausanne, and you want to change signal times at intersections distributed across the entire city, with the objective of trying to improve conditions across the entire city.”

          + +

          Such an expansive aim triggers complications, such as the ripple effect that a change at one intersection can produce across the surrounding area, or changes in driver behavior following changes in traffic-light patterns: For example, if wait times on a particular route increase, drivers may seek alternative routes that feature fewer red lights.

          + +

          The new optimization process developed by Osorio and graduate student Linsen Chong can time traffic lights in large urban areas while accounting for the complex and diverse reactions of individual drivers. Their approach uses high-resolution traffic simulators that describe, in detail, the behavior of drivers in response to changes in travel conditions.

          + +

          In detailed simulations of Lausanne’s traffic, they found that the timings produced by their approach reduced the average travel time for commuters by 22 percent, compared with timings generated by commercial traffic-light timing software.

          + +

          Some cities currently make use of these high-resolution simulators, known as microscopic simulators: Behavior down to the level of individual drivers is simulated to estimate the impact of a given timing pattern. But the complexity of such models makes them computationally intensive. For instance, in the case of Lausanne, more than 12,000 individual drivers are simulated.

          + +

          The new approach allows these models to be used in a practical and computationally efficient way. Other citywide models can be used to help determine proposed timings, but they treat traffic flow simplistically and homogeneously, rather than as a collection of individual travelers with distinct and complex behavior.

          + +

          The new simulation-based optimization model proposed by Osorio and Chong aims to bridge these options, providing a detailed vehicle-level analysis but applying it to city-scale optimization.

          + +

          The system, Osorio says, starts with a modest premise: “What if we combine information from these microscopic simulations with [citywide] information from these simple traffic models that are very computationally efficient and run instantly, but have very low resolution?” The approach combines the accuracy of high-resolution models with the computational efficiency of low-resolution traffic models.

          + +

          The basic system, Osorio says, is also being applied toward different goals: Instead of just minimizing commuting times, it is also being used to minimize fuel consumption, and even to determine the optimal location for services such as vehicle-sharing hubs.

          + +

          The work is currently being extended to help in the design of timing systems that can adapt to changing traffic conditions. Work on this topic is ongoing in collaboration with officials in New York City’s Department of Transportation, focusing on peak-period traffic in areas of Manhattan.

          + +

          That agency’s Mohamad Talas, a deputy director of system engineering who was not involved in the research but is working with the MIT team on testing, says, “Such a model can validate our active traffic-management system in Manhattan, and allow us to fine-tune our processes and improve the network operation.”

          + +

          Talas adds, “I believe that this approach is economically viable, with cost savings for any jurisdiction that needs to assess and improve traffic conditions for a large area of the transportation network.”

          +
          +
          + +
          +
          +

          Topics: Cities, Urban studies and planning, Traffic management, Computer science and technology, Transportation, Automobiles, Civil and environmental engineering, School of Engineering, Research

          +
          +
          + + +
          +
          +
          +

          Comments

          + + +
          + + + +
          +
          + +
          +

          The problem with such an approach for reducing car based commuting time in a city is that it will attract more traffic, until the very level that is prevalent right now (level of commuting time) is reached again. Cities do not have commuting times that are based on how well the traffic is guided. Cities have commuting times that are based on the local average commuter's individual will on how long to sit in a vehicle to commute. If it becomes easier, more people will fill in the resulting spaces because they will realize "hey this use to be 1 hour and now it's just 40 minutes, I'm ok with that" and they use their cars, buy houses/rent places further out, until all is back to the very same levels of wasting time (by commuting) as it is right now. It isn't the traffic light software. It's us.

          +
          + +
          +
          + +
          + + + +
          +
          + +
          +

          Optimizing traffic flow can really only be done in one direction at any one time since the timing of the lights for north bound traffic directly interferes with the traffic flow south bound and the cross town traffic. and traffic does not have constant volumes. and distances between lights are not constant ( even in cities with primarily a grid layout )

          +
          + +
          +
          + +
          + + + +
          +
          + +
          +

          What is so special with this work except a shiny re-branding with MIT's name? Using microsimulator as the black box performance
          evaluator for some (or whatever) objective functions, then find out the "best" timing plans that optimizes the objective function. This is nothing but moot academic exercises. These type of ideas have been there for half century and tons of papers out there. As a seasoned traffic engineer I cannot help but just chuckle for this type of academic exercise that knows little about how real-traffic controller works and how to perform signal optimization in real-life, except just putting together some hype term and start playing numbers in paper.

          +
          + +
          +
          + +
          + + + +
          +
          + +
          +

          interesting to watch streets all over taiwan at evening rush hour when police come and cycle lights manually depending on traffic

          +
          + +
          +
          + +
          + + + +
          +
          + +
          +

          Optimization of signal timing for urban street networks is a more complex task than the simplified simulation models often used for academic research. For example, do these simulation models take into account that there are pedestrians that need adequate time to cross wide streets, there are buses and LRT routes on the same streets. Then there are emergency vehicles that come through and disrupt any kind of optimizaiton routine that is being run. Now, if someone has a areawide microscopic traffic simulation model that can realistically account for all that, and is also tied to a regional travel demand model I will take a look at that. Otherwise, this too far removed from reality for any traffic engineer to take it seriously. However, I think efforts like this are good simplified exercises for training future traffic engineers in graduate schools.

          +
          + +
          +
          + +
          + + + +
          +
          + +
          +

          Robotic cloud connected cars are the ultimate solution and will make traffic lights obsolete in a few decades. Automated taxi service will be the solution to commuters, parkings and urban mobility.

          +
          + +
          +
          + +
          + + + +
          +
          + +
          +

          The author assumes that city officials want to minimize commuting times. Quite to the contrary. In cities like Boston the goal is to reduce the number of cars in the city by making driving as uncomfortable as possible. To this end, many streets in downtown Boston have a light at the next intersection turn red as soon one light turns green. There are times when many seconds go by with not a pedestrian in sight and no car moving in any direction.

          +
          + +
          +
          + +
          + + + +
          +
          + +
          +

          Only ordinary motorists are mentioned in the article, with the goal of minimizing commute times. One commenter suggested timing signals to discourage motorized travel -- though that might also, I'd suggest, increase pollution and
          congestion. Pedestrians, buses and emergency vehicles are mentioned in the comments, so far. So, I'll comment about bicyclists, who are generally slower than motorists except during times of congestion.

          +

          Timing traffic signals on some streets for typical bicyclist speed (say, 15 mph, but dependent on slope) can slow and calm motor traffic and establish preferred bicycle routes, also desirable design goals. This is done in Portland, Oregon, among other cities. Installing a special bicycle signal so bicyclists can merge into position for through travel or a left turn at the next intersection when a street is clear of motor traffic also is a valid design goal. Placing bicyclists on a separate bikeway in the street corridor requires special signals to avoid conflict with turning motor traffic, and inherently results in reduced throughput for either or both, see for example the discussion here: http://john-s-allen.com/blog/?....

          +

          A sufficiently sophisticated analysis would account for the various design goals, some of which are in conflict with one another, and optimize accordingly. Determining what is considered optimal involves political as well as engineering decisions.

          +

          John S. Allen, MIT '75

          +
          + +
          +
          + +
          + + + +
          +
          + +
          +

          Some areas already have systems that do this. For example, in southeast
          Denver, East Arapahoe Rd, east of the freeway, and connecting roads
          north thereof, towards Aurora. Visiting in July 2013, i found that EVERY
          traffic signal in the area was timed to stop me just as I got to it.
          For days straight, it NEVER failed. I had to wonder, is there a special
          transmitter in my car, or do they do this to every car here? And Why?
          Pull out fast or pull out slow, stop at the next signal. 10 mph below
          the speed limit or 10 mph over the speed limit, stop at the next signal.
          (Seeing this, they'll probably mail me a ticket.) See a red light
          ahead, so slow down a bit to avoid stopping; it turns green; speed up,
          and it turns red as I get to it - DAMN! This is the new, moneyed side of
          town - is this a deterrent against outsiders even to tour or even visit
          the area? In the tireder, poorer northwest of the city (demonstrated by
          cheap motels and a rare closed McDonalds), the traffic lights did not
          jam me up the same way. Yes I stopped, but it seemed about typical luck
          basis, not Every Damn Time.

          +
          + +
          +
          + +
          + + + +
          +
          + +
          +

          We have known of most of these possibilities since the 1950s. We have had computers capable of helping since the 1970s. Some cities and suburbs have major computerized traffic signal coordination.
          The dream improved when wireless emerged: If cars can declare their intended routes, the system can use that information to improve timing and economy for everyone. If cars can "listen" for advice, the system can suggest small adjustments to speed and route, yielding even better improvement. Safeguards are needed: Jamming can remove benefits. Hacking can cause huge problems (sabotage). Adding optical signalling (line-of-sight infrared) in parallel with radio could add robustness. (Optical jamming range is shorter.) The system can detect faulty GPS readings or reports, by their proximity to actual base stations.
          Additional benefits are possible if cars can talk to each other, but cars are more hackable than "the system", so vastly more caution is needed (unbreakable encryption and identification, and central authentication with verification against a reputation blacklist). Cars could form ad-hoc wireless networks for connectivity to the system, greatly reducing infrastructure cost. (But the relayed signals must be impervious to man-in-the-middle attacks.)
          Car-to-car declarations of emergency stops can be especially dangerous if forged, and present an opposite danger if they are omitted when relied upon. A car should identify the cars in front of it and authenticate before being ready to trust an emergency declaration from it.

          +
          + +
          +
          + +
          + + + +
          +
          + +
          +

          Something crazy is going on on this site. Every comment and every edit gets "moderated" (okay), but moderation seems to damage many comments by inserting hard line-breaks that I did not put there. (Look at some comments here.)
          SOME kinds of editing can accidentally add hard line-breaks. It happens to me when I copy my text from Disqus and then paste it back in. Soft line-breaks that are there for display somehow become hard line-breaks in the entry. To avoid this, I paste the text into Notepad (actually Notepad2-Mod), Select-all, and then Cut, just to strip the clipboard text of formatting. Then, when I paste it into Disqus, spurious hard line-breaks don't happen.
          I don't think you edit every article, but I can't be sure you have not made some tiny edit. Whatever viewing or editing you do in the review process, please find a way not to mangle every posting that you look at or every posting that you adjust.
          When I edited one of my damaged comments on this page (to remove the spurious hard line-breaks), the edited comment went back to "Hold on, this is waiting to be approved by MIT News", treated like any new comment. (Even though a machine could easily have detected that I only made white-space changes, and let the existing approval apply to the changed version.)

          +
          + +
          +
          + +
          + + + +
          +
          + +
          +

          it is so sad that this country can't do anything right
          our bridges are on the cusp of collapse
          roads are full of potholes
          and so on
          infrastructure everywhere is falling apart

          +
          + +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          + + +
          + +
          +
          + +
          + + + + + +
          + + + + + diff --git a/test/testdata/d54cb7f41c44b9401cfe38bafce2558e96bc967b.json b/test/testdata/d54cb7f41c44b9401cfe38bafce2558e96bc967b.json new file mode 100644 index 00000000..a8249b4c --- /dev/null +++ b/test/testdata/d54cb7f41c44b9401cfe38bafce2558e96bc967b.json @@ -0,0 +1,21 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Cache-Control": "public, max-age=900", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Language": "en", + "Content-Length": "23069", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:54:57 GMT", + "Server": "Apache", + "Vary": "Accept-Encoding", + "X-Content-Type-Options": "nosniff, nosniff", + "X-Frame-Options": "SAMEORIGIN", + "X-Varnish": "1897445370", + "X-Varnish-Cache": "MISS" + }, + "status_code": 200, + "url": "http://news.mit.edu/2014/traffic-lights-theres-a-better-way-0707" +} \ No newline at end of file diff --git a/test/testdata/d6a8269edbc2432aca89834085372f12a0c1137e.html b/test/testdata/d6a8269edbc2432aca89834085372f12a0c1137e.html new file mode 100644 index 00000000..a339681a --- /dev/null +++ b/test/testdata/d6a8269edbc2432aca89834085372f12a0c1137e.html @@ -0,0 +1,1477 @@ + + + + + + European Space Agency picks Plato planet-hunting mission - BBC News + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + +
          +
          +
          + + + + + + +
          + +
          + + + Science & Environment + + + + +
          + +
          + + Science & Environment + + + + +
          + +
          +

          European Space Agency picks Plato planet-hunting mission

          + + + + + +
          +

          A telescope to find rocky worlds around other stars has been selected for launch by the European Space Agency's (Esa) Science Policy Committee.

          Known as Plato, the mission should launch on a Soyuz rocket in 2024.

          The observatory concept was chosen following several years of assessment in competition with other ideas.

          It is expected to cost Esa just over 600 million euros, although hardware contributions from member states will take this closer to a billion (£800m).

          Astronomers have so far found over 1,000 planets beyond our Solar System, but none as yet has been shown to be truly Earth-like in terms of its size and distance from a Sun similar to our own.

          The PLAnetary Transits and Oscillations of stars mission will look to change that.

          It will be tuned specifically to seek out rocky worlds orbiting in the "habitable zone" - the region around a star where water can keep a liquid state.

          "Plato will be our first attempt to find nearby habitable planets around Sun-like stars that we can actually examine in sufficient detail to look for life," said Dr Don Pollacco, the University of Warwick researcher who leads the Plato Science Consortium.

          "Nearly all the small transiting planets discovered so far have been beyond our technology to characterise. Plato will be a game-changer, allowing many Earth-like planets to be detected and confirmed and their atmospheres examined for signs of life.

          "Plato planets will allow us to develop and test theories of planet evolution, understanding the type of small planets in the Universe and the real frequency of Earth-like planets," he told BBC News.

          Plato is not really one telescope but rather a suite of 34 telescopes mounted on a single satellite.

          The intention is for this array to sweep about half the sky, to investigate some of its brightest and nearest stars.

          The observatory will monitor these stars for the tell-tale tiny dips in light that occur when planets move across their faces.

          An important part of this investigation will be to perform an intricate study of the host stars themselves, using their pulsations to probe their structure and properties.

          Such observations, referred to as asteroseismology, would provide key, complementary information for the proper characterisation of the rocky worlds.

          The mission will be led by Dr Heike Rauer at DLR, the German space agency.

          The key British hardware contribution will be the camera system that sits behind the telescope suite.

          This will incorporate 136 charge-coupled devices (CCDs) produced by the e2v company in Chelmsford, Essex. Just under a metre square and having 2.5 billion pixels, the CCD system will be the biggest ever flown in space.

          It seems certain also that the British arm of Airbus Defence and Space (formerly Astrium) will endeavour to lead the construction of the satellite.

          Plato should prove to be a good fit with other next-generation astronomical facilities.

          These will include the ground-based European Extremely Large Telescope (E-ELT), which will have a primary mirror some 39m in diameter. To be built in Chile, this giant should be operating by 2024, and will have the power to investigate the atmospheres of the Plato's newly discovered planets.

          Plato is the third medium-class launch opportunity to be offered under Esa's so-called Cosmic Vision programme, which defines the organisation's space science priorities.

          The first two to be selected were Solar Orbiter, a space telescope to study the Sun, to launch in 2017; and Euclid, a telescope to investigate "dark energy", to fly in 2020.

          Esa will now refine the final design of Plato and select the industrial prime contractor.

          In addition, the agency's national member states must also agree any contributions they wish to make over and above their mandatory commitments.

          Once all this is done, the mission will be formally "adopted" - legal-speak for "final go-ahead". This should happen within the next two years.

          The unanimous selection of Plato by the SPC on Wednesday will be immensely pleasing to the team behind the Eddington space telescope - an Esa mission to find distant planets and do asteroseismology that was cancelled due to budget woes in the early 2000s.

          Jonathan.Amos-INTERNET@bbc.co.uk and follow me on Twitter: @BBCAmos

          +
          +
          +
          +

          Related Topics

          + +
          + + + +
          +

          More on this story

          + +
          +

          Related Internet links

          +

          The BBC is not responsible for the content of external Internet sites

          +
          + + + + + + +
          + +
          + + + + + +
          + + + + +
          + +
          + + + +
          + + + + + + + + + + + + + + + + + diff --git a/test/testdata/d6a8269edbc2432aca89834085372f12a0c1137e.json b/test/testdata/d6a8269edbc2432aca89834085372f12a0c1137e.json new file mode 100644 index 00000000..e214f50b --- /dev/null +++ b/test/testdata/d6a8269edbc2432aca89834085372f12a0c1137e.json @@ -0,0 +1,31 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "0", + "Cache-Control": "private, max-age=60, stale-while-revalidate", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Language": "en", + "Content-Length": "36497", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:53:01 GMT", + "Server": "Apache", + "Set-Cookie": "BBC-UID=aa9b657a75c33f37fce69ca4e8e9668a8c71fb7a7a09b2010a391f4a1f09d7a40Mozilla%2F5.0%20%28Windows%20NT%2010.0%3B%20Win64%3B%20x64%3B%20rv%3A50.0%29%20Gecko%2F20100101%20Firefox%2F50.0; expires=Sat, 22 May 2021 17:53:01 GMT; path=/; domain=.bbc.com", + "Vary": "Accept-Encoding", + "Via": "1.1 varnish", + "X-Cache": "MISS", + "X-Cache-Action": "MISS", + "X-Cache-Age": "0", + "X-Cache-Hits": "0", + "X-Fastly-Cache-Status": "MISS-CLUSTER", + "X-LB-NoCache": "true", + "X-News-Cache-Id": "22106", + "X-News-Data-Centre": "telhc", + "X-PAL-Host": "pal1103.back.live.telhc.local:80", + "X-Served-By": "cache-iad2632-IAD", + "X-Timer": "S1495561980.451061,VS0,VE984" + }, + "status_code": 200, + "url": "http://www.bbc.com/news/science-environment-26267918" +} \ No newline at end of file diff --git a/test/testdata/d73df4613d87d8a704e4e9228fdb3a44e8e1d12a.html b/test/testdata/d73df4613d87d8a704e4e9228fdb3a44e8e1d12a.html new file mode 100644 index 00000000..9d4e34e3 --- /dev/null +++ b/test/testdata/d73df4613d87d8a704e4e9228fdb3a44e8e1d12a.html @@ -0,0 +1 @@ +زندگی نامه علمی دکتر کاووس حسن لی - پایگاه مجلات تخصصی نورSkip to main content
          فهرست مقالات

          زندگی نامه علمی دکتر کاووس حسن لی

          (3 صفحه - از 17 تا 19)

          کلید واژه های ماشینی : ادبیات فارسی دانشگاه شیراز ، برنده‌ی جایزه‌ی کتاب برگزیده‌ی سال ، ادبی ، دانشگاه شیراز ، ادبیات ، زبان و ادبیات فارسی دانشگاه ، برنده‌ی جایزه‌ی کتاب سال سعدی ، بهار ، دانشکده‌ی ادبیات و علوم انسانی ، رنگ

          خلاصه ماشینی:

          "مقالات منتشر شده: - نشانه‌های فمینیسم در آثار سیمین دانشور (مشترک با دکتر سالاری)، مجله‌ی مطالعات زنان، سال پنجم، شماره‌ی 1، بهار و تابستان 1386، انتشار زمستان 1386. - بررسی عناصر زندگی معاصر در شعر سیمین بهبهانی، (مشترک با خانم مریم حیدری)، مجله‌ی علوم اجتماعی و انسانی دانشگاه شیراز، دوره‌‌ی بیست و پنجم، شماره‌ی سوم، پاییز 1385 (پیاپی)، انتشار در تابستان 1386. - ویژگی‌های شعر عاشورایی، (مشترک با دکتر کافی)، فصلنامه‌ی علمی پژوهشی علوم انسانی دانشگاه الزهرا، سال شانزدهم و هفدهم، شماره 61 و 62 زمستان 85 و بهار 86. - پرسش‌های حیرت‌آلود خیام چگونه پدید آمد، (مشترک با دکتر سعید حسام پور) مجله‌ی علوم اجتماعی و انسانی دانشگاه شیراز، دوره‌ی 22، شماره‌ی 3، پاییز 84. - قرینه‌گرایی خیام در رباعیات، (مشترک با دکتر سعید حسام‌پور) نشریه‌ی دانشکده‌ی ادبیات و علوم انسانی دانشگاه شهید باهنر کرمان، شماره‌ی 17 (پیاپی 14)، بهار 1384 (چاپ شده در زمستان 84). - زمان گذران در نگاه بی قرار خیام، (مشترک با دکتر سعید حسام‌پور) نشریه‌ی دانشکده‌ی ادبیات و علوم انسانی دانشگاه تبریز، سال 47، شماره مسلسل 192، پاییز 1383. - تحلیل رنگ در سروده‌های سهراب سپهری، (مشترک با دکتر مصطفی صدیقی) مجله‌ی دانشکده‌ی ادبیات و علوم انسانی دانشگاه باهنر کرمان، دوره‌ی جدید، ش‌ 13، (پیاپی 10)، بهار 1382. - پرتوی از ولی‌الله اعظم در منشور شعر فارسی، کتاب " ادبیات انقلاب، انقلاب ادبیات، (مجموعه مقالات) کنگره‌ی بررسی تأثیر امام خمینی و انقلاب اسلامی بر ادبیات معاصر)، جلد دوم، 1378، انتشارات مؤسسه تنظیم آثار امام، - میزان آزادی شاعر در آفرینش اثرش، نگاه پنج‌شنبه (ویژه‌نامه‌ی روزنامه‌ی خبر) 18 تیر1377، صص 1و2."

          برای مشاهده محتوای مقاله لازم است وارد پایگاه شوید. در صورتی که عضو نیستید از قسمت عضویت اقدام فرمایید.

          \ No newline at end of file diff --git a/test/testdata/d73df4613d87d8a704e4e9228fdb3a44e8e1d12a.json b/test/testdata/d73df4613d87d8a704e4e9228fdb3a44e8e1d12a.json new file mode 100644 index 00000000..cd23edf5 --- /dev/null +++ b/test/testdata/d73df4613d87d8a704e4e9228fdb3a44e8e1d12a.json @@ -0,0 +1,18 @@ +{ + "encoding": "utf-8", + "headers": { + "Cache-Control": "private", + "Content-Encoding": "deflate", + "Content-Length": "11338", + "Content-Type": "text/html; charset=utf-8", + "Date": "Fri, 13 Apr 2018 07:58:03 GMT", + "Set-Cookie": "__RequestVerificationToken=HZEUacSQyc1Bm-Ei1topq9tHwDinTeh7MoMN7_82gbeeroRE7mycLBJAB-7yxAg6Re4ka6O7i1sS05HdqQpxtB7fu55V5QgsyvVMxS2EmAI1; path=/; secure; HttpOnly", + "Strict-Transport-Security": "max-age=3153600", + "Vary": "Accept-Encoding", + "X-Download-Options": "noopen", + "X-Frame-Options": "SAMEORIGIN", + "X-XHTML-Minification-Powered-By": "WebMarkupMin" + }, + "status_code": 200, + "url": "https://www.noormags.ir/view/fa/articlepage/454096/%d8%b2%d9%86%d8%af%da%af%db%8c-%d9%86%d8%a7%d9%85%d9%87-%d8%b9%d9%84%d9%85%db%8c-%d8%af%da%a9%d8%aa%d8%b1-%da%a9%d8%a7%d9%88%d9%88%d8%b3-%d8%ad%d8%b3%d9%86-%d9%84%db%8c" +} \ No newline at end of file diff --git a/test/testdata/d7a6165e517c545de480afbd80093d20f8510548.html b/test/testdata/d7a6165e517c545de480afbd80093d20f8510548.html new file mode 100644 index 00000000..7ca6be76 --- /dev/null +++ b/test/testdata/d7a6165e517c545de480afbd80093d20f8510548.html @@ -0,0 +1,914 @@ + + + + + + + آیت‌الله محمدی گیلانی دارفانی را وداع گفت + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          +
          +
          + + + +
          +
          +
          +
          +
          +
          +
          +
            +
          • طلا، سکه و ارز
          • +
          • بورس
          • +
          • قیمت خودرو
          • +
          +
          + + + + +
          +
          + +
          +
          + + + سه شنبه ۰۲ خرداد ۱۳۹۶ - ۲۲:۲۴ + +
          +
          +
          +
          +
          +
          + + + +
          + + +
          + + + +
          +
          +
          + + + +
          +
          + +
          + + + + - +
          +
          +
          +
          +
          + + + + 93/04/18 :: 01:14 +
          +

          آیت‌الله محمدی گیلانی دارفانی را وداع گفت

          + خبرگزاری فارس: آیت‌الله محمدی گیلانی دارفانی را وداع گفت

          آیت‌الله محمدی گیلانی که از چندی پیش در یکی از بیمارستان‌های تهران بستری شده بود دارفانی را وداع گفت.

          +
          +

          + به گزارش خبرنگار حوزه احزاب خبرگزاری فارس، آیت‌الله محمد محمدی گیلانی که از 30 اردیبهشت ماه سال جاری به دلیل مشکل ریوی در بخش «آی.‌سی.یو» یکی از بیمارستان‌های تهران بستری شده بود، ساعاتی پیش دار فانی را وداع گفت.

          +

          +  آیت‌الله محمد محمدی گیلانی در سال 1307 در روستای دعوی سرا از توابع شهرستان رودسر در استان گیلان به دنیا آمد و در سال 1323 همزمان با تاسیس حوزه علمیه رودسر توسط سید محمد هادی روحانی به حوزه علمیه رودسر رفت و جامع المقدمات را فرا گرفت. در سال 1325 به حوزه علمیه قم رفت و به تکمیل دروس ادبیات و فراگیری دروس دوره سطح پرداخت.

          +

          + با پایان یافتن دوره سطح، به درس خارج آیت‌الله بروجردی راه یافت و در مدت 12سال کتاب صلوه را به اتمام رساند، به موازات آن در درس اصول امام خمینی (ره) نیز حضور پیدا کرد.

          +

          + آیت‌الله بروجردی، علامه سید محمدحسین طباطبایی و حضرت امام خمینی(ره) از استادان آیت الله محمدی گیلانی بودند و کتاب های قضا و قضاوت در اسلام، امامت و خلافت در کتاب و سنت، قرآن و سنن الهی در اجتماع بشر و ترجمه کتاب شفا ابوعلی سینا از آثار این استاد برجسته حوزه علمیه است.

          +

          + وی از مبارزان انقلاب اسلامی و از یاران نزدیک امام خمینی(ره) بود که پس از پیروزی انقلاب اسلامی ریاست دادگاه انقلاب، دیوان عالی کشور، نماینده مردم تهران در سه دوره مجلس خبرگان رهبری، عضویت در مجمع تشخیص مصلحت نظام و دبیری و عضویت در فقهای شورای نگهبان را بر عهده داشته است.

          +

          + همچنین در سال 1388 به پاس سال‌ها تلاش و مجاهدت ایشان در عرصه‌های مختلف قضا نشان درجه یک عدالت از سوی رئیس دولت دهم به وی اهدا شد.

          +

          + انتهای پیام

          + http://fna.ir/ +
          + + + + + + + + +
          +
          + + + +
          + +

          اخبار مرتبط

          آیت‌الله گیلانی از پشتیبانان اصلی نظام بودند93/04/18 - 13:25فردا در گیلان عزای عمومی اعلام شد93/04/18 - 12:46پیام تسلیت لاریجانی به ‌مناسبت درگذشت آیت‌الله محمدی گیلانی93/04/18 - 10:23نایب ‌رئیس مجلس ارتحال آیت‌الله محمدی گیلانی را تسلیت گفت93/04/18 - 10:18‌حجت‌الاسلام سیدحسن خمینی رحلت آیت‌الله محمدی گیلانی را تسلیت گفت93/04/18 - 10:16گیلان به سوگ آیت‌الله محمدی‌گیلانی نشست93/04/18 - 04:47تصاویر کمتر دیده شده آیت‌الله محمدی گیلانی93/04/18 - 03:00
          + + + + +
          + نظرات + +
          + دیدگاه های ارسال شده توسط شما، پس از تایید توسط خبرگزاری فارس در وب سایت منتشر خواهد شد
          + پیام هایی که حاوی تهمت یا افترا باشد منتشر نخواهد شد
          + پیام هایی که به غیر از زبان فارسی یا غیر مرتبط با خبر باشد منتشر نخواهد شد
          +
          + + +
          +
          + +
          + +
          +
          +
          +
          +
          + تنظیمات علاقه مندی های من +
          +
          +
            +
          • +
          • + +
          • +
          • +
          • +
          +
          انتخاب عناوین برتر خبری
          +
          +
          +
          + بازگشت +
            +
          • پربحث ترین ها
          • +
          • سیاسی
          • +
          • اقتصادی
          • +
          • اجتماعی
          • +
          • ورزشی
          • +
          • بین الملل
          • +
          • استانها
          • +
          • فرهنگی
          • +
          • دیدگاه
          • +
          • حماسه و مقاومت
          • +
          • فضای مجازی
          • +
          • صوت و تصویر
          • +
          +
          +
          +
          + +شبکه های اجتماعی + +
          + +
          + +
          + + +
          +
          + + + + + + + +
          +
          + + +
          + + + + + + + + + + diff --git a/test/testdata/d7a6165e517c545de480afbd80093d20f8510548.json b/test/testdata/d7a6165e517c545de480afbd80093d20f8510548.json new file mode 100644 index 00000000..7ef6893f --- /dev/null +++ b/test/testdata/d7a6165e517c545de480afbd80093d20f8510548.json @@ -0,0 +1,14 @@ +{ + "encoding": "utf-8", + "headers": { + "Cache-Control": "private", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 18:06:45 GMT", + "Server": "nginx", + "Transfer-Encoding": "chunked" + }, + "status_code": 200, + "url": "http://www.farsnews.com/newstext.php?nn=13930418000036" +} \ No newline at end of file diff --git a/test/testdata/daf34251f7b673ab6f1e247e65e566649d6ca10b.html b/test/testdata/daf34251f7b673ab6f1e247e65e566649d6ca10b.html new file mode 100644 index 00000000..42224618 --- /dev/null +++ b/test/testdata/daf34251f7b673ab6f1e247e65e566649d6ca10b.html @@ -0,0 +1,11 @@ + +@article{noormags692447, +title = { بررسی فضایل قرآنی در دعای ابوحمزه ثمالی }, +journal = { بینات (موسسه معارف اسلامی امام رضا علیه السلام) }, +number = { 68 }, +year = { 1389 }, +author = { +سلیمانی‌میمند,‌مریم and }, +pages = { 103 -- 124 }, +url = { http://www.noormags.ir/view/fa/articlepage/692447 } +} \ No newline at end of file diff --git a/test/testdata/daf34251f7b673ab6f1e247e65e566649d6ca10b.json b/test/testdata/daf34251f7b673ab6f1e247e65e566649d6ca10b.json new file mode 100644 index 00000000..61c941e8 --- /dev/null +++ b/test/testdata/daf34251f7b673ab6f1e247e65e566649d6ca10b.json @@ -0,0 +1,15 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "private", + "Content-Disposition": "attachment; filename=\"noormags-692447.bib\"", + "Content-Length": "400", + "Content-Type": "application/x-bibtex; charset=UTF-8", + "Date": "Tue, 23 May 2017 17:51:58 GMT", + "Set-Cookie": "CRCIS_SessionId=1c1vpirraw5y32dlazwizf3a; path=/, .ASPXBrowserOverride=Mozilla%2f4.0+(compatible%3b+MSIE+6.0%3b+Windows+CE%3b+IEMobile+8.12%3b+MSIEMobile+6.0); expires=Tue, 30-May-2017 17:51:58 GMT; path=/", + "X-Download-Options": "noopen", + "X-Frame-Options": "SAMEORIGIN" + }, + "status_code": 200, + "url": "http://www.noormags.ir/view/fa/citation/bibtex/692447" +} \ No newline at end of file diff --git a/test/testdata/dc06929cc58c970ec9dc62a5ca81ec1c502093de.html b/test/testdata/dc06929cc58c970ec9dc62a5ca81ec1c502093de.html new file mode 100644 index 00000000..0aafe4e3 --- /dev/null +++ b/test/testdata/dc06929cc58c970ec9dc62a5ca81ec1c502093de.html @@ -0,0 +1 @@ +[{"itemType":"book","title":"Münchenstein - Heimatkunde","oclc":"613273377","url":"https://www.worldcat.org/oclc/613273377","ISBN":["978-3-85673-522-7","3-85673-522-4"],"place":"[Liestal]","contributor":[["Walter","Ramseier"]],"accessDate":"2022-03-15","source":["WorldCat"]}] \ No newline at end of file diff --git a/test/testdata/dc06929cc58c970ec9dc62a5ca81ec1c502093de.json b/test/testdata/dc06929cc58c970ec9dc62a5ca81ec1c502093de.json new file mode 100644 index 00000000..4edd382c --- /dev/null +++ b/test/testdata/dc06929cc58c970ec9dc62a5ca81ec1c502093de.json @@ -0,0 +1,37 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "0", + "Connection": "keep-alive", + "NEL": "{ \"report_to\": \"wm_nel\", \"max_age\": 86400, \"failure_fraction\": 0.05, \"success_fraction\": 0.0}", + "Permissions-Policy": "interest-cohort=()", + "Report-To": "{ \"group\": \"wm_nel\", \"max_age\": 86400, \"endpoints\": [{ \"url\": \"https://intake-logging.wikimedia.org/v1/events?stream=w3c.reportingapi.network_error&schema_uri=/w3c/reportingapi/network_error/1.0.0\" }] }", + "Server-Timing": "cache;desc=\"pass\", host;desc=\"cp3056\"", + "Set-Cookie": "WMF-Last-Access=15-Mar-2022;Path=/;HttpOnly;secure;Expires=Sat, 16 Apr 2022 12:00:00 GMT, WMF-Last-Access-Global=15-Mar-2022;Path=/;Domain=.wikipedia.org;HttpOnly;secure;Expires=Sat, 16 Apr 2022 12:00:00 GMT, GeoIP=IR:09:Mashhad:36.30:59.59:v4; Path=/; secure; Domain=.wikipedia.org", + "Strict-Transport-Security": "max-age=106384710; includeSubDomains; preload", + "X-Cache": "cp3052 miss, cp3056 pass", + "X-Cache-Status": "pass", + "X-Client-IP": "31.14.148.11", + "access-control-allow-headers": "accept, content-type, content-length, cache-control, accept-language, api-user-agent, if-match, if-modified-since, if-none-match, dnt, accept-encoding", + "access-control-allow-methods": "GET,HEAD", + "access-control-allow-origin": "*", + "access-control-expose-headers": "etag", + "cache-control": "private, max-age=0, s-maxage=0, must-revalidate", + "content-length": "278", + "content-location": "https://en.wikipedia.org/api/rest_v1/data/citation/mediawiki/3-85673-522-4", + "content-security-policy": "default-src 'none'; frame-ancestors 'none'", + "content-type": "application/json; charset=utf-8", + "date": "Tue, 15 Mar 2022 14:17:03 GMT", + "referrer-policy": "origin-when-cross-origin", + "server": "restbase1018", + "vary": "Accept-Encoding", + "x-content-security-policy": "default-src 'none'; frame-ancestors 'none'", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-webkit-csp": "default-src 'none'; frame-ancestors 'none'", + "x-xss-protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://en.wikipedia.org/api/rest_v1/data/citation/mediawiki/3-85673-522-4" +} \ No newline at end of file diff --git a/test/testdata/dd5cd8d218b0879225d1a19f294ffdc59bf281a5.html b/test/testdata/dd5cd8d218b0879225d1a19f294ffdc59bf281a5.html new file mode 100644 index 00000000..2902bb0e --- /dev/null +++ b/test/testdata/dd5cd8d218b0879225d1a19f294ffdc59bf281a5.html @@ -0,0 +1,1489 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Hot Rod Stamps; Google on Road; A GM Prospectus + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + + + +
          +
          + + +
          +

          Hot Rod Stamps; Google on Road; A GM Prospectus

          +

          +
          + +
          +
          + +
          + + The new Forever hot rod stamps won’t move your first-class letters any faster, but it surely will give them a customized look.
          + The new Forever hot rod stamps won’t move your first-class letters any faster, but it surely will give them a customized look. + –US POSTAL SERVICE +
          + +
          + +
          +
          + + + + + +
          +

          The Beach Boys immortalized the 1932 Ford “Little Deuce Coupe’’ in their 1963 song and album of that name.

          +

          Now the United States Postal Service is paying homage to hot rodding by offering limited-edition stamps that feature a pair of 1932 Ford roadsters.

          +

          One stamp shows a black roadster with orange flames, and the other a red roadster.

          +

          “These Hot Rod stamps mark the beginning of America’s fascination with customizing fast cars,’’ says postmaster general Patrick Donahoe. *CQ*

          +

          To emphasize the enduring value of the Forever stamp, he added, “They’re just as popular today as they were decades ago. And just like the cars they celebrate, these stamps are timeless in that they’ll be good for mailing First-Class letters anytime in the future.’’

          +
          + Advertisement +
          +
          +
          +
          +
          +

          Like many aspects of pop culture, hot rodding took off in Southern California where dry lake beds north and east of Los Angeles were ideal spots for racing chopped and stripped street cars.

          +

          The ’32 Fords were considered the ideal car to turn into a hot rod. They were plentiful in the 1940s and ‘50s, relatively inexpensive, and had the powerful (for its time) flathead V-8 engine that Ford continued using through 1953.

          +

          Hot rodding pretty much led to drag racing, then to Hot Rod magazine, and the formation of the National Hot Rod Association (NHRA), which eventually became drag racing’s governing body.

          +

          Pete Petersen, who founded Hot Rod magazine and Motor Trend, produced an early hot rod car show at the Los Angeles Armory in 1948 and later founded the Petersen Automotive Museum in Los Angeles.

          +

          The USPS has been doing well by automotive aficionados in recent years. The 2013 muscle car stamps (GTO, Shelby GT-500, Dodge Charger Daytona, Plymouth Hemi ‘Cuda, and Chevelle SS) were still available recently.

          +

          In 2011, there was an Indy 500 centennial stamp, and in 2010 a Pixar series contained, among others, Lightning McQueen and his BFF, the animated (and droll) tow truck Mater.

          + +

          Ogling Google

          +

          Consumer Watchdog, a California non-profit consumer education and advocacy organization, is urging that state’s Department of Motor Vehicles to resist pressure from Google and other groups developing “driverless cars.’’

          +
          + Advertisement +
          +
          +
          +
          +
          +

          “We urge the DMV to follow a sensible and deliberate approach that would require adequate testing and time to analyze the results,’’ is the position of John Simpson, Consumer Watchdog’s director.

          +

          “The whole topic of safety system engineering and the so-called driverless or autonomous car is of worldwide interest,’’ he wrote.

          +

          However, there’s the reverse side of regulation. On one side is the need to protect citizens; on the other side they don’t want to hinder their technology industry’s attempts to be at the forefront of the automotive future.

          + +

          That’s a Wash

          +

          Even old-school parts of the industry are looking to the future.

          +

          Belmont Car Wash & Detailing in Waverly Square marked its 50th anniversary last Saturday (June 21) and is offering 1964 prices on the popular Soft Touch wash ($1, regularly $9.99) and Super Shine wash ($2., regularly $13.99). Those prices are in effect through July 4.

          +

          Owners Paul and Adam Tocci recently completed a massive renovation of the tunnel equipment and customer waiting area of the business their dad founded in 1964.

          + +

          GM Recall Numbers

          +

          By mid-June, the regularly updated graphic done by New York-based Mojomotors.com showed the number of vehicles involved in GM’s 2014 recalls had passed the 13 million mark. Jalopnik.com points out that is more cars than the company sold in the five-year period from 2009-2013. “It would seem that people just don’t care about recalls,’’ says Sam Jackson of Mojomotors.com.

          +

          He’s correct.

          +

          But, just walking around every day, you see people carrying massive key rings. That’s a sign folks should be concerned about GM’s ignition switch recall.

          +
          + Advertisement +
          +
          +
          +
          +
          +

          Tests show the switches are safe with just a single key being used, but most of us walk around with at least a few keys on a ring and usually with a weighty remote fob attached, too. It means GM car owners should get the part replaced because it’s the added weight that can switch the ignition to the “off’’ position.

          +

          GM does many things well, but for years now, even casual observers have to have noticed how the company regularly plays musical chairs along its executive row.

          +

          What that does for corporate memory, long-term sales and model planning, and accountability is obvious. When the music stops, the CEO of the moment is left holding the bag.

          +

          It’s not a new phenomenon.

          +

          Late in the fall of 2000, Oldsmobile sent a product rep to visit the New England Motor Press Association. His message was that the company had good cars in place in the Alero, Intrigue, and Bravada with newer models in the pipeline.

          +

          A month later, GM discontinued the entire Oldsmobile division.

          +

          In this case, it’s Mary Barra who is saddled with the job of cleaning up the recalls and messes left by her predecessors. As the stock sales pitches say, “Past performance is no guarantee of future results,’’ so let’s reverse it and say we hope “GM’s past failures aren’t an indication of future embarrassments.’’

          +

          Barra has said a lot of the right things about the “it’s not my problem’’ attitude at the company. If she can succeed in changing the corporate culture and have a lengthy run at the helm, it would be one of the great success stories in automotive history.

          + + +
          +
          +
          +

          Loading Comments...

          +
          +
          +
          + + +
          + +
          +
          +
          + + + + + + + +
          +
          + + This photo provided by Honda shows the 2017 Honda Civic Hatchback. Honda brings its first five-door Civic hatchback to the U.S. from England and packs a lot of fun and interior room into a spunky package. (Wesley Allison/Honda via AP) +
          +
          + + +
          +
          + + +
          +
          +
          + +
          +
          + + + +
          +
          + + 2017 Honda Accord Hybrid. +
          + +
          + + + + + +
          +
          + + MUSEUM-WORTHY WRECKS: Corvettes rescued from a 2014 sinkhole are on display at the National Corvette Museum in Bowling Green, KY, which also features Swamp Rat dragsters once driven by Don Garlits. +
          + +
          +
          +
          +
          +
          +
          + +
          +
          + + SMOOTH AS SILK: The clean lines of the Audi A6 secure this beauty’s place in the pageant of good looks. +
          + +
          + +
          +
          + + Sport Red 2017 Cascada Sport Touring with Dark Effects Package. +
          + +
          + +
          +
          + + This photo provided by Toyota shows the 2017 Toyota Sienna SE. Toyota’s Sienna van adds a more powerful and more fuel-efficient engine for 2017 and now ranks at the top in gas mileage ratings with competing family vans. (David Dewhurst Photography/Courtesy of Toyota via AP) +
          + +
          + +
          +
          + + 2017 Mazda Mazda6. +
          + +
          + + +
          +
          +
          +
          +
          + +
          +
          + + This undated photo made available by Volkswagen shows the 2015 Volkswagen Golf GTI. The new GTI shed up to 82 pounds from its predecessor's weight and now is just over 3,000 pounds. (Volkswagen via AP) +
          +
          + + +
          +
          + + + +
          +
          + + Anyone would jump to own this 2005 Porsche Carrera GT. +
          + +
          + + + + +
          +
          +
          +
          +
          + +
          +
          + +  +
          + +
          + + + + + + + + +
          +
          +
          +
          +
          + + + + + +
          +
          + + 2017 Ford Explorer. +
          + +
          + +
          +
          + + 2017 Subaru Forrester. +
          +
          + +
          + + + +
          + Cars
          + + + 'What's an easy vehicle to get into?' +
          + + + + April 30, 2017 | 5:00 AM +
          +
          +
          + + +
          +
          +
          +
          +
          +
          +
          +
          + +
          + + + +
          + + + + + + + + + + + + + + + + + +
          +
          +
          + +
          +
          +
          +
          +
          + +
          +
          + + + diff --git a/test/testdata/dd5cd8d218b0879225d1a19f294ffdc59bf281a5.json b/test/testdata/dd5cd8d218b0879225d1a19f294ffdc59bf281a5.json new file mode 100644 index 00000000..fdc687e6 --- /dev/null +++ b/test/testdata/dd5cd8d218b0879225d1a19f294ffdc59bf281a5.json @@ -0,0 +1,27 @@ +{ + "encoding": "UTF-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "0", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "51225", + "Content-Security-Policy": "upgrade-insecure-requests", + "Content-Type": "text/html; charset=UTF-8", + "Date": "Tue, 23 May 2017 17:53:03 GMT", + "Fastly-Debug-Digest": "211494f5142914f4c8f7b99f3f083093b7ae1a95d2e6fb881e7fa6330efd96a3", + "Fastly-SSL": "1", + "Link": "; rel=\"https://api.w.org/\", ; rel=shortlink", + "Server": "Apache", + "Vary": "Accept-Encoding, Origin,Fastly-SSL,Fastly-SSL", + "Via": "1.1 varnish, 1.1 varnish", + "X-Cache": "HIT, MISS", + "X-Cache-Hits": "1, 0", + "X-Pingback": "https://www.boston.com/xmlrpc.php", + "X-Served-By": "cache-jfk8144-JFK, cache-iad2150-IAD", + "X-TTL": "default", + "X-Timer": "S1495561983.158999,VS0,VE10" + }, + "status_code": 200, + "url": "https://www.boston.com/cars/news-and-reviews/2014/06/29/hot-rod-stamps-google-on-road-a-gm-prospectus" +} \ No newline at end of file diff --git a/test/testdata/dfdd0eef0531a736d132b2c1aed316ec40043924.html b/test/testdata/dfdd0eef0531a736d132b2c1aed316ec40043924.html new file mode 100644 index 00000000..b1e08737 --- /dev/null +++ b/test/testdata/dfdd0eef0531a736d132b2c1aed316ec40043924.html @@ -0,0 +1,2814 @@ + + + + + + + + + + + + + + + When Killer Whales Kill: Why the movie "Blackfish" Should Sink Captive Whale Programs | Annelise Sorg + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          +
          + + + + + + + + + + + + + + + + + + + +
          + + +
          + + + + + + + + + + + + + + + + + + + + +
          + +
          + + +
          + + + + + + + + + + + + + + + + + + + + + +
          + + + +
          + + +
          + +
          + +
          +
          +
          + + +
          + + + + + + + +
          + + Region: BC + + + + + + +
          + + + + +
          +
          +
          + +
          + + + + + + + +
          + +
          + + + + + + +
          +
          +
          + + + + + + +
          + + + + + + +
          + THE BLOG +

          + + Featuring fresh takes and real-time analysis from HuffPost's signature lineup of contributors

          +
          + +
          + + +
          + + +
          +
          + + + + +
          +
          + Annelise Sorg Headshot +
          + +
          + +

          When Killer Whales Kill: Why the movie "Blackfish" Should Sink Captive Whale Programs

          + +
          + +
          + + Posted: + + + Updated: + +
          +
          + + + +
          +
          + + + + + + + + Print + +
          +
          + + +
          +
          + + +
          +
          + +
          + +
          + +
          + Getty +
          Getty
          +
          +
          + + + +

          I watched the whole "Blackfish"movie with a big stupid grin on my face.

          I just couldn't help it. I smiled as I watched trainer after trainer get injured, even killed. I kept smiling watching whales drown during capture and hearing the desperate cries of helpless whale mothers as their babies were torn away from them -- not only in the wild, but also in captivity. I watched frustrated captive whales attacking each other and bleeding profusely. It was all so sad and cruel, but I just kept on smiling right through it all. I was aware that anybody in the theatre watching me would think I was a total psychopath who cared nothing about whales or people... But I couldn't stop smiling!

          The reason for my goofy grinning was that I know this whale tale all too well, and there it was unfolding beautifully for public consumption on the big screen. Remember Tilikum, the orca that was shipped from Victoria's Sealand to Orlando's SeaWorld after the death of a trainer? "Blackfish" ( which opens August 2 in Vancouver at the Vancity Theatre) shows us that wasn't the last tragedy associated with this killer whale.

          Sitting in the theatre, I couldn't help but think of the reaction that "Blackfish" will cause within the industry, the public and the government. And that made me smile. I figure if all goes well, "Blackfish" will sink SeaWorld's captive orca whale program, just like "Free Willy" sank the program at the Vancouver Aquarium.

          The reason I know this whale of a story so intimately is that I volunteer for the Vancouver-based registered non-profit society called No Whales In Captivity. I have helped organize hundreds of protests outside the Vancouver Aquarium, lobbied seven different elected boards of commissioners at the Vancouver Park Board, and done thousands of hours of media interviews, school chats and public presentations about the cruel practice of keeping whales in captivity.

          The first "Free Willy" movie was released in 1993. It told the story of a captive orca who is reunited with his family in the ocean. The amazing thing is that this fictional tale became a true story when Keiko, the "Free Willy" whale, was rehabilitated and released back to his family in the North Atlantic. (Watch out for the new documentary, "Keiko: The Untold Story".)

          When the movie "Free Willy" was released in Vancouver, our group was granted permission to set up display tables at all the movie theatres where it was screened. Volunteers collected many signatures on petitions and recruited new protestors at these screenings. Finally in 2001, after many years of protests and meetings, the Vancouver Aquarium closed the orca whale tank forever, shipping the last surviving orca called Bjossa to Sea World in San Diego. Bjossa died at SeaWorld four months later, alone and forgotten in a reserve tank.

          There are still two beluga whales and two dolphins left at the Vancouver Aquarium and we are asking that no more be imported to restock those tanks once these animals die or are sold to another facility. We are calling for a complete phase-out of whale exhibits in Stanley Park and we will continue to protest and lobby government until we stop the importation of new whales and dolphins.

          Orcas, belugas and dolphins are all cetaceans, which means that they all feel and suffer just like we do. Even the American Association for the Advancement of Science (AAAS) -- with a membership of 8,000 -- agrees that cetaceans should be given the right to live free of human intervention, and that includes no captivity for whales.

          However, the Vancouver Aquarium has other plans right now. Aquarium press releases have announced that bigger tanks are to be built and kept stocked with eight beluga whales and eight dolphins for breeding, performing and for the big money maker, the one-on-one with a whale. For a couple of hundred bucks, you too can spend 15 minutes patting the head of a captive whale who scientists believe is more intelligent than any human, solely for that perfect Facebook photo op.

          The good news is that there's still time to stop the Vancouver Aquarium's plans. We checked with the Vancity Theatre and were told that tickets to "Blackfish" are selling like hotcakes. So be sure to get your movie tickets quick and join the growing movement to end dolphin captivity and empty the whale and dolphin tanks in Stanley Park.

          The orca whales are long gone from the Vancouver Aquarium; the beluga whales and the dolphins are next!

          And that's why I just can't stop smiling.

          +
          + + + + + +
          +
          +
          + + + +
          +
          + +
          +
          +
          +
          + + + + + + + + + +
          +
          +
          + +
          + +
          + +
          +
          + + + + + + + + +
          +
          +
          +
          + + + + +
          + + + + +
          + + + +
          + +
          +
          + + + + +
          +
          + + +
          +
          + + + + + + + + + + + + +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/testdata/dfdd0eef0531a736d132b2c1aed316ec40043924.json b/test/testdata/dfdd0eef0531a736d132b2c1aed316ec40043924.json new file mode 100644 index 00000000..6e2b8295 --- /dev/null +++ b/test/testdata/dfdd0eef0531a736d132b2c1aed316ec40043924.json @@ -0,0 +1,19 @@ +{ + "encoding": "utf-8", + "headers": { + "Cache-Control": "max-age=300", + "Content-Encoding": "gzip", + "Content-Length": "39632", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:53:22 GMT", + "Expires": "Tue, 23 May 2017 18:03:22 GMT", + "P3P": "CP='NO P3P'", + "Server": "Apache", + "Vary": "Accept-Encoding", + "X-EC-Lua": "19365-geo", + "X-Mobile-URL": "http://m.huffpost.com/ca/entry/3686306", + "X-UA-Compatible": "IE=edge,chrome=1" + }, + "status_code": 200, + "url": "http://www.huffingtonpost.ca/annelise-sorg/blackfish-killer-whale-seaworld_b_3686306.html" +} \ No newline at end of file diff --git a/test/testdata/e13972a8805813de5259c4d477238c3eadbfd18e.html b/test/testdata/e13972a8805813de5259c4d477238c3eadbfd18e.html new file mode 100644 index 00000000..a8facf19 --- /dev/null +++ b/test/testdata/e13972a8805813de5259c4d477238c3eadbfd18e.html @@ -0,0 +1,502 @@ + + + + + + + + + + + + +March of the Migration + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          +
          + + + + + + +
          +
          +
          +
          +
          + + +
          +
          +
          +Correction to This Article
          +A Sept. 4 Travel article on animal migrations incorrectly identified the director of bird conservation at the National Audubon Society. He is Greg Butcher, not Greg Butler. +
          +

          March of the Migration

          + +
          +
          +Jacques Perrin's documentary, +
          +Jacques Perrin's documentary, "Winged Migration," has helped the animal-watching industry enjoy an uptick in visitors. + +(Mathieu Simonet) + +
          +
          +
          +
          + +
          + +
          + +
          + +
          + +
          By Andrea Sachs
          +Washington Post Staff Writer +
          +Sunday, September 4, 2005 +

          +

          +
          +

          +Along the New Jersey coast in Cape May this summer, the whales are in hiding. One marine tour company has not spotted a humpback in weeks. They are now redirecting passengers' attention to the dolphins. +

          +
          +

          +Each winter, though, whale watchers on the Pacific Coast have the opposite problem: They can't avoid getting jostled by the mammoth creatures, which outnumber the boats in Mexico's San Ignacio Lagoon during the Pacific gray whale's annual migration. And forget about staying drymdasblowhole spray drenches passengers like a garden hose. +

          +

          +"There were whales all over the place," says Warren Stortroen, 73, of St. Paul, Minn., who communed with the Pacific grays on an Earthwatch trip in January. "They would bump into our boats and come up alongside our boat, and we could pet them." +

          +

          +The difference in experiences, and numbers, boils down to the animal world's seasonal pilgrimage called migration. Many travel organizations -- including Earthwatch, which has volunteer vacations centered on migrating Pacific grays -- and wildlife organizations such as the National Audubon Society are bringing together vacationers and congregations of critters. And interest in migrations likely has been fueled by the surprise hit "March of the Penguins," a documentary that tracks the annual travails of Antarctica's emperor penguins. +

          +

          +Spontaneous wildlife viewing, in which you go to a park or the ocean at any time of the year and scour the land, sea or sky for full-time residents, can have iffy results. The creatures can be elusive or spread out. Bird-watchers can spot various species, but in single digits, while whale watchers joke that the mammals time their breaches around restroom breaks. +

          +

          +Yet during a species' migration, the wildlife viewing is fairly certain. The animals are on a tight schedule -- they cannot betray Mother Nature -- as they move in giant numbers to specific locations around the world: the Pacific grays in British Columbia and Baja California; the Jackson Hole elk in Wyoming's National Elk Refuge; the sandhill cranes at Rowe Sanctuary in Gibbon, Neb.; the greater snow geese in Virginia's Chincoteague National Wildlife Refuge. +

          +

          +"There are circumstances that Mother Nature has created to give travelers the most optimal wildlife viewing," says John Gibbons, spokesman for the National Zoo. "Whether we are talking about birds, animals, reptiles, insects or fish, one of the main motivations of migration is food." In other words: Follow that meal, even if it's thousands of miles from home. +

          +

          +Travel based on nature is an ever-widening niche, as more people try to connect with the outdoors and its wild inhabitants. According to the International Ecotourism Society ( http://www.ecotourism.org/ ), in 2002 about 13 percent of 18.6 million American travelers were ecotourists. +

          +

          +"If you want a spectacle, migration is the way to do it," says Greg Butler, director of bird conservation at the Audubon Society. "But it is weather-dependent." +

          +

          +To be sure, the Earth's creatures are fickle. Some shy from bad weather, others from low insect counts. They might linger a little longer at a particular pit stop or get detoured (or destroyed or devoured) by man-made obstacles or predators. But they need nourishment, and they know that when the seasons change and their food sources are about to become scarce, they must leave town. So they move north or south, east or west, to higher or lower elevations or, in the case of wildebeest, around in dizzying circles. +

          +

          +"You get to see such a large number of animals traveling and coexisting together," says Tony Rango, national outings director of the Sierra Club. "This is not a petting zoo. You are in their environment and interacting with them and seeing nature in its splendor." +

          +

          +The largest mammal migration is that of the Serengeti wildebeest, whose 1.5 million-strong herd tirelessly chases the African rainy season; the farthest trek is that of the Arctic tern, which flit from pole to pole, clocking 10,000 miles each way. But the sandhill cranes of the central flyway, with their Hepburnesque necks and statuesque frames, are equally majestic as they blanket Nebraskan fields and Platte River banks en route from Mexico, Texas and New Mexico and then on to Canada, Alaska and Siberia. To the west, more than 30,000 Pacific grays swim from Alaska to Mexico, hewing fairly close to the California shoreline. Monarch butterflies also follow the up-and-down route, though they are much more dispersed, alighting in Texas, California and Mexico. +

          +

          +Because of the sheer number of migratory animals and the involuntary pull of their inner clocks, travelers can easily walk right into a migratory pack. Kevin Smith of Crooked River Ranch, Ore., for example, only had to peer out his car window near Kearney, Neb., to see countless sandhill cranes during their spring migration. He observed thousands more from blinds set up at Rowe Sanctuary. "I watched the birds for hours until it was too dark to see," says Smith, 63, a retired building inspector who has been birding since age 12. "There was no ground not covered by cranes. My wife and I saw more cranes than a person has a right to see in a lifetime." +

          +

          +The sandhill cranes typically arrive in droves, their trumpeting calls filling the air like a high school marching band. They spend their days feeding in corn fields and strutting around like Don Juan, hoping to make a love match. Come nightfall, they flock to the river banks and settle into a comfortable sleeping stance atop skinny legs. For two weeks, this is their life in Nebraska: eat, sleep, flirt. Then, they're off. +

          +

          +To increase your odds of witnessing migrating wildlife, many nature organizations, environmental groups and hobbyists track the route and the migrants' progress. The University of Kansas's Monarch Watch, for one, has an online forum ( http://www.monarchwatch.org/ ) in which viewers post their sightings nationwide. (Things are hatching in the butterfly world: An observer from Baltimore spotted a caterpillar in his milkweed in July.) Also check the weather, since many animals and birds prefer to travel when the sun is out and the skies are blue. Birds, for example, wait for cold fronts to pass and are partial to tail winds. A hurricane can divert or delay a flight plan, and sometimes, if the elements are just right, they might skip over their usual landing zone and keep on flying. +

          +

          +Yet just like the rise and fall of the sun, migrating animals have no choice but to heed nature's call. So if you miss them on the way down, you can always see them on the way back. Or maybe catch them on both ends. +

          +

          +"I would do it again," says Stortroen of the Pacific gray whale migration, "but I want to see it from the other side, from British Columbia." +

          +

          +Even with a restroom break, he won't miss a thing. +

          +

          +For information on bird migrations, contact the National Audubon Society (212-979-3197,http://www.audubon.org) or the National Zoo's Migratory Bird Center (202-633-4800,http://www.nationalzoo.si.edu). For fish, birds and animals: U.S. Fish and Wildlife Service (800-344-WILD,http://www.fws.gov), National Parks Conversation Association (800-628-7275,http://www.npca.org) and World Wildlife Fund (202-293-4800,http://www.worldwildlife.org). For whales: Whale and Dolphin Conservation Society,http://www.wdcs.org. For African migrations: African Wildlife Foundation, 202-939-3333,http://www.awf.org. +

          +
          +
          +
          +
          +
          + +
          + +
          + +
          © 2005 The Washington Post Company
          +
          +
          +
          + + + + + + + + + +
          + + + + + + + + +
          + +
          +
          +
          + + + + +
          +
          + + + + + + + + + + + + diff --git a/test/testdata/e13972a8805813de5259c4d477238c3eadbfd18e.json b/test/testdata/e13972a8805813de5259c4d477238c3eadbfd18e.json new file mode 100644 index 00000000..6580fd9e --- /dev/null +++ b/test/testdata/e13972a8805813de5259c4d477238c3eadbfd18e.json @@ -0,0 +1,22 @@ +{ + "encoding": "ISO-8859-1", + "headers": { + "Age": "0", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Type": "text/html", + "Date": "Tue, 23 May 2017 17:55:18 GMT", + "Last-Modified": "Tue, 02 Aug 2016 19:25:06 GMT", + "Server": "AmazonS3", + "Set-Cookie": "de=;Expires=Thursday, 23-May-2019 17:55:17 GMT; path=/; domain=.washingtonpost.com, client_region=0;Expires=Tuesday, 23-May-2017 18:05:17 GMT; path=/; domain=.washingtonpost.com, X-WP-Split=X;Expires=Thursday, 01-January-1970 00:00:00 GMT; path=/; domain=.washingtonpost.com, devicetype=0;Expires=Friday, 23-June-2017 04:24:17 GMT; path=/; domain=.washingtonpost.com, osfam=0;Expires=Friday, 23-June-2017 04:24:17 GMT; path=/; domain=.washingtonpost.com, rpld1=0:wmflabs.org|20:usa|21:ca|22:san francisco|23:37.785591|24:-122.435661|;Expires=Tuesday, 23-May-2017 18:55:17 GMT; path=/; domain=.washingtonpost.com", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding", + "Via": "1.1 5b7194cd796490b3bb20e0ed10b59026.cloudfront.net (CloudFront)", + "X-Amz-Cf-Id": "Q5uiU0TXEIXMtJXlpbdG_SJELtpnkwnmueBjoFfHULrxJv4Arc_1Zw==", + "X-Cache": "Miss from cloudfront", + "X-Instart-Debug-Header": "auth_status:200, origin:origin-web.washingtonpost.com, cache key modifier:0, num_auth_cookies:6", + "X-Instart-Request-ID": "14652614164744398230:VNQ01-NPPRY09:1495562117:165" + }, + "status_code": 200, + "url": "http://www.washingtonpost.com/wp-dyn/content/article/2005/09/02/AR2005090200822.html" +} \ No newline at end of file diff --git a/test/testdata/e43a8017de92a45dfa218499e7abd25f70236216.html b/test/testdata/e43a8017de92a45dfa218499e7abd25f70236216.html new file mode 100644 index 00000000..0366893d --- /dev/null +++ b/test/testdata/e43a8017de92a45dfa218499e7abd25f70236216.html @@ -0,0 +1,16839 @@ + + +Home | Daily Mail Online + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + +
           
          + + + + + + +
          + +   + +

          Home

          + Updated: 13:43 EDT +
          + + +
          + + + +
          +
          + + +
          +
          +
          + +

          + +

          +
          + + Manchester suicide bomber is 'British-Libyan' Salman Abedi + + + +
          +

          + + Witnesses told of nuts and bolts tearing into young music fans when the blast was detonated in the foyer area of the Manchester Arena moments after a concert by US popstar Ariana Grande ended. The bomb went off in a packed foyer area at a time when mothers and fathers were leaving the venue with their children and other parents were arriving to pick up groups of youngsters. Police this morning confirmed that the suicide bomber, who was known to police, died inside the arena. US security sources said the bomber had travelled to the venue on public transport. A 23-year-old man was arrested by anti-terror officers in the south of the city as police and security services attempt to work out if the suicide bomber was part of a cell. +

          +
          +
          +
          + +
          + +
          + +
          + + +
          +
          +
          + +

          + +

          +
          + + ISIS claim responsibility for Manchester bombing + + + +
          +

          + + ISIS have claimed responsibility after a suicide bomber - named today as Salman Abedi - set off the ball bearing bomb that killed 22 and injured 119 in Manchester. The terror group were quick to call Abedi one of their own as they gloated about the bombing 'in the midst of a gathering of the Crusaders' at an Ariana Grande concert. Eight-year-old Saffie Roussos, 18-year-old Georgina Callander and 26-year-old John Atkinson were among those killed. Of the injured, at least 12 were children were among 59 taken to hospital, with 60 others treated at the scene. +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Georgina Callander, 18, died when a lone attacker detonated an explosive device as thousands of youngsters were leaving Manchester Arena last night. She is pictured with Ariana Grande in 2015. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Nick Haywood, 46, was waiting to collect his daughter Caitlin, 16, pictured together, from the Ariana Grande concert at Manchester Arena when the suicide bomber struck. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + There are unconfirmed reports that the terrorist who slaughtered 22 at Manchester Arena last night was British and that he may have been known to police before the massacre. +

          +
          +
          + +
          + +
          + +
          + +

          + +

          +
          + + + + + +

          + + I feel angry. Every sinew of my body is pulsating with blind rage. I simply cannot comprehend the warped, depraved mentality of anyone who deliberately thousands of children. +

          +
          +
          + +
          + +
          + +
          + +

          + +

          +
          + + + + + +

          + + Her manager Scooter Braun said, 'Our hearts are broken' as they mourned 'the lives of children and loved ones taken by this cowardly act.' +

          +
          +
          + +
          + + +
          + +
          +
          +
          +
          + + +
          + +
          + + +
          +
          +
          +
          +
          + +

          + +

          + + 'So amazing': Trump writes tribute at Holocaust memorial + + + +
          +

          + + Trump left a written tribute at Yad Vashem, Israel's memorial and museum to the Holocaust, which called it 'so amazing'. He laid a wreath and used a speech to pay tribute to the victims of Nazi murder and promise 'never again'. He was accompanied (left) by his wife Melania, and daughter Ivanka and son-in-law Jared Kushner, both of whom are Jewish. Kushner grandparents were Holocaust survivors and many of his relatives were murdered by the Nazis. Trump's written tribute was in contrast to a speech in which he said: 'It was the most savage crime against God and his children, and it is our solemn duty to remember, to mourn, to grieve and to honor every single life that was so cruelly and viciously taken.' +

          +
          + +
          + +
          +
          + +
          +
          +
          + + +
          + +
          + US Daily Mail News Tips + +Daily Mail News Tips + +
          + +
          + +

          + +

          +
          + + + + + +

          + + Trump asked National Security Agency chief and Dan Coats, Director of National Intelligence, to help reject claims his campaign colluded with Vladimir Putin's Kremlin during the election. +

          +
          +
          + +
          + +
          + +
          + +

          + +

          +
          + + + + + +

          + + Trump's budget guru argued Tuesday that the work requirements the administration wants are not meant to keep impoverished Americans from getting help - they 'are not the problem.' +

          +
          +
          + +
          + +
          + +
          +
          +
          +

          + So much for diplomacy! Smirking Israeli ambassador to the US puts his head in his hand after Trump implies Israel is not in the Middle East +

          + + + + +
          +
          +

          + + The Israeli ambassador was left clutching his head in what appeared to be both amusement and despair during Donald Trump's appearance in Israel on Monday. Trump was in a bilateral meeting with President Reuven Rivlin of Israel during his ongoing diplomacy tour when he implied that the country was not, in fact in the Middle East. That left Ambassador Ron Dermer, who was spectating from the sidelines, throwing his hand up to his face - and the whole thing was caught on film. +

          +
          +
          +
          + +
          + +
          + +
          + + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + NEW + + A Virginia woman exposed in a 'Fake Homeless' viral video panhandling was arrested by police on Monday after officers responded to a 'disorderly situation'. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Susan Zirinsky, senior executive producer of Princess Diana: Her Life, Her Death, The Truth, which airs on CBS tonight, said she was shocked by how little the pair knew each other. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Millions of new members have flooded to join Ashley Madison despite the 2015 data breach where hackers released the personal details of cheating spouses on the site. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + White supremacist materials, bomb-making chemicals and a photo of the Oklahoma City bomber were found in the Florida apartment where two neo-Nazis were killed by a Muslim convert. +

          +
          +
          + +
          + +
          + +
          +
          +
          + + +
          + +
          + +

          + +

          +
          + + Woman arrested for outdoor threesome weeks after wedding + + + +
          +

          + + A 19-year-old woman was arrested in Mississippi just over two weeks after she married when she was spotted having a public threesome with co-workers on the deck of a family bar, police said. Amy Hammers (pictured left, and inset with new husband John Wilson), 19, of Pearl River, Louisiana, allegedly engaged in the alfresco group sex with Brandon C Mabery (top-right), 30, of Kennedale, Texas, and Tiffany Thibodeaux (bottom-right), 26, of Biloxi, Louisiana. All worked in sales at alarm company ADT, police said. Sheriff Ricky Adam told the Star Telegram the owner of the bar saw the trio 'Right there. In the middle of the day. In broad daylight. In front of God and everybody.' +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + The second day of jury selection in the Bill Cosby sexual assault trial began Tuesday at the Allegheny County Courthouse in Pittsburgh, Pennsylvania. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Gerald Muskiewicz, 30, was found completely naked at Valley of the Eagles Golf Club in Elyria, Ohio, by firefighters at about 8.30am on Monday days after falling from a nearby cliff. +

          +
          +
          + +
          + +
          + +
          +
          +
          + + +
          + +
          + +

          + +

          +
          + + Michelle and Barack Obama relax at a luxury Italian villa + + + +
          +

          + + The former First Lady modeled a colorful caftan that fell off her shoulder and flat sandals as she and her husband had drinks and a fancy lunch on Monday at Borgo Finocchieto, the stunning Tuscan villa where they are staying during their trip to Siena. While Michelle showed of her chic vacation style, Barack looked comfortable in a gray polo shirt, leaving the collar unbuttoned. The former president wore his sunglasses on top of his head during the meal, and at one point he leaned back and placed his hands behind his head to stretch. Barack and Michelle were dining with other companions, and at the end of the meal, he graciously picked up the tab. +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Nina Robinson left a cellphone recording on a patient's dresser on May 5, when she began to suspect that the 95-year-old was being abused by a distant relative at her home in Los Angeles. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + This image provided by Google shows a demonstration of the company's new product called "Jamboard." Google has designed the giant touch-screen canvas for companies trying to make it easier for their employees to brainstorm as they work on team projects and other assignments. Google is releasing the device to a small group of companies Tuesday, Oct. 25, 2016, before making it widely available in early 2017. (Google via AP) + + + +

          + + The 'Jamboard,' will replace whiteboard. It boasts a 55-inch, ultrahigh-definition screen capable of recognizing the difference between when someone is writing on it with a stylus or finger. +

          +
          +
          + +
          + +
          + +
          +
          +
          + + +
          + +
          + +

          + +

          +
          + + Body-shaming Playboy model Dani Mathes ready to face jail + + + +
          +

          + + Dani Mathers's childhood friend, Melissa Yanez (at right with Mathers) told DailyMail.com the former Playboy Playmate is ready to face the consequences for Snapchatting a naked senior changing at an Los Angeles gym. If found guilty of violating privacy laws, she could spend up to six months in jail. Yanez has known Mathers since they were toddlers and said the backlash from the Snapchat scandal has caused her to suffer depression. +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Taylor Brooks Boyles, a newly divorced math and social studies teacher from Alabama, has been charged with felony engaging in a sex act with an 18-year-old high school senior. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + A two-person panel will determine OJ Simpson's fate in the days after the July hearing, which could very likely lead to his release from the facility on October 1, nine years into his sentence. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Kim, who is half Austrian and half German, has decided to sell her body through the agency Cinderella Escorts to fund her studies, a flat and a car. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Nancy Ann Martin, of Richmond, Virginia, fell over the edge of the East Fork Overlook at Milepost 418 near the Pisgah National Forest in Haywood County, North Carolina, on Friday. +

          +
          +
          + +
          + +
          + +
          +
          +
          + + +
          + +
          + +

          + +

          +
          + + Melania Trump does an outfit change for Italy arrival + + + +
          +

          + NEW + + The 47-year-old First Lady started the day in a $2,730 white dress by Roksanda, but she changed into a black coat featuring gold embroidery around its collar and cuffs on Air Force One after the aircraft left Israel on Tuesday afternoon. Ivanka (inset), 35, opted for a $5,690 Oscar de la Renta frock featuring long sleeves to show respect for her Jewish faith and its dress codes, but she did not change clothes during the flight. +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + President Trump arrived in Rome, Italy Tuesday evening, marking the third country he will visit on his first trip abroad as president. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + President Trump's budget for next year slashes $200 million for development of a new Air Force One, in the latest installment in his effort to trim costs from the program. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + The conflict 'cannot continue forever,' Trump said in Jerusalem. 'The only question is when nations will decide that they have had enough. Enough bloodshed, enough killing.' +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + At a joint appearance with Donald Trump, Netanyahu clobbered Abbas for the payments that are at odds with the principles he outlined earlier in the day at his meeting with the U.S. president. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + Melania and Ivanka Trump wear white at Holocaust memorial + + + +
          +

          + + Melania, 47, modeled a sleeveless, A-line dress that skimmed her calves, while Ivanka, 35, opted for a modest long sleeve frock to show respect for her Jewish faith and its strict dress codes during their visit to Israel's Holocaust memorial Yad Vashem on Tuesday with the president. Melania and Ivanka donned nude shoes with their stark white ensembles, and they both opted to wear their hair loose around their shoulders. +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + American officials have started 'extreme vetting' interviews at Australia's offshore detention centers as Washington makes good on a refugee swap, even though Trump called it a 'dumb deal'. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Trump made the short trip from Jerusalem to Bethlehem to meet Mahmud Abbas, who hopes to convince the him to remain committed to an independent Palestinian state. +

          +
          +
          + +
          + +
          + +
          +
          +
          + + +
          + +
          + +

          + +

          +
          + + Heiress and actress Dina Merrill passes away in NY aged 93 + + + +
          +

          + + Dina Merrill (left in 1954) passed away at the age of 93 on Monday at her estate in East Hampton, New York. One of the most famous American socialites of the 20th century (right in 2007), Nedenia Marjorie Hutton shocked her parents (family winter home Mar-a-Lago inset) and the Park Avenue set with her decision to study acting and later head to Hollywood, where she changed her name at the start of her career. Once there, she landed roles in films including 'Butterfield 8' opposite Elizabeth Taylor, 'Desk Set' with Katherine Hepburn and Spencer Tracy and in her later years, the comedy sequel 'Caddyshack II.' Merill's cause of death was lewy body dementia according to family members. +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + A 20-year-old man who drugged and raped his 16-year-old sister was sentenced to less than a year in prison by a California judge on Wednesday. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Alyssia Sosa, 15, strangled herself inside her family's home in Richmond, Texas, on May 19, two days after the alleged groping incident, and now her family are demanding answers. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Corey Walgren, 16, committed suicide in January in Naperville, Illinois soon after he was questioned about claims he possessed and potentially shared 'child pornography'. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + CEOs at the biggest US companies received an 8.5% raise last year, raking in $11.5M in salary, stock and other compensation. Thomas Rutledge (file) CEO Charter Communications Inc. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Osama bin Laden's fourth and youngest wife, Amal, described how she and her children and other relatives huddled in terror as American SEALs blasted down the walls of the compound in Pakistan. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + A flight to Bristol made an emergency landing at Orlando Sanford International Airport in Florida on Sunday after crew and passengers reported feeling sick. +

          +
          +
          + +
          + +
          + +
          +
          +
          + + +
          + +
          +

          + Sickening moment evil bully, 29, imitates walk of cerebral palsy sufferer then SUCKER PUNCHES him outside Pennsylvania 7-Eleven +

          + + + + +
          +
          +

          + + Barry Baker was filmed mocking the 22-year-old man as he made his way back to his car outside the store in West Chester, Pennsylvania, on May 10. The 29-year-old then punched the man as he stood next to the vehicle before calmly walking away. Surveillance footage captured the attack and Baker was charged with assault. +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + New Jersey Governor Chris Christie told reporters on Monday that President Donald Trump ignored his advice not to hire retired Lt. Gen. Mike Flynn. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + South Korean soldiers fired around 90 machine gun rounds into the air and towards the North after the projectile flew over the border (North Korean dictator Kim Jong-un pictured). +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Former CIA Director John Brennan testified before a House intelligence panel that he called the head of Russia's federal intelligence service to warn about Russian interference in the presidential election. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + John Podesta whacked former House Speaker Newt Gingrich for claiming murdered Democratic staffer Seth Rich was 'assassinated' after giving emails to Wikileaks. +

          +
          +
          + +
          + +
          + +
          +
          +
          + + +
          + +
          + +

          + +

          +
          + + Manchester Arena attack: Trump calls bomber an evil loser + + + +
          +

          + + President Donald Trump said from the historic West Bank town of Bethlehem that he stands in 'absolute solidarity with the people of the United Kingdom' after a suicide bomber killed 22 in Manchester. The bomb hit an Ariana Grande concert as it ended on Monday night, killing 22 people among a panicked crowd of young concertgoers. The attack sparked a nightlong search for loved ones - parents for the children they had accompanied or agreed to pick up, and friends for each other after groups were scattered by the blast. Trump said that he has 'no tolerance' for such attacks, calling the bomber an 'evil loser', whose ideology needs to be 'completely obliterated'. +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + 'This country is calling out for a doctor. We need to know what will cure us. What action do we take? How can we stop the hurt? I wonder if we are too sick to be saved,' says KATIE HOPKINS. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Maarten Smit, 49, was with his daughter Jasmin and took a photograph of the pair with huge grins on their faces as they enjoyed the Ariana Grande concert at the Manchester Arena. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Senior senator Viktor Ozerov, an aide to Russian President Vladimir Putin (pictured) appeared to blame the British government for making the UK vulnerable to terrorist outrages. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + The site reports that Joan had still been sitting in her seat on the front row when the bomb detonated in the venue's lobby after her daughter's performance - and was quick to come to the aid of nearby fans. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Aaron Spears detailed the first moments of the tragic incident to his local Fox News station in Washington - which saw him discuss the 'heartbreaking' realisation it had been a bomb. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + A mother who was waiting to pick up her two children from Manchester Arena last night believes she saw the killer and was standing just 15ft from him before he slaughtered 22 people. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Presenter Susanna Reid was reduced to tears as she discussed the heartbreaking news with co-host Piers Morgan on Tuesday's installment of Good Morning Britain. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Dramatic footage shows officers arresting a 23-year-old man outside a Morrisons supermarket in Chorlton-Cum-Hardy, south Manchester this morning. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Georgina Callander, 18, died when a lone attacker detonated an explosive device as thousands of youngsters were leaving Manchester Arena last night. She is pictured with Ariana Grande in 2015. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + EXCLUSIVE: Nine-year-old Nevie said: 'I was really scared and I was just shaking the whole way back. It was my second concert.' She is the youngest survivor to give her account. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Police in New York City and Boston have been put on alert after the bombing at an Ariana Grande concert in Manchester, UK Monday night. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + A male suicide bomber detonated a homemade bomb inside the foyer of the arena following an Ariana Grande concert last night, at the point where gig-goers are most vulnerable. +

          +
          +
          + +
          + +
          + +
          +
          +
          + + +
          + +
          +

          + 'You almost killed me, dude!' Terrifying moment Harley Davidson biker is dragged along by car after crash as driver refuses to stop  +

          + + + + +
          +
          +

          + + This is the terrifying moment a Harley Davidson biker dodges death when he is scooped on to the back of a speeding car on an LA freeway in the United States. In the video, the biker desperately screams 'stop, stop, stop' as he bangs on the back window of the motor begging the motorist to slow down. +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Anthony Guadiana, 70, died in a car crash in Scituate, Rhode Island, three days before his graduation from the University of Rhode Island Sunday. His daughter walked in his place. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Theresia Brandl donned a cap and gown on Wednesday at her Oakdale, Pennsylvania, nursing home to celebrate her honorary degree from Sto-Rox High School in nearby McKees Rocks. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Judy O'Connor was her 29-year-old son Marty O'Connor's note-taker as he worked towards his master's of business administration degree at Chapman University in California. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Mathew Cherackal, 35, of Portland, yelled that he loved 'Yale about as much as anyone who hasn't actually gone here possibly could,' mid-ceremony on Monday. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Facebook is flooded with 54,000 cases of revenge porn and sextortion every month, documents leaked by staff reveal. In January there were 51,300 revenge porn complaints. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Researchers from Duke University, North Carolina, found that reducing calorie intake by a quarter slows inner ageing by 0.6 years annually versus having a normal diet. +

          +
          +
          + +
          + +
          + +
          +
          +
          + + +
          + +
          + +

          + +

          +
          + + Kate Middleton praised for Pippa's wedding church sketch + + + +
          +

          + + The Duchess of Cambridge's skill as an artist was revealed after her new brother-in-law James Matthews thanked her for a sketch of St Mark's Church in Englefield which featured in the order of service. The drawing by Kate, 35, (right) was spotted on the booklet (inset) carried by James' mother Jane (left) as she left the church on Saturday. +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + The calf's 16-year-old mother, named Ringer, became pregnant while on birth control. Her baby died on Saturday night just after 9pm at SeaWorld's Orlando Aquatica water park. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Freddie Gray, 25, died in custody after his neck was broken while he was handcuffed and shackled but left unrestrained in the back of a police van in April 2015. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + The long-awaited Hydrus rollercoaster has opened on New Jersey's Seaside Heights pier. Hydrus replaces the Jet Star rollercoaster, which was dumped into the ocean by superstorm Sandy in 2012 +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Sacked waitress Viviana Ross was all smiles as she stepped out in London, after details of her night of 'incredible sex' with Orlando Bloom were revealed. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + By carefully matching your skincare and diet to your changing hormones, it's possible to make common skin problems like dry patches, greasiness and breakouts a thing of the past. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Researchers from Harvard Medical School found repeating the same seemingly trivial questions at erratic intervals in the style of the TV sleuth Columbo (pictured) made people away vital details. +

          +
          +
          + +
          + +
          + +
          +
          +
          + + +
          + +
          + +

          + +

          +
          + + Ex-wife of Russian oligarch demands $15bn in divorce + + + +
          +

          + + Natalia Potanina, 55, has filed a lawsuit at a Moscow court claiming $15billion from her ex Vladimir Potanin. The pair split in 2014 but are still working out a settlement. Ms Potanina has seen two previous attempts to get her hands on shares in mining giant Norilsk Nickel and investment firm Interros International turned down, as judges ruled Mr Potanin does not own them outright. But she has filed a fresh appeal, claiming she should be entitled to half the profits, even if she cannot have the shares. +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Peter Boghossian and James Liddle from Portland State University admitted a paper they wrote which received praise for its criticisms of hypermasculinity was actually a hoax. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Yesterday, scientists from Amsterdam announced the discovery of 40 new genes linked to human intelligence, and found that many people with the genes were also on the autistic spectrum (stock). +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + A New Jersey restaurant worker has been fired after they wrote an offensive note on a receipt for a police officer, calling the cop a 'pig' - in both English and Spanish. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + A Harvard study claims teaching hospitals are better with patients than other hospitals, making a 1.5 percent difference in mortality rates. Experts say this translates to 58,000 lives a year. +

          +
          +
          + +
          + +
          + +
          +
          +
          + + +
          + +
          +
          +
          +
          + +
          + +
          +
          + +
          + +
          + +
          +
          +
          +
          + +
          +
          +
          + Bing +
          + + + + + + + + +
          + +
          + + + + +
          +
          + +
          + +
          + + +
          + + + +
          + + +
          + +
          +
          + +   +   +

          Femail Today

          + + +
          + +
          + + +
          +
          + + +
          +
          + + +
          + +
          + + +
          +
          +
          +

          + Touching moment toddler who was born deaf hears clearly for the first time in her life thanks to a cochlear implant +

          + + + + +
          +
          +

          + + Annabelle Lawless (left and right), one, had her cochlear implant activated and was able to hear clearly. The little girl from Boise, Idaho, made news when she first heard her mother Sarah Jo's (left) voice with hearing aids at three months old in July. When the implant was turned on, Annabelle's eyes go wide behind her baby blue glasses. She quickly looks around before she seems overwhelmed with the new sensations. +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Parichat 'Pang' Chatsri, 26, was made to quit after a photo went viral of her in a tight-fitting lilac nurse's uniform at a private hospital in Isan, Thailand. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + María Lorena Ramírez defeated 500 runners from 12 countries in the female category of the Ultra Trail Cerro Rojo in Puebla, in central Mexico, finishing the race in seven hours and three minutes. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Newly released dashcam footage has captured the moment police arrived at an Ohio nursing home to find their local chief and two nurses dead after an ex-lover went on a deadly rampage. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + A police officer ended up pulling a Taser on a man who was just chatting to a friend who had pulled up alongside his house in her car. The incident happened in North Enid, Oklahoma. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Federal Consumer Product Safety Commission recalled 6,300 electric stoves in connection with the death of plumber, David Dufresne Jr (pictured), 52, who was electrocuted in 2016. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + According to data from the New York City-based Bank of America Merrill Lynch Apple is $803 billion (£618 billion) - which tops the GDP of all US cities except for Los Angeles and New York (stock image). +

          +
          +
          + +
          + +
          + +
          +
          +
          + + +
          + +
          + +

          + +

          +
          + + New Jersey woman finds note from HH Holmes in her Bible + + + +
          +

          + + A New Jersey family say they opened the Good Book to find a letter from a very bad man: America's first serial killer. Claire Fanelle (pictured inset) found the Bible, which was stuffed with newspaper clippings and had been owned by her mother, while cleaning, NBC 10 reported on Monday. Along with the clippings was a yellowed old letter with a sinister name (top-right) at the bottom: that of HH Holmes (pictured left), who claimed to have killed as many as 27 people before he was arrested, tried and executed. Holmes was infamous for constructing his 'Murder Castle' - a hotel (bottom-right) full of soundproofed rooms, trapdoors and gas lines in which he would asphyxiate guests, then dissect and sell their bodies. +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Researchers from Tufts University in Medford, Massachusetts, found that worm self-healing is encoded in the electrical activity of cells, taking scientists a step closer to human regeneration. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + NEW + + A video has emerged of professional golfer Amanda Blumenherst's two-year-old son, Will, failing miserably to putt a ball despite being centimeters from the target at an Arizona course. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Robert Kennedy III is dating Montana Coady, the former wife of Star mogul Chris Albrecht. They confirmed their relationship at a friend's wedding in Cabo San Lucas this weekend. +

          +
          +
          + +
          + +
          + +
          + +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + For example, dairy products can have a lesser effect on cholesterol than would be predicted on the basis of their saturated fat content if eaten with other foods, Danish scientists found. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Lillian Rimmel went into an Orlando CVS store ten minutes before 10pm Friday night and was stunned when she realized she was alone in the store, and locked inside. +

          +
          +
          + +
          + +
          + +
          +
          +
          + + +
          + +
          + +

          + +

          +
          + + German flight attendant shows off her yoga skills + + + +
          +

          + + Glamorous German flight attendant Evi Schwarzfischer, from Regensburg shows off her impressive yoga moves at the beaches she visits after landing. The 28-year-old, who's also a qualified fitness instructor, unsurprisingly has a growing legion of fans on Instagram. +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + The owners of the Washington, DC- based bar, Diet Starts Monday, removed the item they dubbed the 'Pill Cosby', on Monday afternoon, hours after pictures of the cocktail circulated online. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Scientists from the Mushroom Technology Research Centre of La Rioja in Spain aimed to evaluate the influence of different cooking methods on the nutritional value of mushrooms. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Mark Mahoney, 59, and owner of the Shamrock Social Club in Hollywood has been inking the rich and famous for 40 years - and his fans include Lana Del Rey, Lady Gaga and David Beckham. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + A group of diabetics taking metformin had drastic changes in their gut flora, found Swedish and Spanish researchers.A probiotic diet may be beneficial for people with diabetes, say scientists. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + A University of Maryland student was forced to apologise to after a controversial speech sparked online outrage in her home country. She spoke about China in the speech on May 21. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Vanity Fair have released four Star Wars covers, celebrating 40 years of the hugely successful film franchise - one of which stars the late Carrie Fisher. +

          +
          +
          + +
          + +
          + +
          +
          +
          + + +
          + +
          + +

          + +

          +
          + + Pilot captures Boeing 747 cockpit in stunning photo series + + + +
          +

          + + JPC Van Heijst admits that he's often in a pretty good position to capture the earth from above - because he's a pilot with an air freight company. The Dutchman is a first officer with Cargolux and flies Boeing 747s around the world. He's also very handy with a camera and has captured hundreds of stunning pictures from the cockpit. +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Poignant pictures released by the Library of Congress show worshippers bowing their heads in prayer at churches and synagogues across Manhattan. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + A swath of the hillside gave way in an area called Mud Creek on Saturday night, covering about one-third of a mile of road and changing the Big Sur coastline immediately below. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + While they are bitter enemies today, America and Iran shared a strong alliance during the Second World War as the Allies used the country as a supply route to help Russia battle Hitler. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + The man, nicknamed 'Mikhail', was aged around 30 when he died between the seventh and ninth century AD. He was found buried at the Ust-Ivanovka burial site in the Primorsky region of Siberia. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Two gay men aged 20 and 23 have been caned 85 times each in Aceh, Indonesia, marking the first time the punishment has been used for homosexuality under strict sharia law in the province. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Joseph Liscinsky was 'startled' when he spotted the massive snake just a foot away from him at his home in Pembroke Pines, Florida, on Friday morning. +

          +
          +
          + +
          + +
          + +
          +
          +
          + + +
          + +
          + +

          + +

          +
          + + Joel Osteen under fire for hand signal at son's graduation + + + +
          +

          + + Joel Osteen and his wife Victoria documented their son Jonathan's graduation from the University of Texas on Saturday by taking photos of themselves throwing a 'Hook 'em Horns' hand signal. The spirited hand gesture has long been used at the university to represent its Texas Longhorn Bevo mascot. But it clearly didn't bode well with some of their millions of Twitter followers who complained the megachurch family was using the devil's sign. +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + In the inner regions of a spinning black hole, space and time are mixed so that travel back in time is tantalisingly close to possible, says a Durham University Professor of Physics. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + An underground black market has been uncovered in China with dealers smuggling placentas out of hospitals. The Beijing News revealed the report on May 14. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Spanish mountaineer Kilian Jornet has set the record for the fastest ascent of Mount Everest without fixed ropes or extra oxygen. His team announced his success earlier today. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Scientists from Tampere University of Technology, in Finland, have developed a soft, gripping device (pictured) that can sense and pick up objects, mimicking the ferocious Venus flytrap plant. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + But a breakthrough study from a team of scientists from Iowa State University has found that one gene had a 'dramatic difference' in its impact on memory and cognitive function. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Dexter the giant rabbit - the 'little' brother of Simon the enormous rabbit that froze to death on a flight last month - is expected to become the largest in the world - already weighing more than three babies. +

          +
          +
          + +
          + +
          + +
          +
          +
          + + +
          + +
          +

          + Shocking moment a demolition crew accidentally knocks down the WRONG home in a Baltimore neighborhood +

          + + + + +
          +
          +

          + + A demolition crew accidentally knocked down a property that had been bought out for redevelopment. Officials in Baltimore said the city received several complaints about a bowing exterior wall and a crack between two rowhouses at 212 E. Fort Avenue (left side of building). After city inspectors deemed the property unstable, the Baltimore City Department of Housing said crews were sent to carryout an emergency demolition. But while knocking down the three-floor rowhouse at 212 E. Fort Avenue, the neighboring property at 214 E. Fort Avenue (right side of building) was partially collapsed. A video of the incident shows part of a brick wall on the third floor toppling onto the roof next door. The 214 E. Fort Avenue property was a former pet shop called the Laundry Mutt that had been bought out and set to be redeveloped by another owner. Authorities said no one was injured in the mix-up. +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + The new device, which can also be used as a touchscreen tablet, will cost £799 ($799) and go on sale on 15 June, the technology giant confirmed at an event in Shanghai. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Federal authorities say a California man who worked as an engineer for a defense contractor has pleaded guilty to attempting to sell sensitive information used in satellites to Russia. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Females are more likely to suffer sleep deprivation symptoms such as depression, said Queensland researchers. These include depression, memory problems and difficulty concentrating. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Researchers from Radboud University in Nijmegen, the Netherlands, say the skill allows us to judge trajectories and react to them at lightning speeds. +

          +
          +
          + +
          + +
          + +
          +
          +
          + + +
          + +
          + +

          + +

          +
          + + Rebel Wilson calls herself 'cashed-up bogan' in court + + + +
          +

          + + Rebel Wilson (left) described herself as a 'cashed-up bogan' as she took the stand to give evidence in one of the most bizarre celebrity trials in recent memory. Wilson (pictured right as a child) is suing the publishers of Woman's Day for a series of articles she says suggested she lied about her working class upbringing in Sydney's north-west. When Wilson, 37, made a brief appearance on the witness stand on Tuesday, she described herself as a proud bogan. 'I definitely consider myself a bogan, although now I'd probably be a cashed-up bogan,' she said. 'I use it very endearingly.' +

          +
          +
          +
          + +
          + +
          + +
          + +
          +
          + +

          + +

          +
          + + + + + +

          + + The hottest day ever recorded was June 10, 1913, in California's Death Valley, where the mercury rose to 56.7C. England's current record is 38.5C. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + A tourist is suing a Las Vegas Strip hotel, claiming a life-sized mannequin in his darkened room caused him to flee and suffer injuries. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + This Wednesday, April 26, 2017, photo shows Google's web address, in Philadelphia. Google is keeping an eye on what you're buying offline in addition to monitoring your online shopping in its latest attempt to sell more digital advertising. The offline tracking of most credit and debit card transactions will help Google to automatically inform merchants when digital ads appearing on its vast marketing network translate into sales at a brick-and-mortar store. Google plans to unveil the store-sales measurement tool Tuesday, May 23, 2017, in San Francisco at an annual conference it hosts for its advertisers. (AP Photo/Matt Rourke) + + + +

          + + Google already monitors your online shopping - but now it's also keeping an eye on what you're buying in real-world stores as part of its latest effort to sell more digital advertising. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Yolanda and Ross Straus have been charged with murder after their adopted special needs son, Brandon, 21, died with burns on 10-12 per cent of his body and lethal levels of medication in his system. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Seven militants were killed during an intelligence-gathering raid by US Special Forces troops against an al Qaeda compound in Yemen on Tuesday morning, US officials said. +

          +
          +
          + +
          + +
          + +
          +
          +
          + + +
          + +
          + +

          + +

          +
          + + Brazil twins celebrate 100th birthday with sassy shoot + + + +
          +

          + + Maria Pignaton Pontin and Paulina Pignaton Pandolfi are becoming centenarians on Wednesday, and decided to ring in the event with a special photo shoot. The images see the pair of women, who have have 65 children, grandchildren and great-grandchildren, enjoying a picnic in flowing tulle skirts. Photographer Camila Lima offered the shoot to the ladies as a birthday gift. +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Google's Deep Mind AI beat the world's top-ranked player of a 3,000 year old board game by just half a point at an event held in the eastern Chinese water town of Wuzhen. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Researchers from the University of Cambridge hope their findings could help in the development of new ways to treat hypoxia - lack of oxygen - in patients. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + More than 2.6 million individuals born between 1987 and 2012 were assessed by the team at the Karolinska Institutet in Sweden for the latest study on premature babies. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + This is the amazing moment artist Craig Greco breaks down in tears after seeing color for the first time. Craig, from Missouri, USA, was surprised with a pair of 'life-changing' Enchroma glasses. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + This is the dangerous moment a car is blasted up into the air before it flips and crashes after hitting a stray manhole cover in the south western Russian city of Kazan. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + In video of the incident, the man is seen shooting victim Harold Strudwick in front of a market in Deerfield Beach, Florida, earlier this month during a road rage confrontation. +

          +
          +
          + +
          + +
          + +
          +
          +
          + + +
          + +
          +

          + Cute or creepy? Gigantic lizard 'who suffers with anxiety' can't stop giving his owner cuddles and kisses +

          + + + + +
          +
          +

          + + Sarah Crow, from Michigan, USA, couldn't resist taking in rescue lizard Winston after she heard about his difficult start to life. This giant lizard reptile gets separation anxiety if his owner leaves him for more than a day which leads to him causing some destruction in her house. He may not be a cat or dog, but that doesn't stop nine-year-old Winston from being very playful and he loves running through a feline tunnel. Ms Crow, from Lansing, Michigan, USA, said: 'Winston is extremely affectionate. After basking, he likes to hang out. Often he will stop basking to sit or interact with me.' +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Martin Galindo-Larios, Jr., 37, a married father of two, died Monday after being sickened in the nacho cheese botulism outbreak at the Valley Oak Food & Fuel gas station near Sacramento, California. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Republican state representative Karl Oliver, of Winona, hit out at the leadership of Louisiana after New Orleans removed four monuments to figures from that era. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + This photo taken May 16, 2017, shows Cali getting some affection from Anita Stout at the Broken Promises Animal Sanctuary in Howell Township, N.J. The dog went missing from a NY family two years ago and showed up emaciated and bleeding two months ago in South Jersey. The sanctuary has been nursing her back to health and found her family, now living in North Carolina. (Thomas P. Costello  /The Asbury Park Press via AP) + + + +

          + + A five-year-old dog named Cali disappeared from her family's house in Long Island in 2015. But in April, she was found at a rest stop in New Jersey and brought back to life. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Nearly 740,000 foreigners who were supposed to leave the United States during a recent 12-month period overstayed their visas, the Homeland Security Department said Monday. +

          +
          +
          + +
          + +
          + +
          +
          +
          + + +
          + +
          +

          + Doggone it! Hilarious moment a newsreader is caught off guard when a LABRADOR appears under her desk on live TV  +

          + + + + +
          +
          +

          + + This is the moment a reporter for Russian station Mir 24 (World 24) was hilariously interrupted while delivering a bulletin when she spotted a black Labrador beneath her desk. In the video, which has now gone viral, she appears confused for a moment then gasps in surprise when the dog jumps up. Despite her shocked reaction, locals watching suggested it was a publicity stunt by the network, whose logo is a black Labrador. +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + The white-spotted giant was caught by fisherman Shawn Steward, from Oxnard, California, in the Channel Islands. Moonfish are the first fish known to by fully warm-blooded. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Marcia Demarcus (right) who was high when she killed Caelyn Olds (left) in a hit-and-run in Georgia in October pleaded guilty to vehicular homicide and was sentenced to 30 total years on Monday. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + A New York man was sentenced to 24 years in federal prison on Monday for smuggling more than 880lbs of pot on commercial flights from San Francisco to Charlotte. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Jeff Bezos, who owns space firm, Blue Origin, was speaking during a Q&A with children at Seattle's Museum of Flight. +

          +
          +
          + +
          + +
          + +
          +
          +
          + + +
          + +
          +

          + Dramatic moment a mechanic who took his client's $190,000 Porsche for a joyride is caught on camera losing control and crashing +

          + + + + +
          +
          +

          + + A blundering mechanic crashed a £170,000 car after taking it out for a test drive. He drove the luxury motor out of the garage and started speeding around the town of Aksay in Russia. Shocking CCTV (above) shows the moment he lost control of the expensive Porsche. It spins across the road trailing a cloud of smoke behind it before coming to rest against the curb. The car had been taken into the garage for a series of diagnostic tests and should have been easy to repair. +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + A jumpy kitten has been captured fleeing a dog by using a wall as an escape route. The HD video shows the white fluffy cat as it flees the clutches of an excited Maltese terrier. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Scientists at St Jude Children's Research Hospital in New York were trying to devise a drug to kill tumors when they realized it had another clearer benefit: fighting the flu. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Two US astronauts have completed  what NASA described as a 'critical' spacewalk to repair a failed piece of equipment that helps power the International Space Station. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Nokia and Apple have settled their legal disputes after signing an agreement to work together. Apple will soon resume selling Nokia digital health products in retail and online stores. +

          +
          +
          + +
          + +
          + +
          +
          +
          + + +
          + +
          +

          + Buzz off! Elephant herd runs away from a swarm of bees flying up their trunks and into their ears  +

          + + + + +
          +
          +

          + + Video footage has caught the hilarious moment dozens of elephants are spooked by a swarm of bees. The video was taken by a tourist at Kruger National Park in South Africa. In the footage, the elephants can be seen charging with their ears flapping. +

          +
          +
          +
          + +
          + +
          + +
          + + +
          + +
          +

          Today's hottestfashion finds

          • Ivanka

            Look lovely in lace with Ivanka's Oscar de la Renta dress

            Read more ›
          • Kate

            Be the best dressed guest like Kate in Alexander McQueen

            Read more ›
          • Kendall

            Go girly in pink ruffles in Lisa Marie Fernandez like Kendall

            Read more ›
          • Kim

            Go hands free with a belt bag like Kim

            Read more ›
          +
          + +
          +
          + + +
          +
          + +   +   +

          DON'T MISS

          + + +
          + +
          + +
          +
          +
          +
          + + +
          + +
          +
          + +
          +
          + +
          + + +
          + +
          + + +
          +
          +
          +

          + Feeding frenzy! Giant ball of bait lures flock of seagulls - but what was lurking underwater is even more sinister +

          + + + + +
          +
          +

          + + Two amazed jetskiers witnessed a flock of seagulls feeding on a giant ball of bait but became scared when they realised the lure was attracting sharks also off the coast of Gold Coast suburb Coolangatta. Annette Hill and her partner Scott recorded the rare encounter on a GoPro by putting the device under the water and filming. The footage revealed a frenzy of sharks feeding on the baitfish as the birds tried to fight from above for a bite sized piece of the action. +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + From the baby who slept right through the photoshoot to the passport booth grump who burst into tears, children don't take kindly to being told to strike a pose - as these comedy snaps show. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Guests at Aruba's Renaissance Island have found themselves unlikely bedfellows with hot-pink flamingos, who've become Instagram stars after posing for selfies. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + China installs glass slide over scenic Yellow River + + + +
          +

          + + Glass bridges in China has taken over thousands of tourists' fear in the past years, but a scenic spot in Shanxi Province has stepped up the game. A 500-meter-long glass slide was constructed close to the Yellow River in northern China, allowing tourists to glide down after reaching to its highest viewing point. The slide opened on May 19 for a test run and expected to open in no time. +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          +
          + +   +   +

          Showbiz extra

          + + +
          + +
          + +
          +
          +
          +
          + + +
          + + +
          + + + + +
          + +
          + + +
          + +
          +
          + +
          +
          + +
          + + +
          + + + +
          + + +
          + +
          + + +
          +
          +
          +

          + Rub a dub dub! Panda pictured excitedly taking a bath and splashing about in its enclosure +

          + + + + +
          +
          +

          + + Hilarious footage shows the moment a panda takes a bath in its enclosure in China. The footage was posted online on May 16 and shows the animal splashing around and enjoying the water. +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Krystal Gordon, from Bundaberg, wore a bikini for the first time at age 31. After negative comments on her viral photo she chose to ignore her bullies and organise a nude photo shoot. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Molly Schuyler, the world's reigning hamburger-eating champion, has triumphed again, setting a new world record for scarfing down a plate piled high with beefburgers. +

          +
          +
          + +
          + +
          + +
          +
          +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Juliet Thompson, 35, and Frank Gallen, 39, from Melbourne, revealed how they sold their businesses and gave away nearly all of their belongings for an adventurous life on the road in their camper van. +

          +
          +
          + +
          + +
          + +
          +
          +
          + +

          + +

          +
          + + + + + +

          + + Melbourne-based bride Kerrin Walton is 'very close' with her grandmother Alison Walter so she asked her to be in the bridal party as a flower girl. +

          +
          +
          + +
          + +
          + +
          +
          +
          +

          + How DOES he do it? Man appears to stop a metal, moving fan using only his head and tongue in puzzling trick  +

          + + + + +
          +
          +

          + + This is the bizarre moment Brad Byers, from America, appears to stop a moving fan using only his head or tongue in a puzzling trick. But sharp-eyed viewers think they've cottoned onto how he does it. The video starts with Brad, who addresses the camera front on, wearing a quirky red shirt. +

          +
          +
          +
          + +
          + +
          + +
          +
          +
          +
          + +   +   +

          TOP SPORT STORIES

          + + +
          + +
          + +
          +
          + +
          +
          + +
          + +
          + + + +
          +
          +
          +
          + + +
          + +
          +
          + +
          +
          + +
          + + +
          + +
          + +
          + + + + + + + + + + + + + + + +
          +
          + +
          +
          + + + +
          + + + + + + + + + + + + + +
          + + + + + + + + + + + + + + +
           
          + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/testdata/e43a8017de92a45dfa218499e7abd25f70236216.json b/test/testdata/e43a8017de92a45dfa218499e7abd25f70236216.json new file mode 100644 index 00000000..e74596d2 --- /dev/null +++ b/test/testdata/e43a8017de92a45dfa218499e7abd25f70236216.json @@ -0,0 +1,18 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "max-age=31", + "Connection": "close", + "Content-Encoding": "gzip", + "Content-Type": "text/html;charset=UTF-8", + "Date": "Tue, 23 May 2017 17:53:10 GMT", + "Vary": "User-Agent, Accept-Encoding", + "X-MOL-GEORESP": "us", + "X-rs-ops": "10.250.203.249:6081", + "x-rs-ben": "cljfe-a6:8181", + "x-rs-time": "Tue, 23 May 2017 17-50-06 GMT", + "x-storage": "channels" + }, + "status_code": 200, + "url": "http://www.dailymail.co.uk/ushome/index.html" +} \ No newline at end of file diff --git a/test/testdata/e5190dae71b80e13739352b4dfacf791319d9fb6.html b/test/testdata/e5190dae71b80e13739352b4dfacf791319d9fb6.html new file mode 100644 index 00000000..7d9bae38 --- /dev/null +++ b/test/testdata/e5190dae71b80e13739352b4dfacf791319d9fb6.html @@ -0,0 +1,165 @@ +International Business, World News & Global Stock Market Analysis

          Latest News

          QUOTE FINDER

          Pro News and Analysis

          CNBC ProHere's what the top mutual funds of 2021 are betting on this year
          CNBC ProThese trends will 'define our future,' RBC says. Here's how to play them
          CNBC ProThese stocks have interesting setups going into 2022, Raymond James says
          CNBC ProJMP downgrades Peloton to market perform, says consumer interest is slumping
          CNBC ProHere are Goldman's favorite dividend and buyback picks to play a volatile 2022
          \ No newline at end of file diff --git a/test/testdata/e5190dae71b80e13739352b4dfacf791319d9fb6.json b/test/testdata/e5190dae71b80e13739352b4dfacf791319d9fb6.json new file mode 100644 index 00000000..7423e08c --- /dev/null +++ b/test/testdata/e5190dae71b80e13739352b4dfacf791319d9fb6.json @@ -0,0 +1,20 @@ +{ + "encoding": "utf-8", + "headers": { + "Access-Control-Allow-Origin": "*", + "Cache-Control": "max-age=15", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "185028", + "Content-Security-Policy": "frame-ancestors 'self' *.cnbc.com *.acorns.com;", + "Content-Type": "text/html; charset=utf-8", + "Date": "Sun, 02 Jan 2022 14:59:11 GMT", + "Expires": "Sun, 02 Jan 2022 14:59:26 GMT", + "Link": ";rel=\"preconnect\",;rel=\"preconnect\",;rel=\"preconnect\", ;rel=\"preconnect\",;rel=\"preconnect\"", + "Vary": "Accept-Encoding, User-Agent", + "X-Aicache-OS": "xxx.xx.15.200:81, xx.x.211.57:80", + "X-Request-Id": "81ea3ce2-1767-45f3-a94d-946e8b591a55" + }, + "status_code": 200, + "url": "https://www.cnbc.com/world/?region=world" +} \ No newline at end of file diff --git a/test/testdata/e62d35bf8b743ea83f1996cf91a9dda4dc291640.html b/test/testdata/e62d35bf8b743ea83f1996cf91a9dda4dc291640.html new file mode 100644 index 00000000..a5de6262 --- /dev/null +++ b/test/testdata/e62d35bf8b743ea83f1996cf91a9dda4dc291640.html @@ -0,0 +1,687 @@ + + + + + + 11-سپتامبر-...-آرماگدون | حدیث-راه-عشق | خانه کتاب و ادبیات ایران + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + + + + + +
          +
          +
          +
          + + + + + + + +
          + + + + + +
          + + + ورود + + ثبت نام + + + + + + +
          +
          +
          +
          + + +
          + + + + +
          +
          +
          +
          +
          + 11 سپتامبر ... آرماگدون | خانه کتاب و ادبیات ایران +
          +
          +
          + صفحات اولیه کتاب +

          + 11 سپتامبر ... آرماگدون

          +

          + + + عراق - تاریخ - حمله ایالات متحده ، 2003 - 2011 م. + + + واقعه 11 سپتامبر 2001م. + + +

          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          پدیدآور + + نويسنده : + + کریمی ، نجمه + - + + + نويسنده : + + یزدخواستی ، فروغ + - + + + نويسنده : + + مختاری ، صفورا + + +
          ناشر + + + + حدیث راه عشق + + (برای تماس با ناشر و خرید کتاب کلیک کنید) + + + + +
          شابک978-964-95633-4-3
          تاریخ نشر + +13860618 +
          قیمت +12,000
          کد دیویی973.929
          زبان کتابفارسی
          محل نشراصفهان - اصفهان
          توضیحات + جلد - + 150 صفحه - + تالیف - + چاپ 2 +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          معرفی مختصر کتاب
          +

          + در فصل نخست کتاب، حادثه‌ی 11 سپتامبر و چند و چون آن، اعم از ویرانی‌های حادثه، عاملان حادثه، نیز نظریه‌های مطرح درباره‌ی این حادثه بحث و تحلیل شده است. بررسی مساله‌ی تروریسم، هم چنین بررسی استراتژی منافع مشترک امریکا و اسرائیل در جنگ امریکا با عراق از دیگر مباحث بخش نخست است. در فصل دوم، با بررسی "موعود" از دیدگاه ادیان الهی و تحلیل وقایع "آرماگدون" (از واژگان موعود)، جریان نوظهور "صهیونیسم مسیحی" بررسی می‌گردد؛ سپس با تحلیل جریان‌ "محافظه‌کاران جدید (New conser vative) در آمریکا، برخی چهره‌های شاخص این جریان معرفی می‌شوند. +

          +
          +
          +
          +
          +
          + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/testdata/e62d35bf8b743ea83f1996cf91a9dda4dc291640.json b/test/testdata/e62d35bf8b743ea83f1996cf91a9dda4dc291640.json new file mode 100644 index 00000000..05dea215 --- /dev/null +++ b/test/testdata/e62d35bf8b743ea83f1996cf91a9dda4dc291640.json @@ -0,0 +1,22 @@ +{ + "encoding": "utf-8", + "headers": { + "AR-ATIME": "0.098", + "AR-CACHE": "BYPASS", + "AR-PoweredBy": "Arvan Cloud (arvancloud.com)", + "AR-Request-ID": "a588eb852cb52c5caec45a528e7af1e9", + "AR-SID": "2012", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Security-Policy": "upgrade-insecure-requests", + "Content-Type": "text/html; charset=utf-8", + "Date": "Sat, 27 Aug 2022 23:06:10 GMT", + "Keep-Alive": "timeout=65", + "Server": "ArvanCloud", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding, Accept-Encoding", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://ketab.ir/book/13a52229-5e3f-479e-8954-092b65e85923" +} \ No newline at end of file diff --git a/test/testdata/e877d12116ffc6d668ae2451f271641af5ea914b.html b/test/testdata/e877d12116ffc6d668ae2451f271641af5ea914b.html new file mode 100644 index 00000000..fcef22ab --- /dev/null +++ b/test/testdata/e877d12116ffc6d668ae2451f271641af5ea914b.html @@ -0,0 +1,2016 @@ + + + + + + +University of Tasmania | University of Tasmania + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + + + +
          +
          +
          +
          +
          +
          + + +
          +
          +
          +
          + +
          +

          We are on campus with COVID safety at the heart of all we do.

          + + + See updates on our coronavirus response + +
          + +
          +
          + +
          +
          +
          +
          +
          +
          + + +
          + +
          + +
          +
          + + University of Tasmania logo + +
          + +
          + + + + + +
          + +
          +
          +
          + + + + + +
          + + +
          + + +
          + +
          +
          +
          + +
          + + +
          + +

          I'm interested in

          +
          + + + + +
          + +
          +

          Are you an international student?

          +
          + + +
          +
          +
          +
          +

          + We want to provide content that's relevant to you. Your options are + stored in a browser cookie which you can delete at any time via the link + below. +

          +
          +
          + + +
          +
          +
          + + + + + + +
          + + +
          + + + + + + +
          + +
          + + + + + +
          + + + +
          + +
          +

          + #1 in climate + action globally + +

          +
          + + + + + + +
          + +
          + +
          + +
          + +

          Browse our courses

          + +
          + + + + +
          +
          +
          +
          +

          The community panel helping the Uni make a good move.

          + +

          The Shake Up is an unvarnished conversation between the Uni and the community, where we’ll all talk about how the University is consolidating in Hobart’s CBD, and what the best version of that move looks like.

          + Find out more about this community panel +
          +
          +
          + + +
          +
          + + +
          +
          +
          +
          +
          +

          Leading the way for a sustainable future

          +

          The University of Tasmania has ranked #1 on climate action for universities internationally.

          +

          We are officially the tertiary sector’s world-leader in taking climate action, with the prestigious Times Higher Education (THE) Impact Rankings rating us number one in climate action globally for 2022.

          + Learn more +
          +
          +
          + + +
          +
          + +
          +
          + + + + +
          +
          + + + +
          +
          + + + + +
          +
          +
          + Still wondering where to begin? +
          + +
          +
          + + + + +
          +
          +
          +

          The University of Tasmania uses cookies to deliver content that’s relevant to you. We rely on cookies to remember your preferences, provide personalised content, and to analyse our website traffic. You consent to our cookies if you click “Accept”. Please refer to our privacy policy for more information.

          + +
          +
          + + +
          +
          +
          + + + + + + + + +
          + + + + + \ No newline at end of file diff --git a/test/testdata/e877d12116ffc6d668ae2451f271641af5ea914b.json b/test/testdata/e877d12116ffc6d668ae2451f271641af5ea914b.json new file mode 100644 index 00000000..b47a5072 --- /dev/null +++ b/test/testdata/e877d12116ffc6d668ae2451f271641af5ea914b.json @@ -0,0 +1,28 @@ +{ + "encoding": "utf-8", + "headers": { + "Age": "2664", + "Cache-Control": "max-age=0, private", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Type": "text/html; charset=utf-8", + "Date": "Fri, 26 Aug 2022 08:17:10 GMT", + "Expires": "Fri, 26 Aug 2022 09:07:09 GMT", + "Last-Modified": "Fri, 26 Aug 2022 06:52:04 GMT", + "Matrix-Upstream": "web-golive-upstream", + "Pragma": "cache", + "Server": "openresty", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding, Accept-Encoding", + "Via": "1.1 squizedge.net", + "X-Cache": "HIT from squizedge.net", + "X-Content-Type-Options": "nosniff", + "X-DEBUG-Phase": "1", + "X-DEBUG-Referer": "0", + "X-Frame-Options": "SAMEORIGIN", + "X-Request-ID": "f43a1713-be46-4fde-a3cd-8160aca49179", + "X-upgrade-enabled": "off" + }, + "status_code": 200, + "url": "https://www.utas.edu.au/" +} \ No newline at end of file diff --git a/test/testdata/e972d4184ae01754942a8d7c9785445cfa3f2934.html b/test/testdata/e972d4184ae01754942a8d7c9785445cfa3f2934.html new file mode 100644 index 00000000..5d49ae3f --- /dev/null +++ b/test/testdata/e972d4184ae01754942a8d7c9785445cfa3f2934.html @@ -0,0 +1,632 @@ + پایگاه اطلاع رسانی شورای نگهبان - shora-gc.ir

          پایگاه اطلاع رسانی شورای نگهبان - shora-gc.ir

          +
          عکس
          +
          +
          +
          + + گزارش تصویری جلسه شورای نگهبان ۱۲ خرداد ۱۳۹۹ + +
          +

          + گزارش تصویری جلسه شورای نگهبان ۱۲ خرداد ۱۳۹۹ +

          +
          صبح امروز دوشنبه (۱۲ خرداد ۱۳۹۹) جلسه شورای نگهبان به ریاست آیت الله جنتی برگزار شد.
          +
          +
          +
          +
          +
          +
          + + گزارش تصویری اولین نشست خبری سخنگوی شورای نگهبان در سال ۱۳۹۹ + +
          +

          + گزارش تصویری اولین نشست خبری سخنگوی شورای نگهبان در سال ۱۳۹۹ +

          +
          اولین نشست خبری دکتر کدخدایی سخنگوی شورای نگهبان در سال ۱۳۹۹ روز شنبه ۱۰ خرداد همزمان به صورت آنلاین و حضوری برگزار شد.
          +
          +
          + +
          + + گزارش تصویری جلسه شورای نگهبان ۳۱ اردیبهشت ۱۳۹۹ + +
          +

          + گزارش تصویری جلسه شورای نگهبان ۳۱ اردیبهشت ۱۳۹۹ +

          +
          عصر امروز چهارشنبه (۳۱ اردیبهشت ماه ۱۳۹۹) جلسه شورای نگهبان با حضور اکثریت فقها و حقوقدانان برگزار شد. عبدالناصر همتی رئیس کل بانک مرکزی نیز میهمان ویژه این جلسه بود.
          +
          +
          + + +
          + + گزارش تصویری جلسه شورای نگهبان ۳۰ اردیبهشت ۱۳۹۹ + +
          +

          + گزارش تصویری جلسه شورای نگهبان ۳۰ اردیبهشت ۱۳۹۹ +

          +
          عصر امروز سه‌شنبه (۳۰ اردیبهشت ۱۳۹۹) جلسه شورای نگهبان با حضور اکثریت فقها و حقوقدانان برگزار شد.
          +
          +
          + +
          +
          + + گزارش تصویری جلسه شورای نگهبان ۲۴ اردیبهشت ۱۳۹۹ + +
          +

          + گزارش تصویری جلسه شورای نگهبان ۲۴ اردیبهشت ۱۳۹۹ +

          +
          عصر امروز چهارشنبه (۲۴ اردیبهشت ۱۳۹۹) جلسه شورای نگهبان با حضور اکثریت فقها و حقوقدانان به ریاست آیت الله جنتی برگزار شد.
          +
          +
          + + +
          + + گزارش تصویری جلسه شورای نگهبان ۱۷ اردیبهشت ۱۳۹۹ + +
          +

          + گزارش تصویری جلسه شورای نگهبان ۱۷ اردیبهشت ۱۳۹۹ +

          +
          صبح امروز چهارشنبه (۱۷ اردیبهشت ۱۳۹۹) جلسه شورای نگهبان به ریاست آیت الله جنتی برگزار شد.
          +
          +
          + +
          + + گزارش تصویری جلسه شورای نگهبان ۳ اردیبهشت ۱۳۹۹ + +
          +

          + گزارش تصویری جلسه شورای نگهبان ۳ اردیبهشت ۱۳۹۹ +

          +
          صبح امروز چهارشنبه (۳ اردیبهشت ۱۳۹۹) جلسه شورای نگهبان به ریاست آیت الله جنتی برگزار شد.
          +
          +
          + +
          + +
          +
          +
          +
          +
          +
          فیلم
          +
          +
          + + + + + متن و حاشیه‌های نشست خبری سخنگوی شورای نگهبان + +

          + متن و حاشیه‌های نشست خبری سخنگوی شورای نگهبان +

          +
          +
          + + + + + برنامه گفتگو محور «بنیانگذار» با حضور آیت‌الله جنتی دبیر شورای نگهبان + +

          + برنامه گفتگو محور «بنیانگذار» با حضور آیت‌الله جنتی دبیر شورای نگهبان +

          +
          +
          + + + + + نشست خبری سخنگوی شورای نگهبان ۱۰ خرداد ۱۳۹۹ + +

          + نشست خبری سخنگوی شورای نگهبان ۱۰ خرداد ۱۳۹۹ +

          +
          +
          + + + + + آخرین نشست خبری دکتر کدخدایی در سال ۱۳۹۸ + +

          + آخرین نشست خبری دکتر کدخدایی در سال ۱۳۹۸ +

          +
          +
          + + + + + نشست خبری بین‌المللی سخنگوی شورای نگهبان + +

          + نشست خبری بین‌المللی سخنگوی شورای نگهبان +

          +
          +
          +
          +
          \ No newline at end of file diff --git a/test/testdata/e972d4184ae01754942a8d7c9785445cfa3f2934.json b/test/testdata/e972d4184ae01754942a8d7c9785445cfa3f2934.json new file mode 100644 index 00000000..95356096 --- /dev/null +++ b/test/testdata/e972d4184ae01754942a8d7c9785445cfa3f2934.json @@ -0,0 +1,20 @@ +{ + "encoding": "utf-8", + "headers": { + "Cache-Control": "no-store, no-cache, must-revalidate, post-check=0, pre-check=0", + "Connection": "Keep-Alive", + "Content-Encoding": "gzip", + "Content-Length": "18442", + "Content-Type": "text/html; charset=utf-8", + "Date": "Sun, 07 Jun 2020 07:59:36 GMT", + "Expires": "Mon, 26 Jul 1997 05:00:00 GMT", + "Keep-Alive": "timeout=15", + "Pragma": "no-cache", + "Server": "Apache", + "Set-Cookie": "version=deleted; expires=Thu, 01-Jan-1970 00:00:01 GMT; Max-Age=0, client_visitor_view_type=original; expires=Tue, 07-Jul-2020 07:59:37 GMT; Max-Age=2592000; path=/", + "Vary": "Accept-Encoding", + "X-Powered-By": "PHP/5.6.21" + }, + "status_code": 200, + "url": "https://www.shora-gc.ir/" +} \ No newline at end of file diff --git a/test/testdata/eb028afa0ae3c6d99945aabf67b0e7bf0ba5489c.html b/test/testdata/eb028afa0ae3c6d99945aabf67b0e7bf0ba5489c.html new file mode 100644 index 00000000..0f121503 --- /dev/null +++ b/test/testdata/eb028afa0ae3c6d99945aabf67b0e7bf0ba5489c.html @@ -0,0 +1,708 @@ + + + + + + آموزش-گام-به-گام-پیکربندی-مسیریابهای-میکروتیک:-آمادگی-آزمون-MTCNA | نشرگستر | خانه کتاب و ادبیات ایران + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + + + + + +
          +
          +
          +
          + + + + + + + +
          + + + + + +
          + + + ورود + + ثبت نام + + + + + + +
          +
          +
          +
          + + +
          + + + + +
          +
          +
          +
          +
          + آموزش گام به گام پیکربندی مسیریابهای میکروتیک: آمادگی آزمون MTCNA | خانه کتاب و ادبیات ایران +
          +
          +
          + صفحات اولیه کتاب +

          + آموزش گام به گام پیکربندی مسیریابهای میکروتیک: آمادگی آزمون MTCNA

          +

          + + + ارتباط بین‌شبکه‌ای + + + ارتباطات بی‌سیم + + + مسیریاب‌ها (شبکه‌های کامپیوتری) + + +

          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          پدیدآور + + نويسنده : + + فخررحیمی ، علیرضا + - + + + نويسنده : + + فخررحیمی ، الهام + - + + + مقدمه : + + نادرپور ، آرش + - + + + مقدمه : + + شهبازیان ، وحید + - + + + مقدمه : + + مقدم ، رضا + - + + + مقدمه : + + جابری ، امیر + + +
          ناشر + + + + نشرگستر + + (برای تماس با ناشر و خرید کتاب کلیک کنید) + + + + +
          شابک978-600-5883-43-5
          تاریخ نشر + +13910308 +
          قیمت +140,000
          کد دیویی004.6
          زبان کتابفارسی
          محل نشرتهران - تهران
          توضیحات + جلد - + 386 صفحه - + تالیف - + چاپ 1 +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          معرفی مختصر کتاب
          +

          + +

          +
          +
          +
          +
          +
          + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/testdata/eb028afa0ae3c6d99945aabf67b0e7bf0ba5489c.json b/test/testdata/eb028afa0ae3c6d99945aabf67b0e7bf0ba5489c.json new file mode 100644 index 00000000..f13cca59 --- /dev/null +++ b/test/testdata/eb028afa0ae3c6d99945aabf67b0e7bf0ba5489c.json @@ -0,0 +1,22 @@ +{ + "encoding": "utf-8", + "headers": { + "AR-ATIME": "0.095", + "AR-CACHE": "BYPASS", + "AR-PoweredBy": "Arvan Cloud (arvancloud.com)", + "AR-Request-ID": "eb39a82381c520adfacf4435c7bcbfe9", + "AR-SID": "2012", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Security-Policy": "upgrade-insecure-requests", + "Content-Type": "text/html; charset=utf-8", + "Date": "Sat, 27 Aug 2022 23:05:06 GMT", + "Keep-Alive": "timeout=65", + "Server": "ArvanCloud", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding, Accept-Encoding", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://ketab.ir/book/f37fad8e-8f0b-4cd9-8875-f5de0e0d86ef" +} \ No newline at end of file diff --git a/test/testdata/ed0d95dac354a4a733fe686f502e51bf5bc1eb43.html b/test/testdata/ed0d95dac354a4a733fe686f502e51bf5bc1eb43.html new file mode 100644 index 00000000..9c5fe5e2 --- /dev/null +++ b/test/testdata/ed0d95dac354a4a733fe686f502e51bf5bc1eb43.html @@ -0,0 +1,147 @@ + اصل هفتاد و سوم
          + +
          کد خبر: ۱۸۱۵
          تاریخ انتشار: ۲۳ بهمن ۱۳۸۹ - ۰۹:۰۵- 12 February 2011
          اصل هفتاد و سوم :
          +
          +شرح و تفسیر قوانین عادی در صلاحیت مجلس شورای اسلامی است. مفاد این اصل مانع از تفسیری که دادرسان, در مقام تمیز حق, از قوانین می کنند نیست.
          +

          +
          +تفسیر 1
          +
          +
          +
          +
          +

          +
          +شماره 78108 تاریخ 22/12/1373
          +
          +شورای‌ محترم‌ نگهبان‌
          +

          +همان‌ گونه‌ که‌ عنایت‌ دارند مطابق‌ اصل‌ 73 قانون‌ اساسی‌، قوانین‌ عادی‌ در موارد ابهام‌ و اجمال‌، تفسیر می‌شوند معهذا نظر به‌ اینکه‌ اصولاً در اکثر قریب‌ به‌ تمامی‌ موارد بین‌ وضع‌ قانون‌ و تفسیر آن‌ فاصله‌ زمانی‌ وجود دارد و در این‌ مدت‌ با اجرای‌ قانون‌ اصلی‌، برای‌ اشخاص‌، حقوقی‌ ایجاد می‌شود و در بسیار از اوقات‌ نیز این‌ حقوق به‌ غیر، منتقل‌ و یا به‌ ارث‌ می‌رسد مضافاً اینکه‌ اجرای‌ قوانین‌ مزبور در موارد معتنابهی‌ مستلزم‌ ارجاع‌ امر به‌ قوانین‌ دیگر نیز هست‌ (کما اینکه‌ هزینه‌ اعتبار طرح‌ها که‌ موضوع‌ قانون‌ بودجه‌ است‌ در بسیاری‌ از موارد ملازمه‌ با پرداخت‌ حقوق کارگر یا خرید اجناس‌ دارد و این‌ امر بر اساس‌ قوانین‌ کار و مدنی‌، حقوقی‌ را برای‌ طرف‌، ایجاد می‌کند) و در نتیجه‌ بر حسب‌ قوانین‌ مرتبط‌ با یکدیگر و در پی‌ اجرای‌ آنها پیچیدگی‌های‌ زیادی‌ در امور مردم‌ به‌ وجود می‌آید.
          +
          +با توجه‌ به‌ این‌ پیچیدگی‌ها هنگامی‌ که‌ تفسیر مؤخّر انجام‌ می‌یابد این‌ سئوال‌ مطرح‌ می‌شود که‌ مطابق‌ اصول‌ کلی‌ آیا تفسیر قانون‌ به‌ لحاظ‌ آثار و نتایج‌ مترتب‌ بر آن‌ تا چه‌ حد باید و می‌تواند عطف‌ به‌ گذشته‌ شود و حقوق ایجاد شده‌ را منتفی‌ نماید و یا کدام‌ دسته‌ از حقوق و آثار را می‌تواند نادیده‌ بگیرد؟
          +
          +به‌ عنوان‌ مثال‌ به‌ موجب‌ قانونی‌ که‌ تحت‌ عنوان‌ اعاده‌ به‌ خدمت‌ کارکنان‌ بازنشسته‌ دولت‌ در سال 1361 به‌ تصویب‌ مجلس‌ رسید دولت‌ اجازه‌ یافت‌ تعدادی‌ از بازنشستگان‌ را اعاده‌ به‌ خدمت‌ نماید.
          +
          +در سال‌ 1366 نیز قانون‌ تعدیل‌ نیروی‌ انسانی‌ تصویب‌ شد و در آن‌ دولت‌ اجازه‌ یافت‌ افراد بازنشسته‌ را به‌ صورت‌ موقت‌ به‌ خدمت‌ بگیرد. از آن‌ پس‌ دولت‌ به‌ دو صورت‌ زیر عمل‌ کرده‌ است:
          +
          +الف‌ ـ اعاده‌ به‌ خدمت‌ تعدادی‌ از بازنشستگان‌ متخصص‌ به‌ استناد قانون‌ مصوب‌ سال‌ 1361.
          +
          +ب‌ ـ تصویب‌ اشتغال‌ موقت‌ برخی‌ از بازنشستگان‌ به‌ استناد ماده‌ 8 قانون‌ تعدیل‌ نیروی‌ انسانی‌ مصوب‌ 1366.
          +
          +قابل‌ ذکر است‌ که‌ مصوبات‌ اعاده‌ به‌ خدمت‌ دولت‌ در فاصله‌ دی‌ ماه‌ 1366 تا خرداد 1373 که‌ بالغ‌ بر 46 تصویب‌نامه‌ و ناظر به‌ تعداد 130 نفر است‌ نیز مورد ایراد ریاست‌ محترم‌ جمهوری‌ وقت‌ به‌ استناد اصل‌ 126 و ریاست‌ محترم‌ مجلس‌ شورای‌ اسلامی‌ به‌ استناد اصل‌ 138 قرار نگرفته‌ است.
          +
          +در سال‌ 1373 مجلس‌ شورای‌ اسلامی‌ با تصویب‌ قانون‌ تفسیر ماده‌ 8 قانون‌ تعدیل‌ نیروی‌ انسانی‌ مقرر داشت:
          +
          +«ممنوعیت‌ اشتغال‌ به‌ کار بازنشستگان‌ موضوع‌ ماده‌ 8 قانون‌ نحوه‌ تعدیل‌ نیروی‌ انسانی‌ شامل‌ اعاده‌ به‌ خدمت‌ بازنشستگان‌ نیز می‌گردد.»
          +
          +اکنون‌ اگر قانون‌ تفسیر فوق الذکر (مصوب‌ 1373) در این‌ قضیه‌، عطف‌ به‌ ماسبق‌ گردد و شامل‌ افرادی‌ هم‌ بشود که‌ در فاصله‌ سالهای‌ 1366 تا 1373 اعاده‌ به‌ خدمت‌ شده‌اند و مسئولیت‌های‌ مختلفی‌ را ایفا کرده‌ و تعدادی‌ از آنها تصمیمات‌ مهم‌ مالی‌ و اداری‌ فراوانی‌ گرفته‌اند موجب‌ می‌شود حقوق ایشان‌ و حقوق دیگر اشخاص‌ و حق‌ کلیه‌ اموری‌ که‌ بر اساس‌ تصمیمات‌ ایشان‌ محقق‌ شده‌ است‌ (از جمله‌ معاملات‌ و عقودی‌ که‌ با تجویز آنان‌ صورت‌ گرفته‌ است‌) محل‌ اشکال‌ قرار گیرد زیرا در قانون‌ تفسیر، مقرر شده‌ است‌ که‌ افراد مزبور، ممنوعیت‌ از اشتغال‌ داشته‌اند و علی‌القاعده‌ بر امر ممنوع‌، آثار امر مجاز نباید مترتّب‌ شود.
          +
          +اشاره‌ به‌ این‌ نکته‌ بجاست‌ که‌ با توجه‌ به‌ قواعد و اصول‌ حقوق اداری‌، امور و حقوق مزبور بر اساس‌ فهم‌ مجری‌ از قانون‌ حاکم‌ در زمان‌ خود ایجاد شده‌ است‌ به‌ ویژه‌آن‌که‌تفسیر، علی‌ الاصول‌ در مواردی‌ صورت‌ می‌گیرد که‌ قانون‌، مشتمل‌ بر چند معنی‌ است‌ و اتخاذ هر یک‌ از آنها قبل‌ از تغییر مرجع‌ ذی‌صلاح‌، مغایر با قانون‌ اصلی‌ نمی‌باشد و همانطور که‌ در بالا یاد آوری‌ شد در موضوع‌ مطروحه‌، ریاست‌ محترم‌ جمهوری‌ وقت‌ و ریاست‌ محترم‌ مجلس‌ شورای‌ اسلامی‌ نیز اعلام‌ مغایرت‌ مصوبات‌ را با قانون‌ اعلام‌ نداشته‌اند.
          +
          +با عنایت‌ به‌ آنچه‌ گذشت‌ خواهشمند است‌ با توجه‌ به‌ اصول‌ 73 و 98 قانون‌ اساسی‌ نظریه‌ تفسیری‌ شورای‌ محترم‌ نگهبان‌ را در مورد اینکه‌ آیا با قوانین‌ تفسیری‌ می‌توان‌ حقوق و شرایطی‌ را که‌ طبق‌ قوانین‌ اولیه‌ برای‌ اشخاص‌ ایجاد شده‌ است‌ خصوصاً در موارد زیر به‌ طور یک‌ جانبه‌ زایل‌ نمود، اعلام‌ فرمایند:
          +
          +1_ هنگامی‌ که‌ قانون‌ تفسیری‌ جدید، حوزه‌ شمول‌ قانون‌ اصلی‌ را توسعه‌ می‌دهد به‌ نحوی‌ که‌ برای‌ افراد در زمان‌ مقدم‌ بر تصویب‌، تکالیفی‌ ایجاد می‌نماید.
          +
          +2_ تجدید یا تضییع‌ حقوق استخدامی‌ که‌ در زمان‌ گذشته‌ برای‌ افراد ایجاد شده‌ است.
          +
          +3_ تأثیر در عقود و معاملات‌ گذشته‌ و سایر تصمیماتی‌ که‌ برای‌ اشخاص‌ غیر دولتی‌، حقوقی‌ ایجاد کرده‌ است.
          +
          +4_ تسرّی‌ مستقیم‌ یا غیرمستقیم‌ مقررات‌ جزائی‌ به‌فعل‌ یا ترک‌فعلی‌ که‌ در گذشته‌ انجام‌ شده‌ است‌.
          +
          +رئیس‌ جمهور _ اکبر هاشمی‌ رفسنجانی
          +

          +
          +
          +
          +
          +شماره 583/21/76 تاریخ 10/3/1376
          +
          +ریاست‌ محترم‌ جمهوری‌ اسلامی‌ ایران‌
          +

          +با سلام‌، نامه‌ شماره‌ 78108 مورخ‌ 22/12/1373 در جلسه‌ مورخ‌ 7/3/1376 شورای‌ نگهبان‌ مطرح‌ شد و نظر تفسیری‌ شورا بدین‌ شرح‌ اعلام‌ می‌گردد:
          +
          +«1_ مقصود از تفسیر، بیان‌ مراد مقنّن‌ است‌ بنابراین‌ تضییق‌ و توسعه‌ قانون‌ در مواردی‌ که‌ رفع‌ ابهام‌ قانون‌ نیست‌، تفسیر، تلقی‌ نمی‌شود.
          +
          +2_ تفسیر از زمان‌ بیان‌ مراد مقنّن‌ در کلیه‌ موارد لازم‌ الاجرا است‌. بنابراین‌ در مواردی‌ که‌ مربوط‌ به‌ گذشته‌ است‌ و مجریان‌ برداشت‌ دیگری‌ از قانون‌ داشته‌اند و آن‌ را به‌ مرحله‌ اجراء گذاشته‌اند تفسیرق قانون‌ به‌ موارد مختومة‌ مذکور، تسرّی‌ نمی‌یابد.»
          +
          +دبیر شورای‌ نگهبان _ احمد جنتی‌
          +

          +
          +
          +
          +
          +استفساریه‌ پیرامون‌ نظریة‌ 10/3/1376 شورای‌ نگهبان‌
          +
          +شماره 100/40/18448 تاریخ 30/3/1379
          +
          +حضرت‌ آیت‌الله جنتی
          +
          +دبیر محترم‌ شورای‌ نگهبان
          +

          +با سلام‌ و تحیت
          +
          +همانگونه‌ که‌ استحضار دارید شورای‌ محترم‌ نگهبان‌ طی‌ نظریه‌ شماره‌ 583/21/76 مورخ‌ 10/3/1376 در خصوص‌ تفسیر قوانین‌ چنین‌ اعلام‌ نظر فرموده‌اند:
          +
          +1_ ...
          +
          +2_ تفسیر از زمان‌ بیان‌ مراد مقنن‌ در کلیه‌ موارد لازم‌ الاجرا است‌. بنابراین‌، در مواردی‌ که‌ مربوط‌ به‌ گذشته‌ است‌ و مجریان‌ برداشت‌ دیگری‌ از قانون‌ داشته‌اند و آن‌ را به‌ مرحله‌ اجراء گذاشته‌اند، تفسیر قانون‌ به‌ موارد مختومه‌ مذکور تسری‌ ندارد.
          +
          +استنباط‌ این‌ وزارت‌ از توضیح‌ قسمت‌ اخیر نظریه‌ تفسیری‌ آن‌ شورای‌ محترم‌ این‌ است‌ که‌ چنانچه‌ موضوعی‌ مختومه‌ نشده‌ بلکه‌ بعد از تفسیر قانون‌ به‌ اجراء در می‌آید و امکان‌ استفاده‌ از نظریه‌ تفسیری‌ نسبت‌ به‌ مورد وجود دارد، اولی‌ تبعیّت‌ از نظریة‌ تفسیری‌ است‌ زیرا با انجام‌ تفسیر معنای‌ واقعی‌ و حقیقت‌ هدف‌ قانونگذار برای‌ مجریان‌ روشن‌ وابهامات‌و شبهاتی‌ که‌ در استنباط‌ از آن‌ وجود داشته‌ زائل‌ می‌گردد. لذا مستدعی‌ است‌ نظریه‌ شورای محترم‌ نگهبان‌ را در این‌ خصوص‌ به‌ این‌ وزارت‌ ابلاغ‌ فرمائید.
          +
          +حبیب‌الله بیطرف _ وزیر نیرو
          +

          +
          +
          +
          +
          +شماره‌ 1540/21/79 تاریخ 20/10/1379
          +
          +جناب‌ آقای‌ مهندس‌ بیطرف
          +
          +وزیر محترم‌ نیرو
          +

          +عطف‌ به‌ نامه‌ شماره‌ 100/40/18448 مورخ‌ 30/3/1379 مبنی‌ بر تقاضای‌ ارائه‌ توضیح‌ نسبت‌ به‌ نظر تفسیری‌ شماره‌ 583/21/76 مورخ‌ 10/3/1376 بدینوسیله‌ اعلام‌ می‌گردد:
          +
          +«همانطوری‌ که‌ در نظریة‌ تفسیری‌ شماره‌ 583/21/76 مورخ‌ 10/3/1376 آمده‌ است‌ که‌ تفسیر از زمان‌ بیان‌ مراد مقنن‌ در کلیه‌ موارد لازم‌ الاجراء است‌ با این‌ قید که‌ به‌ موارد مختومه‌ تسری‌ نمی‌یابد.
          +
          +بنابراین‌ چنانکه‌ تا هنگام‌ لازم‌ الاجراء شدن‌ تفسیر قانون‌، موضوعی‌ مختومه‌ نشده‌ باشد باید مطابق‌ نظریه‌ تفسیری‌ اقدام‌ گردد.»
          +
          +
          دبیر شورای‌ نگهبان _ احمد جنتی
          ارسال نظر
          + +
          +
          +
          نام:
          +
          +
          +
          +
          +
          ایمیل:
          +
          +
          +
          +
          +
          +
          +
          * نظر:
          +
          +
          +
          + + +
          +
          * captcha:
          +
          +
          +
          +
          + + + +
          +
          + +
          +
          +
          +
          +
          آخرین اخبار

          فیلم|متن و حاشیه‌های نشست خبری سخنگوی شورای نگهبان

          گام مهم شورای نگهبان در حمایت از یگان حفاظت محیط زیست

          فیلم|برنامه گفتگو محور «بنیانگذار» با حضور آیت‌الله جنتی دبیر شورای نگهبان

          ناگفته‌های آیت‌الله جنتی از امام (ره)، مدیریت رهبر انقلاب، عزل آیت‌الله منتظری، شورای رهبری و...

          فیلم|روایت متفاوت آیت‌الله یزدی از ۴۱ سال رهبری انقلاب و نظام اسلامی

          حوزویان در واکاوی ابعاد شخصیتی امام خمینی(ره) رسالت مهمی را برعهده دارند

          امام (ره) حق بزرگی گردن همه ما دارد

          بیانیه شورای نگهبان به مناسبت سالروز عروج ملکوتی امام خمینی (ره) و قیام ۱۵ خرداد ۱۳۴۲

          امام خمینی (ره) و حضرت آقا میدان را برای فعالیت زنان باز کردند

          گزارش تصویری جلسه شورای نگهبان ۱۲ خرداد ۱۳۹۹

          پربازدید ها

          قانون انتخابات مجلس شورای اسلامی

          ایجاد سامانه دریافت گزارش تخلفات انتخاباتی از سوی شورای نگهبان

          تایید لایحه اصلاح قانون تعیین تکلیف تابعیت فرزندان حاصل از ازدواج زنان ایرانی با مردان خارجی

          برای شنیدن راهکارهای مفید گوش شنوا داریم اما توجهی به باج‌خواهی و لابی نمی‌کنیم/ شورای نگهبان برای کسی فرش قرمز پهن نمی‌کند

          فیلم|روایت خبرنگار صداوسیما از روند بررسی صلاحیت‌ها در هیات مرکزی نظارت بر انتخابات

          احراز هویت در انتخابات مجلس قطعا الکترونیکی خواهد بود

          امنای ملّت و امّت؛ گذری بر تاریخچه شورای نگهبان

          موشن گرافی|معرفی شورای نگهبان و نحوه انتخاب اعضاء

          نظر شورای نگهبان درباره آخرین مصوبات مجلس اعلام شد

          فیلم|روزهای پرکار شورای نگهبان

          \ No newline at end of file diff --git a/test/testdata/ed0d95dac354a4a733fe686f502e51bf5bc1eb43.json b/test/testdata/ed0d95dac354a4a733fe686f502e51bf5bc1eb43.json new file mode 100644 index 00000000..47763ffa --- /dev/null +++ b/test/testdata/ed0d95dac354a4a733fe686f502e51bf5bc1eb43.json @@ -0,0 +1,19 @@ +{ + "encoding": "utf-8", + "headers": { + "Cache-Control": "no-store, no-cache, must-revalidate, post-check=0, pre-check=0", + "Connection": "Keep-Alive", + "Content-Encoding": "gzip", + "Content-Length": "17544", + "Content-Type": "text/html; charset=utf-8", + "Date": "Sun, 07 Jun 2020 08:03:40 GMT", + "Expires": "Mon, 26 Jul 1997 05:00:00 GMT", + "Keep-Alive": "timeout=15", + "Pragma": "no-cache", + "Server": "Apache", + "Vary": "Accept-Encoding", + "X-Powered-By": "PHP/5.6.21" + }, + "status_code": 200, + "url": "https://www.shora-gc.ir/fa/news/1815/%D8%A7%D8%B5%D9%84-%D9%87%D9%81%D8%AA%D8%A7%D8%AF-%D9%88-%D8%B3%D9%88%D9%85" +} \ No newline at end of file diff --git a/test/testdata/ed653123f732625ca1372e841eebdd1b279ee0e4.html b/test/testdata/ed653123f732625ca1372e841eebdd1b279ee0e4.html new file mode 100644 index 00000000..992becea --- /dev/null +++ b/test/testdata/ed653123f732625ca1372e841eebdd1b279ee0e4.html @@ -0,0 +1,1114 @@ + + + + + + + + + + + + + + + + Juliette Kayyem Is Running for Governor of Massachusetts + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + +
          + +
          + + +
          +
          + +
          + + Advertisement + +
          + + + +
          + + + +
          + + +
          + +
          + +
          + + + +
          + + + + +
          + +
          + In This Section: + +
          + + +
          + + +
          + +
          + +
          + + + + + + +
          + + +
           
          + +
          + + +
          + + +
          + +
          + + +
          +
          +
          + + +
          +
          +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/testdata/ed653123f732625ca1372e841eebdd1b279ee0e4.json b/test/testdata/ed653123f732625ca1372e841eebdd1b279ee0e4.json new file mode 100644 index 00000000..512107c3 --- /dev/null +++ b/test/testdata/ed653123f732625ca1372e841eebdd1b279ee0e4.json @@ -0,0 +1,23 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "max-age=900, must-revalidate", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Type": "text/html; charset=UTF-8", + "Date": "Tue, 23 May 2017 17:53:08 GMT", + "Last-Modified": "Tue, 23 May 2017 17:53:08 GMT", + "Link": "; rel=\"https://api.w.org/\", ; rel=shortlink", + "Server": "nginx", + "Transfer-Encoding": "chunked", + "Vary": "Cookie", + "X-Cache": "MISS", + "X-Pingback": "http://www.bostonmagazine.com/xmlrpc.php", + "X-Powered-By": "PHP/5.6.30", + "X-batcache": "True", + "X-device": "_desktop_", + "x-cache-key": "http://_desktop_www.bostonmagazine.comGET/news/blog/2013/08/21/juliette-kayyem-jumps-in-for-guv/" + }, + "status_code": 200, + "url": "http://www.bostonmagazine.com/news/blog/2013/08/21/juliette-kayyem-jumps-in-for-guv/" +} \ No newline at end of file diff --git a/test/testdata/ee3462cb39f288b9bb019ebfa00df4ced100f71c.html b/test/testdata/ee3462cb39f288b9bb019ebfa00df4ced100f71c.html new file mode 100644 index 00000000..517bb0fa --- /dev/null +++ b/test/testdata/ee3462cb39f288b9bb019ebfa00df4ced100f71c.html @@ -0,0 +1,451 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Apple might buy Beats for $3.2 billion + + + + + + + + + + + + + + + +
          + +
          +
          +
          + + + + + + + +
          +

          Apple might buy Beats for $3.2 billion

          + + + + + +
          +
          +
          +
          +
          +
          +
          +
            +
          • +
            +

            A lot of rumors have been flying around that Apple is in talks with Jimmy Iovine and Dr. Dre, co-founders of the trendy Beats Electronics headphones and streaming music service for an estimated $3.2 billion.

            +
            + +
            +

            Sounds like a good match to me. It’s all about style when it comes to Apple and Beats headphones definitely have style. But it’s also about streaming.

            +
            + +
            +

            Roughly a decade ago Apple launched the iPod and the iStore and most folks thought they were pretty cool, but now the pundits are beginning to say that the days of digital downloads is ending, the new cool is streaming. And that may be the prize Apple is looking for.

            +
            + +
            +

            Of course there is also the question about what’s going to happen with Beats Electronics arrangement with HP who builds Beats audio into their laptops.

            +
            + +
            +

            “Lost amid the buzz about Apple's potential acquisition of Beats Electronics is Beat's long-standing partnership with another major technology company, HP. Hewlett-Packard directly competes with Apple in the competitive laptop and desktop computer market, and for several years, nearly all its mainstream and high-end PCs have featured Beats Audio technology and Beats branding.

            +
            + +
            +

            “Should Beats Electronics become part of the Apple mothership, it's hard to imagine Apple would allow the HP partnership to continue. That could be a blow to HP, as its Beats branding is one of the few unique bullet points the company can point to in a line of PCs that are perfectly fine, but generally lack sizzle or standout features,” said Dan Ackerman in a CNET article.

            +
            + +
            +

            I’m not so sure that Apple is going to pull the plug on HP’s deal with Beats. (At least not for a while, if ever.) Assuming that Apple isn’t buying the company just for the colorful headphones (they could easily design and sell colorful headphones of their own and people would buy them) they are more interested in the streaming service. And the trick with monthly services is to get as many subscribers as possible. Apple wouldn’t really care how subscribers sign on or what device they are using. Sure they would love to have everyone using an Apple product to access the service but I think this is one of the few cases where Apple might just decide to play nice for a change.

            +
            + +
            +

            Besides, it’s not like HP is dominating the laptop market these days.

            +
            + +
          • + +
          +
          + +
          + +
          +
          + +
          +
          +
          +
          +
          +

          Related Stories

          +
          + + + + + +
          +

          What Is Kratom and Is It Dangerous?

          + + + + + +
          +
          +
          + + + + + +
          +

          The best platform to create your e-commerce website

          + + + + + +
          +
          +
          + + + + + +
          +

          Is My Site Worth My Audience’s Time?

          + + + + + +
          +
          +
          +
          +
          + +
          + + + + + + + + + + + diff --git a/test/testdata/ee3462cb39f288b9bb019ebfa00df4ced100f71c.json b/test/testdata/ee3462cb39f288b9bb019ebfa00df4ced100f71c.json new file mode 100644 index 00000000..29b9c37a --- /dev/null +++ b/test/testdata/ee3462cb39f288b9bb019ebfa00df4ced100f71c.json @@ -0,0 +1,22 @@ +{ + "encoding": "UTF-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "0", + "Cache-Control": "max-age=60, public", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "7859", + "Content-Type": "text/html; charset=UTF-8", + "Date": "Tue, 23 May 2017 17:55:10 GMT", + "Vary": "Accept-Encoding", + "Via": "1.1 varnish", + "X-Cache": "MISS", + "X-Cache-Hits": "0", + "X-Powered-By": "PHP/5.5.9-1ubuntu4.20", + "X-Served-By": "cache-iad2649-IAD", + "X-Timer": "S1495562110.164485,VS0,VE581" + }, + "status_code": 200, + "url": "http://www.tgdaily.com/web/100381-apple-might-buy-beats-for-32-billion" +} \ No newline at end of file diff --git a/test/testdata/f02601af068bc9e3c340d21a60e10b68f1a1669c.html b/test/testdata/f02601af068bc9e3c340d21a60e10b68f1a1669c.html new file mode 100644 index 00000000..6c61a993 --- /dev/null +++ b/test/testdata/f02601af068bc9e3c340d21a60e10b68f1a1669c.html @@ -0,0 +1,12 @@ +@article{10.2307/30078788, + ISSN = {07908113}, + URL = {http://www.jstor.org/stable/30078788}, + author = {Humphrey Lloyd}, + journal = {The Transactions of the Royal Irish Academy}, + pages = {171--177}, + publisher = {Royal Irish Academy}, + title = {On a New Case of Interference of the Rays of Light}, + volume = {17}, + year = {1831} +} + diff --git a/test/testdata/f02601af068bc9e3c340d21a60e10b68f1a1669c.json b/test/testdata/f02601af068bc9e3c340d21a60e10b68f1a1669c.json new file mode 100644 index 00000000..6c7dc117 --- /dev/null +++ b/test/testdata/f02601af068bc9e3c340d21a60e10b68f1a1669c.json @@ -0,0 +1,24 @@ +{ + "encoding": "ISO-8859-1", + "headers": { + "Accept-Ranges": "bytes", + "Connection": "keep-alive", + "Content-Disposition": "attachment;filename=10.2307_30078788.txt;", + "Content-Encoding": "gzip", + "Content-Type": "text/plain", + "Date": "Fri, 28 May 2021 11:39:35 GMT", + "Server": "Apache/2.4.29 (Ubuntu)", + "Set-Cookie": "ReferringRequestId=citation-export:1db893c8cec511ca21cacaf0c8eced2f; Path=/; SameSite=Lax; Secure", + "Vary": "Cookie,Accept-Encoding,Fastly-SSL,Origin,X-Requested-Host", + "Via": "1.1 varnish", + "X-Cache": "MISS", + "X-Cache-Hits": "0", + "X-Frame-Options": "SAMEORIGIN", + "X-JSTOR-Restarts": "0", + "X-Served-By": "cache-fra19142-FRA", + "X-Timer": "S1622201975.856472,VS0,VE372", + "transfer-encoding": "chunked" + }, + "status_code": 200, + "url": "https://www.jstor.org/citation/text/30078788" +} \ No newline at end of file diff --git a/test/testdata/f21b22316b63d66edf6289dc822626435e7d510f.html b/test/testdata/f21b22316b63d66edf6289dc822626435e7d510f.html new file mode 100644 index 00000000..706ad7c7 --- /dev/null +++ b/test/testdata/f21b22316b63d66edf6289dc822626435e7d510f.html @@ -0,0 +1,862 @@ + + + + + + + + Male vs. Female Entrepreneurs: How Are They Different? + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + +
          + +
          + + + +
          + +
          +
          + +
          + +
          +
          + +
          +
          +
          +
          +
          + Product and service reviews are conducted independently by our editorial team, but we sometimes make money when you click on links. + Learn more. +
          +
          + + Start Your Business + + + Entrepreneurs + +
          + +
          + +

          Male vs. Female Entrepreneurs: How Are They Different?

          + +
          + +
          +
          +
          + +
          +
          +
          + + Male vs. Female Entrepreneurs: How Are They Different? +
          + + Credit: Brian A. Jackson/Shutterstock +
          +

          + Female entrepreneurs might be outdoing men when it comes to running successful businesses this year.

          +

          + About 40 percent of women surveyed started running their business within the last five years, and nearly 70 percent of them expect their revenue to increase this year, according to Bank of America's spring 2014 Small Business Owner Report.

          +

          + While nearly one-third of the women surveyed said they think they have less access to capital and new business opportunities than male small business owners do, 18 percent of women said they think they have more access to clients than men do.

          +

          + Moreover, the survey found that women plan to hire more than men do: 56 percent of women plan to hire more employees this year, as opposed to 50 percent of men, and 68 percent of women expect their business to continue growing over the next five years. [5 Industries Where Women-Owned Businesses Survive Longer ]

          +

          + The survey also found some interesting differences between female entrepreneurs and their male counterparts. When asked about their key character traits, 58 percent of women considered multitasking to be a strength, versus only 40 percent of men. Women were also 10 percent more likely to list creativity and 5 percent more likely to list empathy as key character traits for employees. On the other hand, 30 percent of men listed confidence as their strongest attribute, as opposed to only 24 percent of women.

          +

          + While 72 percent of small business owners admitted they've made significant personal sacrifices in order to run their business, the results showed that the sacrifices female entrepreneurs make are significantly different from those of their male counterparts.

          +

          + According to the findings, women are more likely to sacrifice time for themselves and their social lives for their businesses, whereas men are more likely to sacrifice time with their spouse and time with their children. Women are also more likely to hire their children, while 27 percent of men said that it would be better if their children did not work for their business.

          +

          + Despite these differences, both men and women cited "not spending enough time with my loved ones" as their top regret. They were also in agreement about their greatest accomplishments: having enough money to support their families, being their own boss and doing what they love.

          +

          + Originally published on Business News Daily.

          + +
          + + + +
          + + + +
          + + + + +
          +
          start-your-business
          + See All +
          + + + +
          + + +
          + + +
          +
          + +
          +
          + +
          + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/testdata/f21b22316b63d66edf6289dc822626435e7d510f.json b/test/testdata/f21b22316b63d66edf6289dc822626435e7d510f.json new file mode 100644 index 00000000..e3127ca7 --- /dev/null +++ b/test/testdata/f21b22316b63d66edf6289dc822626435e7d510f.json @@ -0,0 +1,19 @@ +{ + "encoding": "UTF-8", + "headers": { + "Cache-Control": "max-age=0, no-cache", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "18491", + "Content-Type": "text/html; charset=UTF-8", + "Date": "Tue, 23 May 2017 17:53:57 GMT", + "Expires": "Tue, 23 May 2017 17:53:57 GMT", + "Pragma": "no-cache", + "Server": "nginx", + "Set-Cookie": "__uzma=59247735090a24.83809983; expires=Fri, 21-May-2027 17:53:57 GMT; Max-Age=315360000; path=/, __uzmd=1495562037; expires=Fri, 21-May-2027 17:53:57 GMT; Max-Age=315360000; path=/, __uzmc=345991097772; expires=Fri, 21-May-2027 17:53:57 GMT; Max-Age=315360000; path=/, __uzmb=1495562037; expires=Fri, 21-May-2027 17:53:57 GMT; Max-Age=315360000; path=/", + "Surrogate-Control": "content=\"ESI/1.0\"", + "Vary": "Accept-Encoding" + }, + "status_code": 200, + "url": "http://www.businessnewsdaily.com/6762-male-female-entrepreneurs.html?cmpid=514642_20140715_27858876" +} \ No newline at end of file diff --git a/test/testdata/f5746cf11768798eb08f658e8c4b8918fd455cbc.html b/test/testdata/f5746cf11768798eb08f658e8c4b8918fd455cbc.html new file mode 100644 index 00000000..c4c695e4 --- /dev/null +++ b/test/testdata/f5746cf11768798eb08f658e8c4b8918fd455cbc.html @@ -0,0 +1,693 @@ + + + + + + روش‌های-تحقیق-تلفیقی | ثامن-الحجج | خانه کتاب و ادبیات ایران + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + + + + + +
          +
          +
          +
          + + + + + + + +
          + + + + + +
          + + + ورود + + ثبت نام + + + + + + +
          +
          +
          +
          + + +
          + + + + +
          +
          +
          +
          +
          + روش‌های تحقیق تلفیقی | خانه کتاب و ادبیات ایران +
          +
          +
          + صفحات اولیه کتاب +

          + روش‌های تحقیق تلفیقی

          +

          + + + علوم اجتماعی - تحقیق - روش‌شناسی + + + تحقیق - ارزشیابی + + +

          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          پدیدآور + + نويسنده : + + کرسول ، جان + - + + + نويسنده : + + پلانو‌کلارک ، ویکی + - + + + مترجم : + + نیازی ، محسن + - + + + مترجم : + + زارعی ، عباس + + +
          ناشر + + + + ثامن الحجج + + (برای تماس با ناشر و خرید کتاب کلیک کنید) + + + + +
          شابک978-964-2823-35-2
          تاریخ نشر + +13870305 +
          قیمت +
          کد دیویی300.72
          زبان کتابفارسی
          محل نشرتهران - تهران
          توضیحات + جلد 1 - + 172 صفحه - + ترجمه - + چاپ 1 +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          معرفی مختصر کتاب
          +

          + "روش‌های تحقیق تلفیقی"، طرحی تحقیقی شامل فرضیه‌های فلسفی و شیوه‌های تحقیق است. تحقیق تلفیقی به عنوان روش‌شناسی، شامل فرضیه‌های فلسفی است که مسیر جمع‌آوری و تحلیل اطلاعات و ترکیب رویکردهای کیفی و کمی مراحل بی‌شمار فرآیند تحقیق را هدایت می‌کند. این تحقیق به عنوان یک شیوه بر جمع‌آوری، تحلیل و تلفیق اطلاعات کمی و کیفی در یک پژوهش مجزا یا مجموعه‌ای از پژوهش‌ها تمرکز می‌کند و مهم‌ترین فرض آن این است که به کارگیری ترکیب رویکردهای کمی و کیفی درک بهتری نسبت به زمانی که هر رویکرد را جداگانه بکارمی‌بریم، از موضوع تحقیق به دست می‌دهد. آن‌چه که اهمیت استفاده از روش تحقیق تلفیقی را نشان می‌دهد این است که محققان می‌توانند اعداد را درمتن و لغات شرکت‌کنندگان بگنجانند و لغات شرکت‌کنندگان را همراه با اعداد، گرایش‌ها و نتایج آماری ارائه کنند، در نتیجه ترکیبی از هر دو نوع داده‌ها می‌تواند تحلیل کامل‌تری ازموضوع تحقیق را به وجود آورد. روش تحقیق تلفیقی توسط نویسندگان به عنوان یک شیوۀ قابل قبول در علوم اجتماعی پذیرفته شده، هم‌چنین بسیاری از دانشمندان به این شیوه علاقه‌مند شده‌اند و همین امر باعث ارتقای این شیوه در طی چند دهۀ اخیر شده است. بر همین اساس نگارندگان در کتاب حاضر به موضوع روش‌های تحقیق تلفیقی و میزان اهمیت و کاربرد آن در تحقیقات پرداخته‌اند و مباحثی چون: شناخت شیوه‌های تحقیق تلفیقی، بررسی عوامل بنیادین در تحقیق، یافتن و ارزیابی کردن پژوهش‌های تلفیقی، انتخاب روش تحقیق تلفیقی و شیوه‌های نوشتن طرح تحقیق تلفیقی را بررسی کرده‌اند. +

          +
          +
          +
          +
          +
          + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/testdata/f5746cf11768798eb08f658e8c4b8918fd455cbc.json b/test/testdata/f5746cf11768798eb08f658e8c4b8918fd455cbc.json new file mode 100644 index 00000000..363367e3 --- /dev/null +++ b/test/testdata/f5746cf11768798eb08f658e8c4b8918fd455cbc.json @@ -0,0 +1,22 @@ +{ + "encoding": "utf-8", + "headers": { + "AR-ATIME": "0.117", + "AR-CACHE": "BYPASS", + "AR-PoweredBy": "Arvan Cloud (arvancloud.com)", + "AR-Request-ID": "4c31d3e435ada3e2000c87ea8cab264f", + "AR-SID": "2012", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Security-Policy": "upgrade-insecure-requests", + "Content-Type": "text/html; charset=utf-8", + "Date": "Sat, 27 Aug 2022 22:57:21 GMT", + "Keep-Alive": "timeout=65", + "Server": "ArvanCloud", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding, Accept-Encoding", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://ketab.ir/book/667c900a-69bd-4a1a-a651-1870d2f63a68" +} \ No newline at end of file diff --git a/test/testdata/f5ed3a88709359f26ae3b0cfe746990b21c678b4.html b/test/testdata/f5ed3a88709359f26ae3b0cfe746990b21c678b4.html new file mode 100644 index 00000000..51134b7b --- /dev/null +++ b/test/testdata/f5ed3a88709359f26ae3b0cfe746990b21c678b4.html @@ -0,0 +1,1375 @@ + + + +BBC NEWS | Business | Inside the Bentley factory + + + + + + + + + + +BBC NEWS + + + + + + + + + + +Americas +Africa +Europe +Middle East +South Asia +Asia Pacific + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          BBCiNEWS  SPORT  WEATHER  WORLD SERVICE  A-Z INDEX    

          + + +
          BBC News World Edition
          + + + + + + + + + + + + + + + + + + + + + + + + + + +
             +  You are in: Business   + + +
           + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          News Front Page
          Africa
          Americas
          Asia-Pacific
          Europe
          Middle East
          South Asia
          UK
          Business
          E-Commerce
          Economy
          Market Data
          Entertainment
          Science/Nature
          Technology
          Health
          -------------
          Talking Point
          -------------
          Country Profiles
          In Depth
          -------------
          Programmes
          -------------
          + + + + + + + + + + + + + + + + + + + + + +
          BBC Sport
          BBC Weather
          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          SERVICES +
          ------------- +
          + + + + + + + + + + + + + + + + + +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          EDITIONS +
          +
           + + + + + + +Tuesday, 24 December, 2002, 10:09 GMT + + + + + + +
          Inside the Bentley factory
          + + + + + + + +
          + Bentley cars + +
          The Bentley "sweetshop" excites customers
          + +
          + + + + + +
          +
          + + + + + + + + + + + + + + + + + +
          + +
          + + + + + +
          + + +
          + + + + + +

          +An impressive selection of multi-coloured Bentleys greet us as we enter the factory that has been building Bentley motor cars for decades. +

          + + + + +
          + + +
          + Bentley + +
          Bentley's "trimming ladies" hard at work
          + +
          + + +
          + + + +"Customers get quite excited when they see all the colours. We call it the sweetshop," says our guide, Bentley worker Dave Maddock, who has been with the company for 28 years. +

          +Beyond the bulky Bentley bodies, and despite the factory's size, this part of the manufacturing plant feels more like a workshop than a car factory. +

          +Of the 2,500-strong Bentley labour force, 1,000 people work in "wood or trim", says Mr Maddock. +

          +"Compared with some other car manufacturers, it's quite sedate, really," he observes. +

          +Leather and trim +

          +As we move further into the factory, an overpowering smell of leather is rising from large rolls and piles of full-size cow hides that are about to be cut into smaller sizes. +

          + + + + +
          + + +
          + Cow hides + +
          Bentleys should smell of leather
          + +
          + + +
          + + + +The hides are rolled out flat on large tables and slashed into smaller pieces, sometimes by a robot, sometimes by men wielding large pairs of sharp scissors. +

          +Nearby, a row of women operate old-fashioned Singer sewing machines, stitching deadly accurate seams which shape the leather into lush seat and headrest covers. +

          +Slicing away excess edging with a razor-sharp small knife, "trimming lady" Helen Cowling is adamant. "No drink," she quips. +

          +Active sales force +

          +The atmosphere is pleasant, and so are the workers. +

          + + + + +
          + + +
          + Worker cuts cow hide into shape + +
          A thousand people work in "wood or trim"
          + +
          + + +
          + + + +They seem to be well aware that they are essential to Bentley's sales efforts. +

          +Whenever we halt to look more closely at their work, they stop to explain what they are doing. +

          +Prospective Bentley customers often visit the factory. +

          +They like to know what they are buying. +

          +And they like to make their mark. +

          +Steering wheels signed by the footballer David Beckham and the singer Cliff Richard are prominently displayed. +

          +"We enjoy celebrity endorsement of our products as much as anybody," says Richard Charlesworth, the man customers talk to if they want their Bentleys personalised. +

          +"We have customers asking us for things no marketing department could imagine," his boss John Killick, director of Bentley Mulliner, adds. +

          +Light wood +

          +"One customer even wanted to come and cut his own veneer," Mr Maddock says as he introduces us to his own team. +

          + + + + +
          + + +
          + Worker cuts wood laminate into shape + +
          One customer wanted to cut his own veneer
          + +
          + + +
          + + + +A group of craftsmen are busy meticulously forming exotic wood veneers into shining dashboards, door trimmings or gear sticks. +

          +Mr Maddock explains the difference between veneer made from Amboyna - Indonesian and very expensive - and Burr Walnut - made from the root of a Californian fruit tree. +

          +Or Madrona, Vavona, or Birdseye Maple. +

          +Most veneers are stored in a special moisture cupboard in order to make them easier to shape. +

          +Bentley buyers can choose any wood they like, though "some veneers are avoided if they are environmentally unsound, or if they don't work with our paint process", Mr Maddock explains. +

          +In recent years, many buyers have chosen light wood veneer which looks more contemporary than some of the darker woods, he observes. +

          +Building car bodies +

          +At last we get to see where the large steel body panels, which have been pressed by Mayflower, are put together. +

          + + + + +
          + + +
          + Worker makes sure the body is perfect + +
          Bentley customers want perfect bodies
          + +
          + + +
          + + + +In the past, some Bentleys were built using a mixture of steel and aluminium panels, but this was "bad for the paint process", Mr Maddock says. +

          +So these days, no aluminium is used. +

          +There is only one presser in the whole factory, the one that puts together the body skin and the inner frame. +

          +Each of the shiny silver bodies is carefully inspected, and any faults are scribbled directly onto the panels with thick felt tip pens, before the car is sent back for further work. +

          +German parts +

          +On entering the engine room, we are met by deadly silence. +

          +The workers are on their tea break, explains Mr Maddock. +

          + + + + + + + + + + +
          +
          + + + + + + + +
          + + + If we can't break the parts here, they will survive when the customers get to the
          + + + + +

          + + + + + + + + + + +
          + John Minshull, Bentley lab manager
          + + +
          + + + + + + + + +
          +
          + + + + + + +Rows of huge engines which are about to fill the space under the Bentleys' bonnets are lined up. +

          +"The engine components come from Germany," says Dough Dickson, board member responsible for manufacturing. +

          +"But we build the engines here." +

          +Bentley's German parent Volkswagen has pushed through some other changes too, as part of a modest modernisation process. +

          +This time the changes can be seen in the paint shop. +

          +"We used to spray by hand, we now spray by robot," Mr Maddock says. +

          +And finally, the cars glide down the assembly line where seats and steering wheels, stereos and trim are fitted. +

          +Breaking Bentleys +

          +VW has also earmarked large chunks of its investment for quality and logistics improvements. +

          +The quality testing is done in the laboratory where a selection of curious machines are stretching seat leather, rubbing floor carpets and exposing car bodies to hours of hot, salt mist. +

          +Insisting that his measuring machines are so accurate they can "measure the weight of a finger print", lab manager John Minshull is adamant. +

          +"If we can't break the parts here, they will survive when the customers get to them." +

          +Beyond using such wrecking machines, Bentley's lab technicians also resort to more instinctive measuring methods. +

          +Individual parts are "smell-tested to make sure they smell like a Bentley should", Mr Minshull says. + + + +

          +
          + + + + +
           + + + + +
          See also:

          + +
          + 27 Sep 02 | Business + + +
          + +
          + 16 Sep 02 | England + + +
          + +
          + 07 Mar 02 | Business + + +
          + +
          + 06 Mar 02 | Business + + +
          + +
          + 07 Jan 02 | England + + +
          + +
          + 20 Dec 01 | England + + +
          + +
          + 23 Nov 01 | Business + + +
          + +
          + 19 Oct 01 | Business + + +
          + +
          + + + +
          +
          Internet links:

          + +
          The BBC is not responsible for the content of external internet sites

          +
          + + + + + + +
          Links to more Business stories are at the foot of the page.

          +
          + + + + + + +
          +
          + + + + + + + + + +
          E-mail this story to a friend
          + + + + + + + + + + +
          + +
          + + +Links to more Business stories +
          +
          + + +
          +
          + +
          + + + + +
          + + + + + + + + + + + + + + +
          © BBC + +^^ +Back to top +

          + + + + + + + + +
          +News Front Page + | + + +Africa + | + + +Americas + | + + +Asia-Pacific + | + + +Europe + | + + +Middle East + | +
          + +South Asia + | + + +UK + | + + +Business + | + + +Entertainment + | + + +Science/Nature + | +
          + +Technology + | + + +Health + | + + +Talking Point + | + +Country Profiles + + | + +In Depth + + | +
          + +Programmes + + +
          + + +

          + + + + + + +
          + + + + + + + + + + + + + + + + + +
          + + + + diff --git a/test/testdata/f5ed3a88709359f26ae3b0cfe746990b21c678b4.json b/test/testdata/f5ed3a88709359f26ae3b0cfe746990b21c678b4.json new file mode 100644 index 00000000..b745e17d --- /dev/null +++ b/test/testdata/f5ed3a88709359f26ae3b0cfe746990b21c678b4.json @@ -0,0 +1,17 @@ +{ + "encoding": "ISO-8859-1", + "headers": { + "Cache-Control": "max-age=0", + "Connection": "Keep-Alive", + "Content-Type": "text/html", + "Date": "Tue, 23 May 2017 17:52:59 GMT", + "Expires": "Tue, 23 May 2017 17:52:59 GMT", + "Keep-Alive": "timeout=5, max=392", + "Server": "Apache", + "Set-Cookie": "BBC-UID=a589c22407964f4bf5fe4ea85194246863a07a8f705025d0e53608f6e094351b0Mozilla%2f5%2e0%20%28Windows%20NT%2010%2e0%3b%20Win64%3b%20x64%3b%20rv%3a50%2e0%29%20Gecko%2f20100101%20Firefox%2f50%2e0; expires=Wed, 23-May-18 17:52:59 GMT; path=/; domain=bbc.co.uk;, BBC-UID=a589c22407964f4bf5fe4ea85194246863a07a8f705025d0e53608f6e094351b0Mozilla%2f5%2e0%20%28Windows%20NT%2010%2e0%3b%20Win64%3b%20x64%3b%20rv%3a50%2e0%29%20Gecko%2f20100101%20Firefox%2f50%2e0; expires=Wed, 23-May-18 17:52:59 GMT; path=/; domain=bbc.co.uk;", + "Transfer-Encoding": "chunked", + "Vary": "X-CDN" + }, + "status_code": 200, + "url": "http://news.bbc.co.uk/2/hi/business/2570109.stm" +} \ No newline at end of file diff --git a/test/testdata/f606d5650c9e4e4654f78cdb262e1234af7b02c7.html b/test/testdata/f606d5650c9e4e4654f78cdb262e1234af7b02c7.html new file mode 100644 index 00000000..5b9e9129 --- /dev/null +++ b/test/testdata/f606d5650c9e4e4654f78cdb262e1234af7b02c7.html @@ -0,0 +1,5179 @@ + + + + + + + The New York Times - Breaking News, World News & Multimedia + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + +
          + + + + + +
          +
          + + + + +
          +
          + + + + + + + + + +
          +
          + +
          + +
          + +
          +

          Top News

          + +
          + +
          +
          +
          Developing
          +
          +
          + +
          + + + +
          +
          + +
          +
          + +
          +
          + +

          Assailant Died in Blast That Killed 22 in Manchester

          + + + +

          • The suspect was identified as Salman Abedi, a Briton of Libyan descent who lived near the arena. The police said his ID was found at the scene.
          • +
          • The British government did not make any immediate comment on the Islamic State’s claim of responsibility.

          + +

          +  Comments +

          + +
          + + +
          + + + +
          +
          + +
          +
          + +
          + +
          +
          + +
          See the aftermath of the bombing, in photographs. + +
          + +
          +
          + +
          +
          + + +
          +
          + +

          On the Scene in Manchester

          + + + +

          The Times has reporters in Manchester, where witnesses described the carnage, and the police said they had carried out raids.

          + + +
          +
          +
          + +
          +
          +
          +
          +
          +
          + +

          + +
          +
          + +
          +
          + + +

          +

          + + +
          +
          + +
          +
          +
          + + + +
          + + + + + + + + + +
          + +
          +
          +
          +
          + + +
          + +
          + +
          + +
          + +
          +
          + +

          Russia Contacts With Trump Team Worried Ex-C.I.A. Chief

          + + + +

          • John O. Brennan told senators on Tuesday that he became concerned last year that the Russian government was trying to influence members of the Trump campaign.
          • +
          • It is the first time the former C.I.A. director has publicly acknowledged that he was concerned about possible ties between Russia and Trump associates.

          + + +
          + + +
          +
          +
          + +

          + + +

          +

          + + + +
          +
          + +
          +
          +
          +
          +
          +

          Related Article

          + +
          +
          +
          + +

          Budget Slashes Aid to Poor and Offers Huge Tax Cuts

          + + + +

          President Trump’s budget proposal calls for spending more than $2.6 billion for border security — including $1.6 billion to begin work on a border wall — and slashing more than $800 billion from Medicaid.

          + +

          +  Comments +

          + +
          +
          + +
          + +
          + +
          +
          + + + + +
          +
          +
          +

          Got a confidential news tip?

          +

          The New York Times offers several ways to get in touch with and provide materials to our journalists. Learn more.

          +
          +
          +
          + + +
          + +
          + +
          + +
          + + + +
          + +
          + + + + +
          +
          + + The Daily Logo + +
          +

          Audio

          +

          + + Listen to ‘The Daily’ + +

          +

          The latest from President Trump’s trip abroad; the continuing saga of Michael Flynn; and developments after the bombing in Manchester, England.

          +
          + Audio +
          +
          +
          +
          + +
          +
          + + +
          + + + +
          +
          + + + +
          +
          +
          +
          + +

          Circa Now

          +

          The Tricky Etiquette of Co-Working Spaces

          + +
          + +
          + + + +

          + To get a glimpse of what manners will be like in the office of the future, it behooves us to look at the co-working spaces of today.

          + + +
          +
          + +

          Tech Tip

          +

          How to Expand Wi-Fi in Your Home

          + +
          + +
          + + + +

          + If parts of your house are not getting a decent Wi-Fi signal from your router, hardware, software and maybe an empty beer can might help.

          + + +
          +
          + + + + + + +
          +
          +
          + + +
          +
          + +
          +
          + A Los Angeles interchange. California can write its own auto emissions standards because of a waiver granted under the Clean Air Act. + + + Credit + Melissa Lyttle for The New York Times +
          +
          + +

          Fighting Trump on Climate, California Becomes a Global Force

          + +

          The state has been at the leading edge of the resistance to President Trump. But of all the battles, none has the global implications of climate change.

          + + + + + +
          +
          +
          +
          + +

          Sir Roger Moore, Who Played a Wry James Bond, Dies

          + +
          +
          + +
          +
          + + + +

          + The British actor brought tongue-in-cheek humor to the James Bond persona in seven films. His family announced his death in a statement on Twitter. He was 89.

          + + +
          +
          +
          +
          + +

          Jared Kushner’s Other Real Estate Empire

          + +
          +
          + +
          +
          + + + +

          + Baltimore-area renters complain about a property owner they say is neglectful and litigious. Few know their landlord is the president’s son-in-law.

          + +

          +  Comments +

          + +
          +
          +
          +
          + +

          Firebrand Sheriff, Voted Out of Office, Has No Regrets

          + +
          + +
          + + + +

          + Joe Arpaio, the former Arizona sheriff known for being tough on inmates and accused of targeting Latinos, reflects on his reputation and his decades in law enforcement.

          + + +
          +
          +
          + + + +
          + + +
          +
          +
          + + + + + +
          +
          +
          + +
          +
          +

          Morning Briefing: Australia Edition

          +

          The news and stories that matter to readers in Australia. Sign up to get it by email, Monday through Friday.

          + +
          +
          +
          + + + +
          + + + + +
          +
          +
          + +
          +
          +

          Morning Briefing: Asia Edition

          +

          The news and stories that matter to readers in Asia. Sign up to get it by email, Monday through Friday.

          + +
          +
          +
          + + + + +
          + + + + + +
          +
          +
          + +
          +
          +

          Morning Briefing: Europe Edition

          +

          The news and stories that matter to readers in Europe. Sign up to get it by email, Monday through Friday.

          +
          +
          +
          + + + +
          + + + + +
          +
          +
          + +
          +
          +

          Morning Briefing

          +

          The news and stories that matter. Delivered to your inbox Monday through Friday.

          + + +
          +
          +
          + + +
          + +
          + +
          + + +
          + +
          + +
          +
          + + + + + +
          +
          +
          + +
          +

          +

          +
          + +
          +
          + +
          +
          + +
          +
          +
          +
          + +
          +
          +
          +

          +

          +
          + +
          +
          + +
          +
          + +
          +
          +
          + +
          + +
          + +
          + + + +
          + + +
          + +
          + +
          + +
          + +
          + +
          +
          +
          +
          + +

          Manchester, United in Grief and Kindness

          + +
          + +
          + + + +

          + This was an attack on the city’s very soul. But terror’s spite only redoubles people’s decency.

          + + +
          + +
          + +
          +
          + +
          +
          +
          +
          + +

          Beware of Sheriff David Clarke

          + +
          + +
          + + +

          + We in Milwaukee are relieved to be rid of him, but worried about the damage he could do in the Trump administration.

          + + +
          + + + +
          + +
          +
          + +
          + +
          +
          +
          +
          + +
          + +
          + +

          User Subscriptions

          + + + + + +
          + +
          + +
          + + + +
          + +
          + + + + +
          +
          +
          +

          Watching

          +
          +
          +
          +
          +
          + + + + +
          + +
          + +
          + + +
          + +
          + + +
          + +
          + +
          + + + + +
          +
          +
          +
          Loading...
          +
          + +
          +
          + +
          + +
          + + + +
          + +
          + +
          + +
          +
          +

          Sections

          + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          +
          + +
          + +
          + + +
          +
          +
          +
          + +
          +

          + Real Estate » +

          + + +
          + + +
          +
          +
          +
          +
          +
          +
          +
          + +
          +
          +
          +
          + + +
          + + +
          +
          +
          +
          +
          +
          + + + +
          +
          Loading...
          +
          +
          + + + + +
          + +
          + + + + +
          +
          +
          +
          +

          Go to Home Page »

          +

          + Site Index + + The New York Times + +

          + +
          + + + +
          + + +
          +
          + + + + + + + + + + + + + + + diff --git a/test/testdata/f606d5650c9e4e4654f78cdb262e1234af7b02c7.json b/test/testdata/f606d5650c9e4e4654f78cdb262e1234af7b02c7.json new file mode 100644 index 00000000..ed5e9195 --- /dev/null +++ b/test/testdata/f606d5650c9e4e4654f78cdb262e1234af7b02c7.json @@ -0,0 +1,30 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "154", + "Cache-Control": "no-cache", + "Connection": "close", + "Content-Encoding": "gzip", + "Content-Length": "51336", + "Content-Security-Policy": "default-src data: 'unsafe-inline' 'unsafe-eval' https:; script-src data: 'unsafe-inline' 'unsafe-eval' https: blob:; style-src data: 'unsafe-inline' https:; img-src data: https: blob:; font-src data: https:; connect-src https: wss:; media-src https: blob:; object-src https:; child-src https: data: blob:; form-action https:; block-all-mixed-content;", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:53:27 GMT", + "Server": "Apache", + "Set-Cookie": "nyt-a=f28de7310cb8bf99b334f1eb67dfd5597cebc8be4de4ae4bbfa60ad4b1011811; Expires=Wed, 23 May 2018 17:53:27 GMT; Path=/; Domain=.nytimes.com", + "Vary": "Host, Accept-Encoding, Fastly-SSL", + "X-API-Version": "F-5-5", + "X-Age": "3", + "X-Cache": "HIT", + "X-Cache-Hits": "42", + "X-ESI": "1", + "X-Frame-Options": "DENY", + "X-Origin-Time": "2017-05-23 13:50:54 EDT", + "X-PageType": "homepage", + "X-Served-By": "cache-iad2626-IAD", + "X-Timer": "S1495562008.671043,VS0,VE0", + "ntCoent-Length": "235547" + }, + "status_code": 200, + "url": "https://www.nytimes.com/" +} \ No newline at end of file diff --git a/test/testdata/f6505d9c6b6ce61615b41e12d0ec2dc44ffb7e36.html b/test/testdata/f6505d9c6b6ce61615b41e12d0ec2dc44ffb7e36.html new file mode 100644 index 00000000..753f53a4 --- /dev/null +++ b/test/testdata/f6505d9c6b6ce61615b41e12d0ec2dc44ffb7e36.html @@ -0,0 +1,678 @@ + + + + + + تاریخ-و-تمدن-مغرب | سمت | خانه کتاب و ادبیات ایران + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + + + + + +
          +
          +
          +
          + + + + + + + +
          + + + + + +
          + + + ورود + + ثبت نام + + + + + + +
          +
          +
          +
          + + +
          + + + + +
          +
          +
          +
          +
          + تاریخ و تمدن مغرب | خانه کتاب و ادبیات ایران +
          +
          +
          + صفحات اولیه کتاب +

          + تاریخ و تمدن مغرب

          +

          + + + مراکش - تاریخ + + +

          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          پدیدآور + + نويسنده : + + مونس ، حسین + - + + + مترجم : + + شیخی ، حمیدرضا + + +
          ناشر + + + + سمت + + (برای تماس با ناشر و خرید کتاب کلیک کنید) + + + + +
          شابک978-964-530-036-2
          تاریخ نشر + +13901116 +
          قیمت +63,000
          کد دیویی961
          زبان کتابفارسی
          محل نشرمشهد - خراسان رضوی
          توضیحات + جلد 1 - + 464 صفحه - + ترجمه - + چاپ 2 +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          معرفی مختصر کتاب
          +

          + کتاب حاضر شرح وقایع سرزمین مغرب (مراکش) است که در آن تحولات سیاسی و اجتماعی و مدنی آن دیار از اندکی پیش از فتح اسلامی تا برپایی دولت "اشراف علوی‌فلالی" بازگو شده است. جلد نخست این مجموعه با گفتاری درباره‌ی عصرهای تاریخ مغرب اسلامی آغاز می‌گردد؛ سپس مباحث کتاب در قالب سه عصر پی گرفته می‌شود که عبارت‌اند از: عصر فتوحات (فتح مغرب به دست مسلمانان)، عصر والیان (پس از پایان فتح به دست موسی بن نصیر، عباسیان و کوشش‌های آنان برای حفظ افریقیه، مهلبیان در افریقیه، و سازمان اداری و مالی افریقیه و مغرب در عصر والیان) و عصر نخستین دولت‌های مغربی اسلامی (دولت اغلبیان در افریقیه، دولت اباضی مذهب رستمیان در تاهرت و شرق مغرب میانه، و حکومت صفری مذهب مدراریان در اقلیم تا فللت و سجلماسه، نقاط قوت و ضعف امامت‌های خوارج در مغرب، دولت ادریسیان، و دیگر دولت‌های کوچک مغربی). +

          +
          +
          +
          +
          +
          + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/testdata/f6505d9c6b6ce61615b41e12d0ec2dc44ffb7e36.json b/test/testdata/f6505d9c6b6ce61615b41e12d0ec2dc44ffb7e36.json new file mode 100644 index 00000000..4817c7d8 --- /dev/null +++ b/test/testdata/f6505d9c6b6ce61615b41e12d0ec2dc44ffb7e36.json @@ -0,0 +1,22 @@ +{ + "encoding": "utf-8", + "headers": { + "AR-ATIME": "0.101", + "AR-CACHE": "BYPASS", + "AR-PoweredBy": "Arvan Cloud (arvancloud.com)", + "AR-Request-ID": "ade54f456ff4c949683229a0348b713a", + "AR-SID": "2003", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Security-Policy": "upgrade-insecure-requests", + "Content-Type": "text/html; charset=utf-8", + "Date": "Sat, 27 Aug 2022 23:24:20 GMT", + "Keep-Alive": "timeout=65", + "Server": "ArvanCloud", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding, Accept-Encoding", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://ketab.ir/book/cb1989dc-ba09-4df6-aaee-fcdbd25ad322" +} \ No newline at end of file diff --git a/test/testdata/f6c946908de8408f5cf711ada37baa8b34383f45.html b/test/testdata/f6c946908de8408f5cf711ada37baa8b34383f45.html new file mode 100644 index 00000000..ec792bbe --- /dev/null +++ b/test/testdata/f6c946908de8408f5cf711ada37baa8b34383f45.html @@ -0,0 +1,1203 @@ + + + + + + + Research - Articles - Journals | Research better, faster at HighBeam Research + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + + + + + + + + + + + + + + +
          + +
          + + + + + + +
          + + + + + + diff --git a/test/testdata/f6c946908de8408f5cf711ada37baa8b34383f45.json b/test/testdata/f6c946908de8408f5cf711ada37baa8b34383f45.json new file mode 100644 index 00000000..f5e12954 --- /dev/null +++ b/test/testdata/f6c946908de8408f5cf711ada37baa8b34383f45.json @@ -0,0 +1,18 @@ +{ + "encoding": "utf-8", + "headers": { + "Cache-Control": "private", + "Content-Encoding": "gzip", + "Content-Length": "12710", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:54:03 GMT", + "Server": "Microsoft-IIS/8.5", + "Vary": "Accept-Encoding", + "X-AspNet-Version": "4.0.30319", + "X-AspNetMvc-Version": "4.0", + "X-FRAME-OPTIONS": "SAMEORIGIN", + "X-Powered-By": "ASP.NET" + }, + "status_code": 200, + "url": "https://www.highbeam.com/" +} \ No newline at end of file diff --git a/test/testdata/f8edf4951828ba1ea8541bc4ff29698eea259ec7.html b/test/testdata/f8edf4951828ba1ea8541bc4ff29698eea259ec7.html new file mode 100644 index 00000000..5797cb65 --- /dev/null +++ b/test/testdata/f8edf4951828ba1ea8541bc4ff29698eea259ec7.html @@ -0,0 +1,1984 @@ + + + + + + + + + + +Beyond Obama's Plan: A New Economic Vision for Addressing Climate Change | HuffPost + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + + + + +
          + + +
          +
          +
          +
          +
          + +
          +
          + +
          + + +
          +
          +
          + +
          + + +
          + + + + + + +
          + + + + +
          + + +
          + + + +THE BLOG + + +
          +06/02/2014 10:38 am ET +| + +Updated +Aug 02, 2014 + +
          + + + + + + +
          +

          Beyond Obama's Plan: A New Economic Vision for Addressing Climate Change

          +
          + + + + + + +
          +
          +
          + + + + + + +
          +

          The White House today released its national climate plan for reducing CO2 emissions, warning that climate change is adversely affecting every region of the United States, with dire consequences for the economy. Unfortunately, the new initiatives by the US government to ward off rising temperatures are weak at best. What's sorely missing from the climate change debate is a new economic vision that can quickly transition the US and global economy out of carbon based energy and into renewable energies, while simultaneously increasing productivity and reducing the amount of the earth's resources used in the economic process, ensuring a more prosperous and sustainable society. That vision is now taking hold.

          +

          A powerful new technology revolution is evolving that will allow enterprises and prosumers to make and share their own green electricity, and an increasing array of sustainable physical products and services, at near zero marginal cost, just as billions of prosumers now do with information goods. (Marginal cost is the cost of producing an additional unit of a good or service after the fixed costs have been absorbed). The Communication Internet is converging with a fledgling Energy Internet and nascent automated Transport and Logistics Internet, creating a new technological infrastructure for society--a Third Industrial Revolution--that could fundamentally alter the global economy and usher in an ecological civilization in the first half of the 21st century. Billions of sensors are being attached to resource flows, warehouses, road systems, factory production lines, the electricity transmission grid, offices, homes, stores, and vehicles, continually monitoring their status and performance and feeding big data back to the Internet of Things. By 2030, it is estimated there will be more than 100 trillion sensors connecting the human and natural environment in a global distributed intelligent network.

          +

          Enterprises and prosumers will be able to connect to the Internet of Things (IoT) and use Big Data and analytics to develop predictive algorithms that can speed efficiency, increase productivity, reduce the use of natural resources, and lower the marginal cost of producing renewable energy and manufactured products to near zero. They will be able to share what they've made with others on an emerging Collaborative Commons that is beginning to flourish alongside the conventional capitalist marketplace.

          +

          Zero Marginal Cost Renewable Energy

          +

          For example, the bulk of the energy we use to heat our homes and run our appliances, power our businesses, drive our vehicles, and operate every part of the global economy will be generated at near zero marginal cost and be nearly free in the coming decades. That's already the case for several million early adopters who have transformed their homes and businesses into micro-power plants to harvest renewable energy on-site. Even before the fixed costs for the installation of solar and wind are paid back--often as little as 2 to 8 years--the marginal cost of the harvested energy is nearly free. Unlike fossil fuels and uranium for nuclear power, in which the commodity itself always costs something, the sun collected on rooftops and the wind travelling up the side of buildings are nearly free. The Internet of Things will enable prosumers to monitor their electricity usage in their buildings, optimize their energy efficiency, and share surplus green electricity with others on the Energy Internet.

          +

          The same exponential curves that drove the marginal cost of generating and distributing communication to near zero has touched off a similar revolution in the field of renewable energy. Richard Swanson, the founder of SunPower Corporation, observed the same doubling phenomena in solar that IT companies observed in computer chips. Swanson's law holds that the price of solar photovoltaic (PV) cells tends to drop by 20 percent for every doubling of industry capacity. Crystalline silicon photovoltaic cell prices have fallen dramatically, from $60 a watt in 1976 to $0.66 a watt in 2013.

          +

          Solar cells are capturing more solar energy that strikes them while reducing the cost of harvesting the energy. Solar efficiencies for triple junction solar cells in the laboratory have reached 41 percent. Thin film has hit 20 percent efficiency in the laboratory. If this trend continues at the current pace--and most studies actually show an acceleration in exponentiality--solar energy will be as cheap as the current average retail price of electricity today by 2020 and half the price of coal electricity today by 2030.

          +

          The impact on society of near zero marginal cost solar energy is all the more pronounced when we consider the vast potential of these energy sources. The sun beams 470 exajoules of energy to Earth every 88 minutes--equaling the amount of energy human beings use in a year. If we could grab hold of one-tenth of 1 percent of the sun's energy that reaches Earth, it would give us six times the energy we now use across the global economy.

          +

          Like solar radiation, wind is ubiquitous and blows everywhere in the world--although its strength and frequency varies. A Stanford University study on global wind capacity concluded that if 20 percent of the world's available wind was harvested, it would generate seven times more electricity than we currently use to run the entire global economy. Wind capacity has been growing exponentially since the early 1990s and has already reached parity with conventionally generated electricity from fossil fuels and nuclear power in many regions of the world. In the past quarter century, wind-turbine productivity increased 100-fold and the average capacity per turbine grew by more than 1,000 percent. Increased performance and productivity has significantly reduced the cost of production, installation, and maintenance, leading to a growth rate of more than 30 percent per year between 1998 and 2007, or a doubling of capacity every two and a half years. Industry analysts forecast that the harvesting technology for solar and small wind power will be as cheap as cell phones and laptops within fifteen years.

          +

          Local, regional, and national governments around the world have instituted feed-in tariffs in the past few years, guaranteeing a premium price for renewable energy above the market value of other energies for a set period of usually 15 to 20 years to encourage early adopters to invest in the installation of wind, solar, geothermal, biomass, and small hydro renewable energy generation and feed the new green electricity back to the transmission grid. Today, millions of business and homeowners in Europe are taking advantage of feed in tariffs and investing their own capital to install renewable energy harvesting technologies on site. While the up-front capital investment is significant, they are beginning to receive low-interest-rate green loans from banks and credit unions. The banks are more than willing to lend money at reduced interest rates because the premium price of the green energy being produced virtually ensures the loan will be honored.

          +

          Sixty-five countries have instituted feed-in tariffs, and over half of them are in the developing world. Feed-in tariffs have proven to be a powerful policy instrument in moving renewable energy online. Nearly two-thirds of the global wind and 87 percent of global photovoltaic capacity has been spurred by feed-in tariffs. Unfortunately, in the United States, only California, Vermont, Maine, Oregon, Washington, Hawaii, and Rhode Island have implemented even cursory feed-in tariffs.

          +

          Naysayers argue that subsidies for green energy, in the form of feed-in tariffs, are too costly for society. The reality is that they merely speed up adoption and scale, encourage competition, and spur innovation, which further increases the efficiency of renewable energy harvesting technologies and lowers the cost of production and installation. In country after country, solar and wind energy is nearing parity or at parity with conventional fossil-fuel and nuclear power, allowing the government to begin phasing out tariffs. Meanwhile, the older fossil-fuel energies and nuclear power, although mature and well past their prime, continue to be subsidized at levels that far exceed the subsidies extended to renewable energy. Instituting robust feed in tariffs in all 50 states is a much more effective commercial incentive than carbon trading schemes to quickly usher in a post-carbon society.

          +

          Already, 27 percent of the electricity in Germany is being generated by renewable energy - mostly solar and wind--at near zero marginal cost and the percentage of green electricity is expected to exceed 35% by 2020. On Sunday, May 11th 2014, 75% of Germany's electricity demand was generated by renewable energy, a milestone for the world's most robust industrial economy per capita. So much near zero marginal cost electricity was being fed into the nation's power grid that electricity prices plunged into the negative category for much of the day. While the cost of subsidizing the new renewable energies places a relatively small short term burden on businesses and homeowners, in the mid- to long-term, Germany and other countries will enjoy near zero marginal cost energy and a dramatic increase in efficiency and productivity across the economy, resulting in sustainable economic growth far into the future.

          +

          It is particularly interesting to note that in Germany, which is setting the pace for transitioning into green electricity in Europe, the big traditional power and utility companies--E.ON, RWE, EnBW, Vattenfall Europe--owned only 7 percent of the renewable-energy capacity installed by the end of 2011. Individuals, however, "owned 40 percent of the renewable energy capacity, energy niche players 14 percent, farmers 11 percent, various energy-intensive industrial companies 9 percent, and financial companies 11 percent. Small regional utilities and international utilities owned another 7 percent." Nearly half of the German wind turbines are owned by residents of the regions. In other EU countries, the pattern is the same. Consumers are becoming prosumers and generating their own green electricity.

          +

          Gérard Mestrallet, CEO of GDF Suez--the French gas utility--says that just ten years ago the European energy market was dominated almost exclusively by a handful of regional monopolies. "Those days are gone forever," says Mestrallet, now that "some consumers have become producers." Peter Terium, CEO of RWE, the German-based energy company, acknowledges the massive shift taking place in Europe from centralized to distributed power, and says that the bigger power and utility companies "have to adjust to the fact that, in the longer term, earning capacity in conventional electricity generation will be markedly below what we've seen in recent years."

          +

          Had anyone suggested ten years ago that the big power and utility companies of Europe would begin to crumble as millions of small, distributed, renewable-energy micropower players began to generate their own green electricity for the grid, it would have been dismissed as fantasy by the powers that be. Not now. "It is a real revolution," says Mestrallet.

          +

          Nor is Europe alone. In December of 2013, the Chinese government leapt ahead of other countries, announcing that it is dedicating an initial $82 billion to establish a Third Industrial Revolution distributed "Energy Internet" that will serve as the centerpiece of an Internet of Things technology platform and infrastructure. Under the plan, millions of people in neighborhoods and communities across the country, as well as hundreds of thousands of businesses, will be able to produce their own solar- and wind-generated green electricity locally at near zero marginal cost, and share it on a national Energy Internet.

          +

          The Energy Internet, embedded in an Internet of Things platform will change the way power is generated and distributed in society. Already, millions of homeowners, businesses, and neighborhood producer and consumer cooperatives are harvesting clean renewable energy at near zero marginal cost. In the coming era, hundreds of millions of people will produce their own green electricity and share it at near zero marginal cost with each other on an Energy Internet, just as we now generate and share information online. When Internet communications manages green energy, every human being on Earth becomes his or her own source of power, both literally and figuratively. Zero marginal cost energy is "power to the people."

          +

          The Democratization of Manufacturing

          +

          While millions of people are now producing and sharing their own green electricity on an emerging Energy Internet, hundreds of thousands of hobbyists and thousands of startup companies are already printing out their own manufactured products using free software, and cheap recycled plastic, paper, and other locally available feedstock at near zero marginal cost. The additive manufacturing process, powered by electricity generated from renewable energy, uses one tenth of the materials of traditional factory production, resulting in a dramatic reduction in CO2 emissions and the use of the earth's resources. By 2020, prosumers will be able to share their 3D printed products with others on the Collaborative Commons by transporting them in driverless electric and fuel cell vehicles, powered by near zero marginal cost renewable energy, facilitated by an automated Logistics and Transport Internet.

          +

          China is setting the pace in the development of 3D printing. Beihang University is using 3D printing to manufacture sophisticated parts used in rockets and satellites. WinSun, another Chinese company, built ten small houses in less than 24 hours in 2014, using cheap recycled materials. The construction of the houses required very little human labor, and cost less than $5000 a piece to construct, making possible the production of millions of cheap homes at low or near zero marginal cost in China and other developing countries. Tiertime, China's largest producer of desktop 3D printers for use in small businesses and households, unveiled its newest model UP! in 2014. The company is competing head to head with America's leading producers of 3D printers, in the hopes of capturing much of the global market in the years ahead.

          +

          While Great Britain sparked the First Industrial Revolution, and the United States led the world into the Second Industrial Revolution, China has set its sights on leading the world into the Third Industrial Revolution by being the first superpower to build out an Internet of Things infrastructure and accompanying Collaborative Commons. In 2010, China seized the initiative over other countries, announcing its intention to erect an Internet of Things, focusing on the smart Energy Internet and an automated Logistics and Transport Internet, with the goal of meshing them with the Communication Internet to create the infrastructure for a Third Industrial Revolution. The Chinese government expects to invest $800 million on the initial build-out of the Internet of Things by 2015. The Chinese Ministry of Information and Technology forecasts that the IoT market will exceed $80 billion by 2015 and $166 billion by 2020.

          +

          The efficiency and productivity gains of the Third Industrial Revolution are likely to far outstrip those of the First and Second Industrial Revolutions. Several billion people and millions of organizations connected to the Internet of Things allows the human race to share their economic lives in a global Collaborative Commons, in ways previously unimaginable. This turning point in connectivity potentially exceeds even the integration of economic activity wrought by electrification and the accompanying spread of the telephone, radio and television in the 20th century. Cisco systems forecasts that by 2022, the Internet of Things will generate $14.4 trillion in cost efficiency savings and revenue. A General Electric study published in November 2012 concludes that the efficiency gains and productivity advances made possible by a smart industrial Internet could resound across virtually every economic sector by 2025, impacting "approximately one half of the global economy."

          +

          The Sharing Economy on the Collaborative Commons

          +

          Forty percent of the US population is already actively engaged in the sharing economy on the Collaborative Commons. 800,000 individuals in the US are now using car sharing services. In car sharing services, once the fixed costs are absorbed, the marginal cost of sharing the vehicle moves to near zero with each additional user.

          +

          Global transport currently accounts for fifteen percent of global warming emissions. Each car share vehicle eliminates 15 personally owned cars, resulting in a dramatic reduction in both CO2 emissions and the massive amount of material resources, energy, and labor that goes into manufacturing each automobile. In a recent study focused on the city of Ann Harbor, Michigan, Lawrence D. Burns, formerly the corporate vice president of research, development, and planning at General Motors, found that "about 80% fewer shared, coordinated vehicles would be needed than personally owned vehicles to provide the same level of mobility, with less investment." If we were to extrapolate Burns' study on a global scale, it is possible to envision car sharing services eliminating upwards of 800 million of the 1 billion privately owned cars now on the road, for a dramatic reduction in both CO2 emissions and the massive amount of material resources, energy, and labor that goes into manufacturing each automobile. If the remaining 200 million vehicles were powered by green electricity transmitted across the Energy Internet, carbon emissions in the transport sector would be reduced to near zero.

          +

          Buildings are another major contributor to climate change, accounting for approximately one third of global warming emissions. A significant percentage of these emissions come from hotels and resorts. (The travel and tourism sector is one of the largest industries in the world and represents nine percent of global GDP.) Now, millions of homeowners are sharing their apartments and houses with travelers via global online services like Airbnb and Couchsurfing, bypassing commercial hotels. For homeowners and apartment dwellers, whose fixed costs have already been absorbed, the marginal cost of opening up their homes to travelers is near zero. The big brick-and-mortar hotel chains, with their huge operating costs, simply can't compete with cheap short-term rentals or even free accommodations whose marginal costs of operation approach zero. In New York alone, Airbnb's 416,000 guests who stayed in apartments and houses between mid-2012 and mid-2013 cost the New York hotel industry 1 million lost room nights. As millions of homeowners open up their apartments and houses to travelers, we can expect a significant decline in the use of hotels and a corresponding decrease in CO2 emissions.

          +

          Millions of people are also redistributing their used clothing on the Collaborative Commons via online networks like ThredUP. The global textile industry is a major contributor to global warming, accounting for 10 percent of the total carbon impact. ThredUPs 385,000 visitors per month shared over 350,000 items in 2012, and orders are growing by a whopping 51% a month. More people sharing fewer clothes reduces the amount of new clothes purchased, resulting in fewer global warming gas emissions.

          +

          A younger generation is also sharing their tools, their children's toys, and countless other items on the Collaborative Commons. Freecycle, a redistribution network, gifted and passed along 700 million pounds of used items in the past year. If those items were stacked in garbage trucks, they would extend "the equivalent of over thirteen times the height of Mt. Everest."

          +

          In a zero marginal cost society, extreme productivity decreases the amount of information, energy, material resources, labor and logistics costs, necessary to produce and distribute economic goods and services, once fixed costs are absorbed. And the goods and services that are produced at near zero marginal cost are redistributed and shared over and over again on the Collaborative Commons, dramatically reducing the number of things sold, meaning fewer resources are used up and less global warming gases are emitted into the earth's atmosphere.

          +

          0 0 0

          +

          The nations of the world are far more likely to make commitments to CO2 reductions if pegged to the vast economic benefits that come from erecting an Internet of Things platform that can unleash extreme productivity, reduce the marginal cost of producing and distributing renewable energy, 3D printed goods, and services to near zero, and give rise to a sharing circular economy on the Collaborative Commons. If the Third Industrial Revolution becomes the centerpiece of the United Nations Climate Change Conference in December 2015 in Paris, rather than a sideshow, humanity might yet snatch victory from defeat, turn the corner on climate change, and restore the planet to health.

          +

          Jeremy Rifkin is the author The Zero Marginal Cost Society: The Internet of Things, the Collaborative Commons, and the Eclipse of Capitalism. Mr. Rifkin is a principal architect of the European Union's long-term Third Industrial Revolution economic development plan, and an advisor on sustainable development to heads of state around the world. He is the president of the Foundation on Economic Trends in Washington, DC.

          + + + + + + + + + +
          + + + + + + + + +
          +

          +
          + +
          +
          + + + + + + +
          +
          + + + + + + + + +
          +
          + +
          + +
          + + + + + + + + + + + + + + +
          + + + + + + + + + + + + +
          + +
          + + +
          + + + + + + +
          + + +
          +
          +
          +
          +
          + +
          + + +
          + + +
          + + + + + + + + + + + + + + +
          + + +
          + + + + + + + + + + + + + + + +
          +
          + + + + + + + + + + + + + + + + + +
          + + + + + + + + + +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + +
          + + diff --git a/test/testdata/f8edf4951828ba1ea8541bc4ff29698eea259ec7.json b/test/testdata/f8edf4951828ba1ea8541bc4ff29698eea259ec7.json new file mode 100644 index 00000000..3ebc8fb5 --- /dev/null +++ b/test/testdata/f8edf4951828ba1ea8541bc4ff29698eea259ec7.json @@ -0,0 +1,25 @@ +{ + "encoding": "utf-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "0", + "Cache-Control": "max-age=300, public, must_revalidate=false", + "Content-Encoding": "gzip", + "Content-Length": "53989", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:53:25 GMT", + "Server": "ECD (iad/B9C2)", + "Vary": "Accept-Encoding", + "X-Content-Type-Options": "nosniff", + "X-EC-Lua": "19365-geo", + "X-Frame-Options": "ALLOWALL", + "X-HP-Trace-ID": "kJfNhGD3", + "X-HP-Trace-Project": "HPMW/production/70604bb", + "X-Mobile-URL": "http://m.huffpost.com/us/entry/5427656", + "X-Request-Id": "1f2e7809-bdfa-4f9c-8df5-63879a4f206f", + "X-Runtime": "0.216315", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "http://www.huffingtonpost.com/jeremy-rifkin/obamas-climate-change-plan_b_5427656.html" +} \ No newline at end of file diff --git a/test/testdata/fa3202ef901ee686cee10edb0d5bc112c0f4694d.html b/test/testdata/fa3202ef901ee686cee10edb0d5bc112c0f4694d.html new file mode 100644 index 00000000..b8315e67 --- /dev/null +++ b/test/testdata/fa3202ef901ee686cee10edb0d5bc112c0f4694d.html @@ -0,0 +1,738 @@ + + + + + + تصویر کتاب الكامل في التاريخ - جلد 13 - صفحه 1 - ابن اثیر، علی بن محمد + + + + + + + + + + + + + + + + + + + + + + + + +
          +
          + + + + +
          +
          + + + +
          + + +
          +
          +
          + + + +
          +
          +
          + + +
          + +
          +
          + + + + + + + + + +
          + +
          +
          +
          +
          + + +
          +
          +
          +
          + + +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + + + + + + + + +
          +
          +
          + +
          + + + +
          +
          + +
          + +
          + +
          + +
          + + +
          +
          +
          + + + + diff --git a/test/testdata/fa3202ef901ee686cee10edb0d5bc112c0f4694d.json b/test/testdata/fa3202ef901ee686cee10edb0d5bc112c0f4694d.json new file mode 100644 index 00000000..20175b33 --- /dev/null +++ b/test/testdata/fa3202ef901ee686cee10edb0d5bc112c0f4694d.json @@ -0,0 +1,17 @@ +{ + "encoding": "utf-8", + "headers": { + "Cache-Control": "private", + "Content-Encoding": "gzip", + "Content-Length": "16102", + "Content-Type": "text/html; charset=utf-8", + "Date": "Fri, 13 Apr 2018 07:57:49 GMT", + "Server": "Microsoft-IIS/7.5", + "Set-Cookie": "ASP.NET_SessionId=euz1afeym3odnujslawghhn3; path=/; HttpOnly", + "Vary": "Accept-Encoding", + "X-AspNet-Version": "4.0.30319", + "X-Powered-By": "ASP.NET" + }, + "status_code": 200, + "url": "https://www.noorlib.ir/View/fa/Book/BookView/Image/3232" +} \ No newline at end of file diff --git a/test/testdata/fa945b41a6ad4b81ec0412898c48cb1d6eab4f4a.html b/test/testdata/fa945b41a6ad4b81ec0412898c48cb1d6eab4f4a.html new file mode 100644 index 00000000..e6f1e78c --- /dev/null +++ b/test/testdata/fa945b41a6ad4b81ec0412898c48cb1d6eab4f4a.html @@ -0,0 +1,526 @@ + + + + + + + + + + + + + + + + + + + + + + + + + +The evidence that shows Iron Dome is not working | Bulletin of the Atomic Scientists + + + + + + + + + + + + + + + + + + + + + + + +
          + +
          +
          Close
          +
          +
          +
          +
          +
          +

          Overview

          The Doomsday Clock is an internationally recognized design that conveys how close we are to destroying our civilization with dangerous technologies of our own making. First and foremost among these are nuclear weapons, but the dangers include climate-changing technologies, emerging... Read More

          +
          +
          +
          +
          +

          Press Release

          + +
          +
          +
          +
          +
          +
          +
          +
          +
          2017
          +
          +
          2016
          +
          +
          2015
          +
          +
          2012
          +
          +
          2010
          +
          +
          2007
          +
          +
          2002
          +
          +
          1998
          +
          +
          1995
          +
          +
          1991
          +
          +
          1990
          +
          +
          1988
          +
          +
          1984
          +
          +
          1981
          +
          +
          1980
          +
          +
          1974
          +
          +
          1972
          +
          +
          1969
          +
          +
          1968
          +
          +
          1963
          +
          +
          1960
          +
          +
          1953
          +
          +
          1949
          +
          +
          1947
          +
          +
          +
          +
          +
          +
          +
          + +
          +

          The Clock:
          A Brief History

          The Clock brief history
          +
          +
          +
          +
          + +
          +
          + +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +

          Figure 1_new.jpg

          +
          +Figure 1. An Iron Dome interceptor engages a rocket in the proper orientation. The blue dashed line emanating from the forward section of the interceptor depicts the line-of-sight of its laser fuse.
          +
          +
          +
          +
          +
          +
          +

          Figure_02-new.jpg

          +
          +Figure 2. Deciding when to explode: A conceptual diagram showing, via the blue arrow, the correct orientation if an Iron Dome interceptor warhead is to destroy a target rocket warhead.
          +
          +
          +
          +
          +
          +
          +

          Figure_03-new.jpg

          +
          +Figure 3. A slightly more detailed view of the outcome, if an Iron Dome interceptor works as intended, spraying fragments at high speed into a rocket warhead, causing it to explode.
          +
          +
          +
          +
          +
          +
          +

          figure-4-new.jpg

          +
          +Figure 4. A view of damage apparently caused by the detonation of the warhead of this rocket when it hit ground.
          +
          +
          +
          +
          +
          +
          +

          figure-4A-new.jpg

          +
          +Figure 4A. Holes in an empty rocket motor casing suggest that an Iron Dome interceptor warhead exploded too late to detonate the target rocket warhead in the air.
          +
          +
          +
          +
          +
          +
          +

          figure-5-new.jpg

          +
          +Figure 5. This vector diagram shows how a skewed frontal approach would tend to spread fragments from an Iron Dome interceptor warhead in directions unlikely to contact or explode a target rocket warhead. (Vector diagram speeds in feet per second.)
          +
          +
          +
          +
          +
          +
          +

          figure-6-new.jpg

          +
          +Figure 6. This vector diagram of an Iron Dome interceptor attacking a Grad rocket from the side shows how unlikely it would be for fragments from the interceptor warhead to hit the rocket warhead.
          +
          +
          +
          +
          +
          +
          +

          figure-7-new.jpg

          +
          +Figure 7. A vector diagram of a different sidelong approach, showing, again, that the spread of fragments from the Iron Dome interceptor would be unlikely to strike the warhead area of the rockets.
          +
          +
          +
          +
          +
          +
          +

          figure-8-new.jpg

          +
          +Figure 8. An Iron Dome interceptor attacking a rocket from behind would have a low probability of spraying fragements into the rocket warhead. (Vector diagram speeds in feet per second.)
          +
          +
          +
          +
          +
          +
          +

          figure-9-new.jpg

          +
          +Figure 9. A photo from November 2012 shows Iron Dome interceptor contrails that suggest ineffective sidelong or rear approaches to the target rocket.
          +
          +
          +
          +
          +
          +
          +

          figure-10-new.jpg

          +
          +Figure 10. Another 2012 photo suggesting ineffective, non-frontal attacks by Iron Dome interceptors.
          +
          +
          +
          +
          +
          +
          +

          figure-11-new.jpg

          +
          +Figure 11. More apparently ineffective Iron Dome attacks.
          +
          +
          +
          +
          +
          +
          +

          figure-12-new.jpg

          +
          +Figure 12. Two intercept attempts in July 2014 that suggest Iron Dome interceptors attacked in a sidelong orientation unlikely to destroy the target rockets.
          +
          +
          +
          +
          +
          +
          +

          figure-13-new.jpg

          +
          +Figure 13. A contrail photo that suggests another sidelong approach by an Iron Dome interceptor.
          +
          +
          +
          +
          +
          +
          +

          figure-14-new.jpg

          +
          +Figure 14. What an Iron Dome hit looks like in the sky.
          +
          +
          +
          +
          +
          +
          +

          figure-15-new.jpg

          +
          +Figure 15. Published warning times for artillery rockets of varying ranges attacking Israel from the Gaza Strip.
          +
          +
          +
          +
          +
          +
          +

          figure-16-new.jpg

          +
          +Figure 16. A screen shot of the red alert mobile phone app that issues an audible alert of an impending artillery rocket impact in Israel.
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          1 of 17
          +
          +
          +
          +
          +
          19 July 2014

          The evidence that shows Iron Dome is not working

          Theodore A. Postol
          +
          +
          +

          Theodore A. Postol

          A physicist, Postol is professor of science, technology, and national security policy at MIT. His expertise is in ballistic...

          More
          +
          +

          Editor's note: Images referenced in this article can be viewed in the slide show above; captions appear when a cursor is placed over the images. The images can also be seen in a separate slide show found here, or by clicking on the red button to the right of the story's third paragraph.

          In the early weeks of July, the conflict between Palestinians in Gaza and Israel flared up again, resulting in a new round of large-scale rocket attacks, launched by Hamas, operating from Gaza, against Israeli population centers. The last such large-scale rocket attacks occurred in November 2012. 

          Initially, the Israeli military responded to the rocket attacks with air strikes in Gaza, and with protective measures that include deployment of the Iron Dome rocket-defense system and a civil defense effort that includes an efficient system for early warning and sheltering of citizens. As of this writing, only one Israeli had died from Hamas fire, apparently from a mortar round (although that number increased with the Israeli invasion of the Gaza Strip begun late last week).

          During the November 2012 conflict, a detailed review of a large number of photographs of Iron Dome interceptor contrails revealed that the rocket-defense system's success rate was very low—as low as 5 percent or, perhaps, even less. A variety of media outlets have attributed the low casualty number to the supposed effectiveness of the Iron Dome system, quoting Israeli officials as saying it has destroyed 90 percent of the Hamas rockets it targeted. But close study of photographic and video imagery of Iron Dome engagements with Hamas rockets—both in the current conflict and in the 2012 hostilities—shows that the low casualties in Israel from artillery rocket attacks can be ascribed to Israeli civil defense efforts, rather than the performance of the Iron Dome missile defense system.

          The collection of data for Iron Dome's performance in July 2014 is still in progress. The data we have collected so far, however, indicates the performance of Iron Dome has not markedly improved.

          Historical data on civil defense measures—including those taken to protect citizens from V-1 and V-2 rocket bombings of London during World War II—suggest that Israel’s low casualty rate from Hamas rockets is largely attributable to the country's well-developed early-warning and quick-sheltering system for citizens under imminent rocket attack. That is to say, Iron Dome appears to have had no measurable effect on improving the chances of Israelis escaping injury or death from Hamas artillery rocket attacks in Israel.

          What performance characteristics make a rocket defense effective? To successfully intercept an artillery rocket of the type Hamas has been firing, an Iron Dome interceptor must destroy the warhead on the front end of the rocket. If the Iron Dome interceptor instead hits the back end of the target rocket, it will merely damage the expended rocket motor tube, basically an empty pipe, and have essentially no effect on the outcome of the engagement. The pieces of the rocket will still fall in the defended area; the warhead will almost certainly go on to the ground and explode. 

          Destroying an artillery rocket warhead is a considerably more demanding mission than damaging other parts of the targeted rocket—or, in the analagous situation of aircraft defense, successfully damaging an airplane, causing the failure of its mission.

          Analysis of photographs of contrails left by Iron Dome interceptor missiles can show whether or not an attempted rocket intercept could have been successful. Such analysis focuses on two connected facts: To have a realistic chance of destroying an artillery rocket's warhead, an Iron Dome interceptor must approach the rocket from the front—in fact, almost directly head-on. And for all practical purposes, an Iron Dome interceptor has no chance of destroying the warhead if the interceptor engages the rocket from the side or from the back.

          Photographs of Iron Dome contrails indicate that most of the system's interceptors have either been chasing Hamas rockets from behind or engaging those rockets from the side. In both such cases, geometry and the speed of the interceptors and rockets make it extremely unlikely the interceptor will destroy the rocket's warhead.    

          How an Iron Dome interceptor works. To understand why the Iron Dome interceptor must approach the artillery rocket from the front to be effective, it is necessary to understand the basics of how an Iron Dome interceptor is meant to function.

          Figure 1 illustrates a theoretical front-on engagement by an Iron Dome interceptor against a Grad artillery rocket, a weapon initially produced by the Soviet Union in the 1960s, subsequently manufactured by many other countries, and now readily available to Hamas. The blue dashed line emanating from the forward section of the Iron Dome interceptor depicts the line-of-sight of its “laser fuse,” which creates a beam of light that reflects off the front-end of a targeted artillery rocket. Via its control system, the interceptor can then determine when the target rocket is in the process of passing the interceptor. The warhead in the Iron Dome interceptor is placed well behind the fuse assembly, a distance of roughly 3 feet from the laser-fuse aperture. This arrangement gives the fuse enough time to determine where the front of the target-rocket is, to estimate how long it will take for the front of the artillery rocket to pass parallel to the artillery rocket’s warhead, and to detonate the Iron Dome warhead at the moment when it is in position to cause the rocket's warhead also to explode.

          The timing of this sequence of events is critical to performance. The Iron Dome interceptor must account not only for the location of the target-rocket’s warhead, but also for the high crossing speed of the Iron Dome interceptor and the artillery rocket; for any off-parallel orientation of the Iron Dome interceptor relative to the artillery rocket; for the distance between the interceptor and rocket when the interceptor's explosive warhead goes off; and for the speed of the shrapnel fragments shooting from the warhead.

          Figure 2 shows how the fragments from the Iron Dome warhead would move, under the assumption that the crossing speed of the Iron Dome interceptor and artillery rocket—that is, their speed relative to one another—is about 1,200 meters per second. The explosive in the Iron Dome warhead projects fragments at about 2,100 meters per second, perpendicular to the direction the interceptor is traveling. According to standard physics calculations (suggested by the red and yellow vector diagram at the lower right of the figure), the net direction of the cloud of fragments, as experienced by a theoretical observer sitting on the artillery rocket, is shown by the pale blue arrow passing through both the Iron Dome warhead and the artillery rocket’s warhead.

          Figure 3 provides a slightly more vivid and detailed view of the outcome, if an Iron Dome interceptor works as intended. There is, however, only a limited range of possible outcomes that provide a high likelihood of success. Beyond that range, the possibility of success diminishes drastically.

          The many ways that Iron Dome can miss. Because of the uncertainties in the exact crossing speed and geometry of two high-speed missiles, even a perfectly operating Iron Dome fuse may fail to place lethal fragments onto an artillery rocket’s warhead. In addition, unless the distance between the Iron Dome warhead and the warhead of an artillery rocket is small (roughly a meter or so), there will be a greatly diminished chance that a fragment from the Iron Dome warhead will hit, penetrate, and cause the detonation of the artillery-rocket warhead.

          So a front-on engagement does not guarantee that an Iron Dome interceptor will destroy the warhead on the artillery rocket. A front-on engagement geometry merely indicates that an Iron Dome interceptor has a greater-than-zero chance of destroying the target-artillery rocket warhead.

          The consequences of a failure in fuse timing—in what was almost certainly an engagement between an Iron Dome interceptor and the artillery rocket—are shown in Figure 4 and Figure 4A.  

          The photo in Figure 4A shows the magnified front-end of the rocket; holes can be seen in the expended and empty rocket motor casing immediately behind the warhead. In this case, it is nearly certain that the artillery rocket was engaged by an Iron Dome interceptor properly approaching the artillery rocket, front-on. Unfortunately, it seems the timing commands from the fuse resulted in fragments from the exploding Iron Dome warhead hitting the artillery rocket after the warhead had passed. The relatively low density of holes in the artillery rocket’s after-body suggests that the encounter also had a relatively high miss distance—possibly several meters. And as can be seen in Figure 4, there is significant damage in the area where the rocket fell—damage almost certainly caused by the detonation of the rocket's small warhead when it hit the ground. This photograph illustrates that even when the Iron Dome interceptor is in a proper front-on trajectory, it can still fail to destroy the warhead of a target-artillery rocket.

          Figures 5, 6, 7, and 8 are detailed diagrams that indicate how an Iron Dome interceptor would perform if it engaged an artillery rocket from directions other than head-on. They show why the kill rate for an Iron Dome interceptor will be very low when the interceptor does not attack its target almost directly head on.

          As Figure 5 shows, even a moderately skewed approach to the targeted rocket will result in a drastically reduced chance that fragments from an Iron Dome warhead could be sprayed onto the rocket's warhead. Such small but crucial off-frontal errors could result from faults in the master guidance and control system of the Iron Dome interceptor. 

          Figures 6, 7, and 8 show interceptor engagements that approach the targeted artillery rocket from the side or from the back. Careful inspection of the geometry of the fuse sensing beam and the spray pattern of the fragments from an Iron Dome warhead reveals two very serious problems with these kinds of engagements: First, even if the fuse detects the artillery rocket in these angles of approach, it has no way of determining where the warhead is on the rocket. Second, even if the fuse detonates the Iron Dome warhead, by chance, at a time when fragments might be sprayed in the direction of the rocket warhead, in almost all circumstances the result will be a very low density of fragments arriving at the artillery rocket warhead location. Given the small number of fragments that can be dispersed by the Iron Dome warhead, this translates into a very high chance that no fragment will hit the warhead. 

          Making a successful interception even more problematic, the projected target area of the rocket warhead is very small, viewed from the front or back, rather than from the side.  Also, when an Iron Dome interceptor approaches from these side and rear angles, fragments from its warhead are very likely to hit the metal surfaces of a target rocket at low grazing angles, with fragments tending to bounce off the shell of the rocket body or warhead casing. In sum, then, for engagement geometries that are not front-on, the probability that an Iron Dome interceptor will destroy the warhead of an engaged target-artillery rocket will be, for all practical purposes, nearly zero.

          Understanding Iron Dome contrails. If artillery rockets are fired at their maximum range, they can be expected to fall at angles of 60 to 65 degrees relative to horizontal in their descent to a target; they will fall at angles well above 65 degrees when fired at less than maximum range.

          The very steep descent of artillery rockets is important to keep in mind when attempting to visualize what is happening when viewing the photographs that show only the smoke contrails of Iron Dome interceptors attempting to engage artillery rockets. When Iron Dome interceptors explode in the sky, but have contrails showing they have crossed the expected rocket trajectory in a side-on geometry or chased the artillery rocket from behind, it can be said, with a high degree of certainty, that no intercept could have occurred—assuming of course, an artillery rocket was even being engaged.

          Figures 9, 10, and 11 are photographs taken during the artillery rocket attacks in November 2012. They show contrails in the sky that indicate Iron Dome interceptors were attempting to engage target-artillery rockets from behind or from the side. The geometries of the engagement are easily established; the artillery rockets are falling at high elevation angles relative to the ground, and the contrails show Iron Dome interceptors clearly approaching from above or sidelong to any reasonable estimate of a rocket's descent path.

          The photographs in Figures 12 and 13 show intercept attempts in July 2014 that are nearly side-on, and hence, have essentially a zero chance of destroying target rockets, if they are present. 

          Observations colleagues and I made in November 2012 found no more than 20 percent of Iron Dome contrails indicating an engagement geometry that was front-on to the targeted rocket. At that time we estimated the probability of destroying a SCUD warhead in a front-on engagement might be between 30 and 60 percent, meaning that if all other engagements affectively resulted in a zero probability of interception, then the overall intercept rate would be between 6 and 12 percent. Given that less than 20 percent of the engagements we were able to get data on were actually front-on, our best estimate was that the intercept performance of Iron Dome was likely 5 percent or less.

          Daytime visual photographs of Iron Dome debris clouds can show, in many cases, the evidence of a successful intercept, i.e., the destruction of the targeted artillery rocket warhead. Since the Israeli government has been claiming a very high intercept rate—near 90 percent—it should be expected that visual evidence of hits would be common. But we have found only one example of photographic evidence in which it is clear that such a head-on success occurred.

          Figure 14 shows photographic evidence of the destruction of a rocket warhead by an Iron Dome interceptor. In this photograph, the Iron Dome missile is clearly on a trajectory that engages the falling artillery rocket head-on. The large white arrows at the top and bottom of the photograph show the relative directions of the rising Iron Dome interceptor and the falling artillery rocket. An inspection of the debris cloud shows that it is asymmetrical—indicating that two explosions have occurred nearly simultaneously.

          This debris cloud formation is essentially the result of fragments from the Iron Dome warhead hitting the warhead of the artillery rocket and detonating it. The explosive process that led to this observable debris cloud took less than one half of a millisecond, or essentially instantaneously from the perspective of an observer or with regard to the frame rate of a standard video camera, which would take a picture roughly every 30 to 40 milliseconds.

          This photograph is the only successful engagement I have found during very extensive searches of voluminous photographic and video evidence of Iron Dome interceptor activity.

          It could be argued that the details that can be seen in this photograph are sufficiently subtle that they might not be observable in all engagements. This argument is probably correct. All the same, it seems extremely unlikely that the Iron Dome system would be intercepting 90 percent of the artillery rockets it engaged, but result in only one photo among hundreds as evidence of a successful intercept.

          It is absolutely clear: There is no evidence in the public record to show that Iron Dome is performing at an intercept rate of nearly 90 percent. 

          If Iron Dome doesn't work well, why are Israeli casualties from rocket attacks so low? Israel has a vast system of shelters, arranged so citizens can easily find protection within tens of seconds or less of warning. The Israeli rocket attack warning system is sophisticated; Figure 15 shows warning times published by the Israelis for artillery rockets of varying ranges. Figure 16 shows the screen of a mobile phone warning system that issues an audible alert of an impending artillery rocket impact. This particular phone application is called “red alert.” 

          The app's message indicates the general area where an artillery rocket impact is expected; depending on the location of individuals receiving the warning message, they know whether or not to take shelter.

          During the World War II bombing of London by Germany's V-1 and V-2 rockets, seconds of early warning resulted in reductions in casualties and deaths by a factor of two or more, even when the people under attack did nothing more than take expedient measures like falling to the ground before a rocket impact.

          In the World War II bombings in London, the warheads were much larger than those used by Hamas, carrying about 2,000 pounds of explosives; in the Gulf War of 1991, SCUD warheads were also much larger, about 500 pounds each. In the case of the recent artillery rocket attacks against Israel, the overwhelming number of artillery rocket warheads are in the 10- to 20-pound range. These small warhead sizes make early warning and protective sheltering even more effective, because the smaller warheads are very unlikely to penetrate or destroy a shelter.

          These two factors—the small size of the warheads, and the warning and sheltering system—go far toward explaining why there has been only one Israeli death from rocket and mortar attacks. The one Israeli death attributable to the current conflict as of the writing of this article occurred on July 15; a man was hit by shrapnel from an exploding mortar shell near the Israeli border with Gaza, and his death was clearly the result of two unfortunate circumstances: The man was not in a shelter, and he had no warning of the arriving mortar shell.

          Another example of the hazards of not taking shelter occurred in November 2012. Three people were out on a terrace; one of them was hoping to observe the Iron Dome system intercepting incoming artillery rockets. An artillery rocket hit the terrace, killing all three people. Had these people followed the simple procedure of taking shelter, they would be alive today.

          A need for Israeli transparency. I do not know precisely why Iron Dome interceptors are not engaging most artillery rockets using the proper front-on geometry. It is clear that the Iron Dome radar tracking and guidance system is not working as it should work; it is initially sending Iron Dome missiles to intercept points that then result in interceptors not being able to achieve the right engagement geometries when they start the process of homing on targeted artillery rockets. Photographs from November 2012 show such problems, and pictures from July of this year indicate that Iron Dome interceptors are still behaving erratically, resulting in continued low intercept rates.

          If Iron Dome is in fact working at the high levels of performance being claimed, there is systematic data that the Israeli government could present to document the success.

          The Israeli government publishes insurance claim data that occur during different time periods. This data would very clearly show a reduction in ground damage in the areas defended by Iron Dome. This could not be otherwise, given the large number of successful intercepts being claimed by the Israelis and the significant reduction in damage that would occur from destroying artillery rocket warheads that would otherwise explode on the ground, or in or near buildings.

          This is not to say that there would still not be significant insurance claims in areas successfully defended by Iron Dome. A successful intercept can at the very best destroy the explosive warhead carried by the artillery rocket. It cannot destroy the pieces of debris from the artillery rocket itself. This debris will fall whether or not an artillery rocket has been intercepted. Nonetheless, the major contributor to significant damage is the exploding warhead on the artillery rocket. The Israelis have not provided any evidence of a reduction in ground damage that would surely have to accompany the amazing success rates that they have claimed for Iron Dome. 

          In the absence of Israeli data backing claims of Iron Dome efficiency, and based on the unambiguous evidence I have reviewed, a conclusion seems clear: The Israeli government is not telling the truth about Iron Dome to its own population, or to the United States, which has provided the Israeli government with the bulk of the funding needed to design and build the much-heralded but apparently ineffective rocket-defense system.

          +
          +
          +
          + + + + + + +
          +
          + +
          +
          +
          +
          + +
          +
          + +
          + +
          + + + +
          + + + + + + + + + + diff --git a/test/testdata/fa945b41a6ad4b81ec0412898c48cb1d6eab4f4a.json b/test/testdata/fa945b41a6ad4b81ec0412898c48cb1d6eab4f4a.json new file mode 100644 index 00000000..ba357394 --- /dev/null +++ b/test/testdata/fa945b41a6ad4b81ec0412898c48cb1d6eab4f4a.json @@ -0,0 +1,30 @@ +{ + "encoding": "utf-8", + "headers": { + "Age": "0", + "CF-RAY": "3639e0bd145957c5-IAD", + "Cache-Control": "public, max-age=300", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Language": "en", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:54:00 GMT", + "Expires": "Sun, 19 Nov 1978 05:00:00 GMT", + "Last-Modified": "Tue, 23 May 2017 17:53:59 GMT", + "Link": "; rel=\"canonical\",; rel=\"shortlink\"", + "Server": "cloudflare-nginx", + "Set-Cookie": "__cfduid=d9e1a0f3c461a0bdc84ab7e812c7a7a9d1495562039; expires=Wed, 23-May-18 17:53:59 GMT; path=/; domain=.thebulletin.org; HttpOnly", + "Transfer-Encoding": "chunked", + "Vary": "Cookie,Accept-Encoding", + "Via": "1.1 varnish-v4", + "X-Content-Type-Options": "nosniff", + "X-Drupal-Cache": "MISS", + "X-Frame-Options": "SAMEORIGIN", + "X-Generator": "Drupal 7 (http://drupal.org)", + "X-UA-Compatible": "IE=edge", + "X-Varnish": "15930976", + "X-Varnish-Cache": "HIT" + }, + "status_code": 200, + "url": "http://thebulletin.org/evidence-shows-iron-dome-not-working7318" +} \ No newline at end of file diff --git a/test/testdata/fad6860c44e556d3af3ff57b43b426033e3e685d.html b/test/testdata/fad6860c44e556d3af3ff57b43b426033e3e685d.html new file mode 100644 index 00000000..07ebb891 --- /dev/null +++ b/test/testdata/fad6860c44e556d3af3ff57b43b426033e3e685d.html @@ -0,0 +1,9 @@ +TY - BOOK +T1 - Digital Libraries +A1 - Arms, W.Y. +SN - 9780262261340 +T3 - Digital Libraries and Electronic Publishing +UR - https://books.google.com/books?id=pzmt3pcBuGYC +Y1 - 2001 +PB - MIT Press +ER - diff --git a/test/testdata/fad6860c44e556d3af3ff57b43b426033e3e685d.json b/test/testdata/fad6860c44e556d3af3ff57b43b426033e3e685d.json new file mode 100644 index 00000000..f772fec7 --- /dev/null +++ b/test/testdata/fad6860c44e556d3af3ff57b43b426033e3e685d.json @@ -0,0 +1,21 @@ +{ + "encoding": null, + "headers": { + "Alt-Svc": "h3-29=\":443\"; ma=2592000,h3-27=\":443\"; ma=2592000,h3-25=\":443\"; ma=2592000,h3-T050=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"", + "Cache-Control": "private, max-age=0", + "Content-Disposition": "attachment; filename=Digital_Libraries.ris", + "Content-Length": "218", + "Content-Type": "application/x-research-info-systems", + "Date": "Sat, 11 Jul 2020 09:16:20 GMT", + "Expires": "Sat, 11 Jul 2020 09:16:20 GMT", + "P3P": "CP=\"This is not a P3P policy! See g.co/p3phelp for more info.\"", + "Server": "OFE/0.1", + "Set-Cookie": "NID=204=ysOqCE5rx0PsbAIuFkeT39dkk2wDv7b2RMY4zKeraSXJGEo8k4O5pXNewMnYxhou4FM80yasF9WzA4a602B9WlAE0k1Eyc0eXGC4ALIoNpEN7DThVyxTSWVVKA8jJdIugFQHrb9sWFNl1B5U_BSL3EbAhmw2hysWjXzFC5n-uOw; expires=Sun, 10-Jan-2021 09:16:20 GMT; path=/; domain=.google.com; Secure; HttpOnly; SameSite=none", + "Strict-Transport-Security": "max-age=604800", + "X-Content-Type-Options": "nosniff", + "X-Frame-Options": "SAMEORIGIN", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://books.google.com/books/download/?id=pzmt3pcBuGYC&output=ris" +} \ No newline at end of file diff --git a/test/testdata/fb136eb9d3ecb87c6d185e8987b3d780c968d463.html b/test/testdata/fb136eb9d3ecb87c6d185e8987b3d780c968d463.html new file mode 100644 index 00000000..ae383b6d --- /dev/null +++ b/test/testdata/fb136eb9d3ecb87c6d185e8987b3d780c968d463.html @@ -0,0 +1,2101 @@ + + + + + + The Boston Globe + + + + + +The Boston Globe + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + + +
          +
          + + +
          + + +
          + Menu + +
          +
          + + +
          +
          + + +
          +
          + + + + + + + +
          + + +
          + +
          + +
          +
          + +
          +
          + +
          +
          +
          +
          + +
          +
          +
          + +

          +

          + Brennan warned Russia against election meddling +

          + +
          +

          Former CIA Director John Brennan told the House intelligence committee on Tuesday that he was the first US official to call out the Russians for their activities. +

          +
          + +
          + + +

          + +

          + +
          +
          +

          +

          +

          + + //c.o0bg.com/rf/image_90x90/Boston/2011-2020/2017/05/23/BostonGlobe.com/Politics/Images/9589bc0810784c8e8c5c7619662ef295-9589bc0810784c8e8c5c7619662ef295-0.jpg + + Intel director won’t detail Trump conversation + +

          +
          +

          Dan Coats says he won’t comment on a news report that President Trump asked him to publicly deny any collusion between his campaign and Russia. +

          +
          +
          +
          + + + Budget Director Mick Mulvaney speaks to the media about President Donald Trump's proposed fiscal 2018 federal budget in the Press Briefing Room of the White House in Washington, Tuesday, May 23, 2017. (AP Photo/Andrew Harnik) +

          Ground Game +

          +

          The budget process was broken before Trump

          +
          +
          +

          President Trump formally released his first federal budget proposal Tuesday, and it’s already picked up criticism from the left and the right. +

          +
          + +
          +

          Opinion | Michael A. Cohen

          +

          + + //c.o0bg.com/rf/image_90x90/Boston/2011-2020/2017/05/23/BostonGlobe.com/EditorialOpinion/Images/cohen-3621.jpg + + Trump’s smoking gun + +

          +
          +

          What more evidence could possibly be needed that President Trump has committed a high crime and misdemeanor? +

          +
          +
          +

          Perspective

          +

          + + //c.o0bg.com/rf/image_90x90/Boston/2011-2020/2017/05/19/BostonGlobe.com/Magazine/Images/illo0528perspective.jpg + + The only way to avoid getting caught in Trump’s echo chamber + +

          +
          +

          We’re all paying too much attention to Trump, not the issues. Here’s how to muffle the noise. +

          +
          +
          + +

          +

          +

          Fall River man charged with stabbing girlfriend to death

          +
          +

          Fall River police responded to a Linden Street apartment and found Kristina Reis bleeding from an apparent stab wound, authorities said. +

          +
          + +
          +

          +

          +

          + + //c.o0bg.com/rf/image_90x90/Boston/2011-2020/2017/05/23/BostonGlobe.com/Metro/Images/chelsea3.jpg + + Man who died exchanging gunfire with police in Chelsea had chased wife out of home + +

          +
          +

          The Suffolk DA office will lead the investigation into the armed standoff that ended with a 38-year-old man dead. +

          +
          +
          + +

          +

          +

          Worcester police searching for driver who hit 3-year-old and then drove away

          +
          +

          Worcester police are currently searching for the Marquis with the Massachusetts license plate 5NN395. +

          +
          + +
          +

          +

          +

          + + //c.o0bg.com/rf/image_90x90/Boston/2011-2020/2017/05/23/BostonGlobe.com/Metro/Images/ryan_redbones_met.jpg + + Redbones restaurant in Somerville damaged in fire + +

          +
          +

          The Somerville barbecue restaurant was damaged when a small fire started in the ductwork Tuesday morning. +

          +
          +
          +

          +

          +

          + + //c.o0bg.com/rf/image_90x90/Boston/2011-2020/2017/04/01/BostonGlobe.com/Magazine/Images/FIDELITYABIGAILJOHNSON.jpg + + Abby loves bitcoin: Fidelity chief touts digital currency in first major speech + +

          +
          +

          Abigail Johnson, Fidelity Investments chairman, is expected to say digital currency has potential to greatly improve the process of financial transactions. +

          +
          +
          + +
          + +
          + +
          + + +

          +

          +

          Suspected Manchester concert bomber identified

          +
          + +
          + + Forensic investigators searched the property of Salmon Abedi in connection with the deadly Manchester attack. 
+ + +
          +

          Danny Lawson/PA via Associated Press +

          +
          +
          +
          +

          British authorities have identified the suspected Manchester suicide bomber as 22-year-old Salman Abedi. +

          +
          + + +
          +

          +

          +

          + + //c.o0bg.com/rf/image_90x90/Boston/2011-2020/2017/05/23/BostonGlobe.com/Foreign/Images/208169c312f146b182cad2f00ff8eb83-208169c312f146b182cad2f00ff8eb83-0-2299-kmrG--90x90@BostonGlobe.com.jpg + + 8-year-old girl among those killed in Manchester arena blast + +

          +
          +

          Saffie Roussos was among the 22 people who died in the bombing. +

          +
          +
          +

          +

          +

          + + //c.o0bg.com/rf/image_90x90/Boston/2011-2020/2017/05/23/BostonGlobe.com/National/Images/686973322.jpg + + The targeting of women and girls in Manchester may have been intentional + +

          +
          +

          Among other things, this was a concert meant to celebrate female empowerment, and many of the victims were young British women there to take part. +

          +
          +
          +

          Dan Wasserman

          +

          + + //c.o0bg.com/rf/image_90x90/Boston/2011-2020/2017/05/23/BostonGlobe.com/EditorialOpinion/Images/052417MANCHESTER.jpeg + + Mourning for Manchester + +

          +
          +

          Editorial cartoonist Dan Wasserman reacts to the Manchester attack. +

          +
          +
          + +
          + + + Sir Roger Moore during a press conference for UNICEF at the Ritz-Carlton Hotel in Boston in 1997. +

          + +

          +

          Sir Roger Moore dies at 89

          +
          +
          +

          Mr. Moore was the longest-serving movie star to play iconic British spy James Bond in the famed film series. +

          +
          + +
          + + + +
          + New England Patriots wide receiver Julian Edelman (11) celebrates after making a touchdown reception during the second half of the AFC championship NFL football game against the Pittsburgh Steelers, Sunday, Jan. 22, 2017, in Foxborough, Mass. (AP Photo/Elise Amendola)
+ + +
          +

          AP file +

          +
          +
          +

          +

          +

          NFL relaxes its policy on player celebrations

          +
          + +
          +

          Things such as using the football as a prop and group theatrics will no longer be penalized. +

          +
          + +
          +

          Alex Speier

          +

          + + //c.o0bg.com/rf/image_90x90/Boston/2011-2020/2017/05/04/BostonGlobe.com/Sports/Images/tlumacki_redsoxvsorioles_sports_015ports.jpg + + Is John Farrell to blame for Red Sox not living up to expectations? + +

          +
          +

          After three losses in Oakland, there’s been a new run of speculation about the Red Sox manager. +

          +
          +
          + + + Epstein at Yale Class Day. +

          names +

          +

          Theo Epstein tells grads ‘character matters’

          +
          +
          +

          The Cubs president of baseball ops said, “Keep your heads up, and come together to connect and to rally around one another, especially those who need it the most.”

          +
          + +
          +

          +

          +

          + + //c.o0bg.com/rf/image_90x90/Boston/2011-2020/2017/05/19/BostonGlobe.com/Metro/Images/ryan_jaywalkers1_met.jpg + + Fearless, defiant, detested: Meet the Boston jaywalker + +

          +
          +

          Woe to anyone who honks at them in frustration. +

          +
          +
          +
          +
          + +
          +
          +
          + +
          +
          + +

          Trump Today

          +
          +
          +

          +

          +

          + + //c.o0bg.com/rf/image_90x90/Boston/2011-2020/2017/05/22/BostonGlobe.com/Politics/Images/0fc1e2f1824a466ea23495910ff92a75-0fc1e2f1824a466ea23495910ff92a75-0.jpg + + Michael Flynn back in the spotlight + +

          +
          +

          Trump’s former national security adviser may have lies about his contacts with Russian officials. +

          +
          +
          +
          + +

          Opinion & Ideas

          +
          +
          +

          Opinion | Venkat Sumantran, Charles Fine, and David Gonsalvez

          +

          + + //c.o0bg.com/rf/image_90x90/Boston/2011-2020/2017/05/22/BostonGlobe.com/EditorialOpinion/Images/update23FUTUREWEB1.jpg + + How to improve Boston’s infrastructure future + +

          +
          +

          The Go Boston 2030 plan can and should be more ambitious. +

          +
          +
          +
          + +

          Special reports

          +
          +
          +

          +

          +

          + + //c.o0bg.com/rf/image_90x90/Boston/2011-2020/2017/05/19/BostonGlobe.com/Metro/Images/kreiter_syrians_mustafasews4_met-9791.jpg + + The last refugee: Threads of a new life in America + +

          +
          +

          After escaping Syria, all Abdulkader wanted was to work. That wasn’t enough to get a job in America. +

          +
          +
          +
          + +

          Newsletters

          +
          +
          +

          +

          +

          + + //c.o0bg.com/rf/image_90x90/Boston/2011-2020/2016/12/20/BostonGlobe.com/Business/Images/629844334[1].jpg + + Sign up for Talking Points + +

          +
          +

          An afternoon recap of the day’s most important business news, delivered Monday through Friday. +

          +
          +
          +
          + +

          Address: Spring House Hunt

          +
          +
          +

          +

          +

          + + //c.o0bg.com/rf/image_90x90/Boston/2011-2020/2017/04/06/BostonGlobe.com/Lifestyle/Images/realestate.jpg + + Advice on taking real estate photos from a Pulitzer-winning photographer + +

          +
          +

          In America, everyone who has a cellphone thinks they are a professional photographer. They are not. +

          +
          +
          +
          + +

          Investigations

          +
          +
          +

          +

          +

          + + //c.o0bg.com/rf/image_90x90/Boston/2011-2020/2017/03/03/BostonGlobe.com/Foreign/Images/AFP_MB3WQ.jpg + + Documenting Hate: Submit an incident you know about + +

          +
          +

          If you have been a victim or witnessed a hate incident, share your information here so we can investigate. +

          +
          +
          +
          + +

          STAT

          +
          +
          +

          +

          +

          + + //c.o0bg.com/rf/image_90x90/Boston/2011-2020/2017/05/23/BostonGlobe.com/National/Images/Trump.png + + Trump used to be more articulate. What could explain the change? + +

          +
          +

          STAT asked experts to compare Trump’s speech from decades ago to that in 2017. All noticed a deterioration. +

          +
          +
          +
          +
          +
          +
          + +
          +
          + +
          +
          + + + + + + + + + + + + +
          + + + + + + +
          +
          +
          + +
          +
          + We hope you've enjoyed your free articles. +
          +
          + Continue reading by subscribing to Globe.com for just 99¢. +
          +
          +  Already a member? Log in Home +
          +
          +
          +
          +
          + + + + + + + + + + +
          +
          + +
          + Subscriber Log In +

          We hope you've enjoyed your 5 free articles'

          + + + +
          +
          +
          +

          Stay informed with unlimited access to Boston’s trusted news source.

          +
            +
          • High-quality journalism from the region’s largest newsroom
          • +
          • Convenient access across all of your devices
          • +
          • Today’s Headlines daily newsletter
          • +
          • Subscriber-only access to exclusive offers, events, contests, eBooks, and more
          • +
          • Less than 25¢ a week
          • +
          +
          + +
          +
          + Marketing image of BostonGlobe.com +
          +
          +
          + +
          + +
          +
          +
          + + + + + +
          +
          + +
          +
          + +
          +
          + Marketing image of BostonGlobe.com +
          +
          + +
          + + + + + + + + + + + + + + + + + + + + +
          + + + diff --git a/test/testdata/fb136eb9d3ecb87c6d185e8987b3d780c968d463.json b/test/testdata/fb136eb9d3ecb87c6d185e8987b3d780c968d463.json new file mode 100644 index 00000000..259c3d59 --- /dev/null +++ b/test/testdata/fb136eb9d3ecb87c6d185e8987b3d780c968d463.json @@ -0,0 +1,26 @@ +{ + "encoding": "UTF-8", + "headers": { + "Accept-Ranges": "bytes", + "Age": "162", + "Cache-Control": "no-cache, must-revalidate, max-age=0, no-store", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "39568", + "Content-Type": "text/html;charset=UTF-8", + "Date": "Tue, 23 May 2017 17:53:04 GMT", + "Eomportal-Instance": "215", + "Expires": "Thu, 01 Jan 1970 00:00:00 GMT", + "Pragma": "no-cache", + "Server": "BostonGlobe.com Frontend", + "Vary": "Origin, Accept-Encoding", + "Via": "1.1 varnish", + "X-Cache": "HIT", + "X-Cache-Hits": "4", + "X-Served-By": "cache-iad2143-IAD", + "X-TTL": "5m", + "X-Timer": "S1495561984.290037,VS0,VE0" + }, + "status_code": 200, + "url": "http://www.bostonglobe.com/" +} \ No newline at end of file diff --git a/test/testdata/fdf48b54f769185f2ce21519453acc0b7b84b403.html b/test/testdata/fdf48b54f769185f2ce21519453acc0b7b84b403.html new file mode 100644 index 00000000..84ecfef6 --- /dev/null +++ b/test/testdata/fdf48b54f769185f2ce21519453acc0b7b84b403.html @@ -0,0 +1,709 @@ + + + + + + + + + + + + + + + + + + + +Bulletin of the Atomic Scientists | + + + + + + + + + + + + + + + + + + + + + +
          + +
          +
          Close
          +
          +
          +
          +
          +
          +

          Overview

          The Doomsday Clock is an internationally recognized design that conveys how close we are to destroying our civilization with dangerous technologies of our own making. First and foremost among these are nuclear weapons, but the dangers include climate-changing technologies, emerging... Read More

          +
          +
          +
          +
          +

          Press Release

          + +
          +
          +
          +
          +
          +
          +
          +
          +
          2017
          +
          +
          2016
          +
          +
          2015
          +
          +
          2012
          +
          +
          2010
          +
          +
          2007
          +
          +
          2002
          +
          +
          1998
          +
          +
          1995
          +
          +
          1991
          +
          +
          1990
          +
          +
          1988
          +
          +
          1984
          +
          +
          1981
          +
          +
          1980
          +
          +
          1974
          +
          +
          1972
          +
          +
          1969
          +
          +
          1968
          +
          +
          1963
          +
          +
          1960
          +
          +
          1953
          +
          +
          1949
          +
          +
          1947
          +
          +
          +
          +
          +
          +
          +
          + +
          +

          The Clock:
          A Brief History

          The Clock brief history
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +

          shutterstock_585173494.jpg

          +
          +Credit: Shutterstock
          +
          +
          ANALYSIS

          Why a nuclear crisis group?

          Rachel Bronson

          Former diplomats and military officers form a “shadow security council” of experts to provide advice to world leaders in a perilous nuclear age

          +
          +
          +
          +
          +

          shutterstock_244390984.jpg

          +
          +Credit: Shutterstock
          +
          +
          ANALYSIS

          Missiles of the past?

          Jon B. Wolfsthal

          Aging Minuteman III missiles are the most vulnerable and least essential components of the US nuclear arsenal. Should they be replaced -- or simply eliminated?

          +
          +
          +
          +
          +

          atlas_cern_900.jpg

          +
          +Photo credit: Maximilien Brice, CERN
          +
          +
          ANALYSIS

          A national interest, and so much more

          Lawrence M. Krauss

          Why the March for Science must be clear-eyed in its defense of the scientific process as an independently valuable human activity

          +
          +
          +
          +
          +

          Screen Shot 2017-04-18 at 12.41.13 PM.png

          +
          +
          +
          +
          VIDEO

          Why the March for Science?

          Bulletin Staff

          Bulletin Science and Security Board member Raymond Pierrehumbert explains: In an age of alternative facts, "the truth needs an advocate."

          +
          +
          +
          +
          +

          Trump-signs-climate-order.jpg

          +
          +
          +
          +
          Expert commentary

          Trump's climate plan: experts respond

          Dawn Stover

          Top expert commentary on President Trump's effort to roll back Obama administration climate change policies.

          +
          +
          +
          +
          +
          +
          +
          +
          +
          +1
          +
          +
          +
          +
          +
          +2
          +
          +
          +
          +
          +
          +3
          +
          +
          +
          +
          +
          +4
          +
          +
          +
          +
          +
          +5
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + + +
          +
          +
          +

          Nuclear Notebook

          +
          +
          +Nuclear notebook
          +
          +
          +
          March

          Russian nuclear forces, 2017

          Russia is in the middle of a broad modernization of its strategic and nonstrategic nuclear forces, including both new programs and some that have been underway for many years. As of early 2017, the authors estimate that Russia has a military stockpile of roughly 4,300 nuclear warheads assigned for use by long-range strategic launchers and shorter-range tactical nuclear forces. Of these,...

          +
          + +
          +
          + +
          +
          +
          + +
          +
          +
          + +
          + +
          + + + +
          + + + + + + + + + + + + diff --git a/test/testdata/fdf48b54f769185f2ce21519453acc0b7b84b403.json b/test/testdata/fdf48b54f769185f2ce21519453acc0b7b84b403.json new file mode 100644 index 00000000..906fc47c --- /dev/null +++ b/test/testdata/fdf48b54f769185f2ce21519453acc0b7b84b403.json @@ -0,0 +1,30 @@ +{ + "encoding": "utf-8", + "headers": { + "Age": "114", + "CF-RAY": "3639e0bd177f2438-IAD", + "Cache-Control": "public, max-age=300", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Language": "en", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 17:53:59 GMT", + "Expires": "Sun, 19 Nov 1978 05:00:00 GMT", + "Last-Modified": "Tue, 23 May 2017 17:52:03 GMT", + "Link": "; rel=\"canonical\",; rel=\"shortlink\"", + "Server": "cloudflare-nginx", + "Set-Cookie": "__cfduid=d16d6b215ce2f0298d832c9a9c750aa251495562039; expires=Wed, 23-May-18 17:53:59 GMT; path=/; domain=.thebulletin.org; HttpOnly", + "Transfer-Encoding": "chunked", + "Vary": "Cookie,Accept-Encoding", + "Via": "1.1 varnish-v4", + "X-Content-Type-Options": "nosniff", + "X-Drupal-Cache": "MISS", + "X-Frame-Options": "SAMEORIGIN", + "X-Generator": "Drupal 7 (http://drupal.org)", + "X-UA-Compatible": "IE=edge", + "X-Varnish": "12403037 15963998", + "X-Varnish-Cache": "HIT" + }, + "status_code": 200, + "url": "http://thebulletin.org/" +} \ No newline at end of file diff --git a/test/testdata/fec94946a653b0456d71336b36490392c5145dfb.html b/test/testdata/fec94946a653b0456d71336b36490392c5145dfb.html new file mode 100644 index 00000000..2ac20a59 --- /dev/null +++ b/test/testdata/fec94946a653b0456d71336b36490392c5145dfb.html @@ -0,0 +1 @@ +{"indexed":{"date-parts":[[2022,4,4]],"date-time":"2022-04-04T00:31:33Z","timestamp":1649032293580},"reference-count":20,"publisher":"Informa UK Limited","issue":"4","content-domain":{"domain":[],"crossmark-restriction":false},"published-print":{"date-parts":[[2007,11]]},"DOI":"10.1657\/1523-0430(07-512)[zhu]2.0.co;2","type":"journal-article","created":{"date-parts":[[2007,11,7]],"date-time":"2007-11-07T15:33:10Z","timestamp":1194449590000},"page":"658-662","source":"Crossref","is-referenced-by-count":8,"title":"Ostracoda Assemblages in Core Sediments and Their Environmental Significance in a Small Lake in Northwest Tibet, China","prefix":"10.1080","volume":"39","author":[{"given":"Liping","family":"Zhu","sequence":"first","affiliation":[]},{"given":"Xiao","family":"Lin","sequence":"additional","affiliation":[]},{"given":"Yuanfang","family":"Li","sequence":"additional","affiliation":[]},{"given":"Bingyuan","family":"Li","sequence":"additional","affiliation":[]},{"given":"Manping","family":"Xie","sequence":"additional","affiliation":[]}],"member":"301","reference":[{"key":"i1523-0430-39-4-658-Carbonel1","first-page":"413","volume":"62","author":"Carbonel","year":"1988","journal-title":"Palaeogeography Palaeoclimatology Palaeoecology","ISSN":"http:\/\/id.crossref.org\/issn\/0031-0182","issn-type":"print"},{"key":"i1523-0430-39-4-658-Danielopol1","first-page":"65","author":"Danielopol","year":"1993","journal-title":"Ostracoda in the Earth and Life Sciences"},{"key":"i1523-0430-39-4-658-deDeckker1","first-page":"131","volume":"81","author":"de Deckker","year":"1981","journal-title":"Hydrobiologia","ISSN":"http:\/\/id.crossref.org\/issn\/0018-8158","issn-type":"print"},{"key":"i1523-0430-39-4-658-deDeckker2","first-page":"175","author":"de Deckker","year":"1988","journal-title":"Ostracoda in the Earth Sciences"},{"key":"i1523-0430-39-4-658-Feng1","first-page":"633","volume":"43","author":"Feng","year":"1998","journal-title":"Chinese Science Bulletin","ISSN":"http:\/\/id.crossref.org\/issn\/1001-6538","issn-type":"print"},{"key":"i1523-0430-39-4-658-Flohn1","first-page":"182","volume":"130","author":"Flohn","year":"1968","journal-title":"Atmospheric Science Paper","ISSN":"http:\/\/id.crossref.org\/issn\/0067-0340","issn-type":"print"},{"key":"i1523-0430-39-4-658-Krishnaswamy1","first-page":"407","volume":"11","author":"Krishnaswamy","year":"1971","journal-title":"Earth and Planetary Science Letters","ISSN":"http:\/\/id.crossref.org\/issn\/0012-821X","issn-type":"print"},{"key":"i1523-0430-39-4-658-Peng1","first-page":"239","volume":"14","author":"Peng","year":"1997","journal-title":"Acta Micropalaeonotologica Sinica"},{"key":"i1523-0430-39-4-658-Pennington1","first-page":"324","volume":"242","author":"Pennington","year":"1973","journal-title":"Nature","ISSN":"http:\/\/id.crossref.org\/issn\/1476-4687","issn-type":"print"},{"key":"i1523-0430-39-4-658-Scharf1","first-page":"3","author":"Scharf","year":"1993","journal-title":"Ostracoda in the Earth and Life Sciences"},{"key":"i1523-0430-39-4-658-Tang1","first-page":"896","volume":"41","author":"Tang","year":"1999","journal-title":"Acta Botanica Sinica","ISSN":"http:\/\/id.crossref.org\/issn\/0577-7496","issn-type":"print"},{"key":"i1523-0430-39-4-658-Wan1","first-page":"73","volume":"19","author":"Wan","year":"1999","journal-title":"Quaternary Sciences","ISSN":"http:\/\/id.crossref.org\/issn\/1001-7410","issn-type":"print"},{"key":"i1523-0430-39-4-658-Wan2","first-page":"674","volume":"36","author":"Wan","year":"1991","journal-title":"Chinese Science Bulletin","ISSN":"http:\/\/id.crossref.org\/issn\/1001-6538","issn-type":"print"},{"key":"i1523-0430-39-4-658-Wang1","first-page":"1","author":"Wang","year":"1990","journal-title":"Daihai"},{"key":"i1523-0430-39-4-658-Wang2","first-page":"54","volume":"18","author":"Wang","year":"1998","journal-title":"Quaternary Sciences","ISSN":"http:\/\/id.crossref.org\/issn\/1001-7410","issn-type":"print"},{"key":"i1523-0430-39-4-658-Wu1","first-page":"47","volume":"29","author":"Wu","year":"2005","journal-title":"Chinese Journal of Atmospheric Sciences","ISSN":"http:\/\/id.crossref.org\/issn\/0891-3862","issn-type":"print"},{"key":"i1523-0430-39-4-658-Yao1","first-page":"425","volume":"39","author":"Yao","year":"1996","journal-title":"Sciences in China (ser. D)","ISSN":"http:\/\/id.crossref.org\/issn\/1006-9313","issn-type":"print"},{"key":"i1523-0430-39-4-658-Zhang1","first-page":"14","year":"1999","journal-title":"Environmental Changes in Late Cenozoic Era of Karakorum and Kunlun Mountains"},{"key":"i1523-0430-39-4-658-Zheng1","first-page":"410","volume":"39","author":"Zheng","year":"1996","journal-title":"Science in China (ser.D)","ISSN":"http:\/\/id.crossref.org\/issn\/1006-9313","issn-type":"print"},{"key":"i1523-0430-39-4-658-Zhu1","first-page":"430","volume":"45","author":"Zhu","year":"2002","journal-title":"Science in China (ser. D)","ISSN":"http:\/\/id.crossref.org\/issn\/1006-9313","issn-type":"print"}],"container-title":"Arctic, Antarctic, and Alpine Research","original-title":[],"language":"en","deposited":{"date-parts":[[2020,6,4]],"date-time":"2020-06-04T08:31:15Z","timestamp":1591259475000},"score":1,"resource":{"primary":{"URL":"https:\/\/ww.tandfonline.com\/doi\/full\/10.1657\/1523-0430(07-512)[ZHU]2.0.CO;2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2007,11]]},"references-count":20,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2007,11]]}},"alternative-id":["10.1657\/1523-0430(07-512)[ZHU]2.0.CO;2"],"URL":"http:\/\/dx.doi.org\/10.1657\/1523-0430(07-512)[ZHU]2.0.CO;2","relation":{},"ISSN":["1523-0430","1938-4246"],"subject":["Earth-Surface Processes","Ecology, Evolution, Behavior and Systematics","Global and Planetary Change"],"container-title-short":"Arctic, Antarctic, and Alpine Research","published":{"date-parts":[[2007,11]]}} \ No newline at end of file diff --git a/test/testdata/fec94946a653b0456d71336b36490392c5145dfb.json b/test/testdata/fec94946a653b0456d71336b36490392c5145dfb.json new file mode 100644 index 00000000..854e9489 --- /dev/null +++ b/test/testdata/fec94946a653b0456d71336b36490392c5145dfb.json @@ -0,0 +1,24 @@ +{ + "encoding": null, + "headers": { + "access-control-allow-headers": "X-Requested-With, Accept, Accept-Encoding, Accept-Charset, Accept-Language, Accept-Ranges, Cache-Control", + "access-control-allow-origin": "*", + "access-control-expose-headers": "Link", + "connection": "close", + "content-encoding": "gzip", + "content-length": "1659", + "content-type": "application/vnd.citationstyles.csl+json", + "date": "Thu, 09 Jun 2022 10:35:33 GMT", + "link": "; rel=\"canonical\"", + "permissions-policy": "interest-cohort=()", + "server": "Jetty(9.4.40.v20210413)", + "vary": "Accept, Accept-Encoding", + "x-api-pool": "public", + "x-rate-limit-interval": "1s", + "x-rate-limit-limit": "50", + "x-ratelimit-interval": "1s", + "x-ratelimit-limit": "50" + }, + "status_code": 200, + "url": "https://api.crossref.org/v1/works/10.1657%2F1523-0430%2807-512%29%5BZHU%5D2.0.CO%3B2/transform" +} \ No newline at end of file diff --git a/test/testdata/fef78097af2366d041a66839250c48877990c79b.html b/test/testdata/fef78097af2366d041a66839250c48877990c79b.html new file mode 100644 index 00000000..02c258d5 --- /dev/null +++ b/test/testdata/fef78097af2366d041a66839250c48877990c79b.html @@ -0,0 +1,1785 @@ + + + + + + + خبرگزاری فارس | صفحه اصلی | Fars News Agency + + + + + + + + + + + + + + +
          +
          +
          +
          +
          +
          +
          +
            +
          • طلا، سکه و ارز
          • +
          • بورس
          • +
          • قیمت خودرو
          • +
          +
          + + + + +
          +
          + +
          +
          + + + سه شنبه ۰۲ خرداد ۱۳۹۶ - ۲۲:۲۳ + +
          +
          + +
          +
          + + + + + +

          صفحه اصلی خبرگزاری فارس FarsNews

          +
          +
            +
          • صفحه اصلی
          • +
          • برگزیده ها
          • +
          • آخرين اخبار
          • +
          +
          +
          +

          منچستر

          حمله تروریستی در منچستر ۲۲ کشته برجا گذاشت/ تبلیغات انتخاباتی احزاب سیاسی متوقف شد/ داعش حمله منچستر را برعهده گرفت اما آمریکا تایید نمی‌کند

          وقوع انفجاری در محل برگزاری کنسرتی درشهر منچستر انگلیس، تا کنون ۲۲ کشته شامل چندین کودک و ۱۲۰ زخمی بر جای گذاشته است و در پی آن تبلیغات انتخاباتی احزاب سیاسی متوقف و دولت تشکیل جلسه فوری داد.

          پرسپولیس - لخویا

          مرحله یک هشتم نهایی لیگ قهرمانان آسیا

          توقف پرسپولیس بدون هوادار مقابل لخویا/ طارمی دوباره پنالتی خراب کرد!

          دیدار دو تیم پرسپولیس ایران و لخویا قطر در مرحله یک هشتم لیگ قهرمانان آسیا با تساوی بدون گل به پایان رسید.

          محمد صادق کوشکی

          کوشکی در گفت‌وگو با فارس:

          اعلام آرای تفکیکی ریاست جمهوری در این دولت مثل قراردادهای نفتی، سرنوشت کرسنت و قرارداد خرید هواپیماهای مرجوعی محرمانه است

          عضو هیأت علمی دانشگاه تهران گفت: عدم اعلام آراء تفکیکی ریاست جمهوری در این دولت مثل قراردادهای نفتی، سرنوشت کرسنت و قرارداد خرید هواپیماهای مرجوعی محرمانه است.

          سیدحسن حسینی‌شاهرودی

          عضو هیأت رئیسه کمیسیون اقتصادی مجلس:

          قرار بود تا پایان اردیبهشت‌ تکلیف کاسپین روشن‌ شود/ بانک مرکزی مقصر است

          عضو هیأت رئیسه کمیسیون اقتصادی مجلس گفت: وعده داده بودند تا پایان اردیبهشت‌‌ماه تکلیف کاسپین روشن‌ شود و بانک مرکزی از منظر مجلس مقصر است.

          وزارت کشور02

          فارس از اوضاع نابسامان شمارش آراء گزارش می‌دهد

          عملکرد مساله‌دار وزارت کشور و فرمانداری صدای اصلاح‌طلبان را هم در آورد/ سرخو: شمارش آرای من از قلم افتاده

          کاندیدای اصلاح طلب شورای شهر تهران طی نامه‌ای به فرماندار تهران خواستار بازشماری آرای ماخوذه با حضور نماینده این کاندیدا شده است.

          دلار، ارز، بازار سکه و ارز

          رشد قیمت ارزها در بازار/ دلار ۳۷۵۷ تومان+ جدول

          در بازار تهران امروز قیمت دلار 22 تومان، یورو 17 تومان، پوند 8 تومان و درهم 4 تومان افزایش یافت.

          روحانی

          روحانی به منظور زیارت بارگاه مطهر امام رضا(ع) وارد مشهد شد

          رئیس جمهور برای زیارت بارگاه مطهر هشتمین اختر تابناک آسمان ولایت و امامت حضرت علی بن موسی الرضا(ع) وارد مشهد مقدس شد.

          کیومرث عباسی قصری

          سی و ششمین «شب شاعر» برگزار شد

          پاسداشت برای «شاعرِ مردم»/ «قصر شیرین» استعاره‌ای برای شعر عباسی‌قصری

          سی و ششمین آیین «شب شاعر» پاسداشت کیومرث عباسی قصری در حالی برگزار شد که در این مراسم گفته شد که او شاعر مردم است.

          موسوی‌نژاد موسوی‌ نژاد

          موسوی‌نژاد در گفت‌وگو با فارس:

          عدم ارائه نتایج انتخابات به تفکیک استان‌ها یکی از تخلفات سازماندهی‌شده دولت در انتخابات است

          نماینده دوره نهم مجلس شورای اسلامی گفت: عدم ارائه نتایج انتخابات به تفکیک استان‌ها یکی از تخلفات سازماندهی‌شده دولت در بحث انتخابات است و اگر وزارت کشور مدعی است که انتخابات را به خوبی برگزار کرده، چرا از ارائه نتایج آمار به تفکیک استان‌ها استنکاف می‌ورزد.

          موسسه اعتباری کاسپین

          ظهر امروز صورت گرفت

          دومین اجتماع مالباختگان مؤسسه کاسپین در مشهد/ بانک مرکزی پاسخگو نیست

          دومین اجتماع مالباختگان مؤسسه مالی کاسپین امروز در حالی برگزار شد که سپرده‌گذاران از بی‌اعتنایی بانک مرکزی و عدم پاسخگویی مسؤولان به خواست خود گله‌مند هستند.

          علیرضا سلیمی نماینده مجلس دهم

          سلیمی در گفت‌وگو با فارس:

          وزارت کشور سریعاً آمار تفکیکی آرای انتخابات ریاست‌جمهوری در استان‌ها را اعلام کند/ لزوم جلوگیری از ایجاد شبهه در اذهان عمومی

          نماینده محلات و دلیجان در مجلس گفت: وزارت کشور سریعاً آمار تفکیکی آرای انتخابات ریاست‌جمهوری در استان‌ها را اعلام و از ایجاد شبهه در اذهان عمومی جلوگیری کند.

          قاسم خورشیدی سخنگوی ستاد مبارزه قاچاق

          سخنگوی ستاد مبارزه با قاچاق به فارس خبر داد

          جزئیات ورود قاچاق ۳۰ هزار کولرگازی «جنرال»

          سخنگوی ستاد مبارزه با قاچاق کالا و ارز جزئیات ورود ۳۰ هزار کولر گازی «جنرال» قاچاق از طریق گمرک را تشریح کرد.

          متین منتظمی

          منتظمی در گفت وگو با فارس:

          روحانی نمی‌تواند رقیب 16 میلیونی را نادیده بگیرد/بررسی کنند چرا در 8 استان از جمله زادگاه رئیس جمهور شکست خوردند؟

          دبیرکل اتحادیه جامعه اسلامی دانشجویان گفت: دولت دوازدهم بعد از انتخابات با 16 میلیون نفری مواجه شد که رئیسی را به عنوان نامزد خود انتخاب کردند به همین دلیل دیگر نمی‌تواند این رقیب قدرتمند را نادیده بگیرد و باید به مطالبات آنها پاسخ دهد.

          123انتخابات

          شبکه دانشگاهیان انقلاب اسلامی در بیانیه‌ای خطاب به مردم

          دولت قدر ملت را بداند و در خدمتگزاری به آنان ذره‌ای دریغ نکند

          شبکه دانشگاهیان انقلاب اسلامی در بیانیه‌ای خطاب به مردم ایران تاکید کرد: از دولت انتظار داریم حال که ملت عزیز ایران به تکلیف خود در عمل نمودند، قدر این ملت را بداند و در خدمتگزاری به آنان ذره‌ای دریغ نکند.

          بورس

          فارس از آن سوی تالار شیشه‌ای گزارش می‌دهد

          درجا زدن شاخص کل بورس روی 81 هزار و 124 واحد/ بازار سهام در بی‌رمقی کامل

          معامله‌گران بورس تهران امروز شاهد معاملاتی کم عمق و رخوت‌آور در کلیت معاملات بازار سهام بودند که فیلم‌های وسترن و شهرهای متروکه در غرب وحشی را به ذهن‌ها متبادر می‌کرد.

          فرهاد تجری

          تجری در پاسخ به فارس:

          نمایندگان حق دارند معترض روند نظارت انتخابات شوراها باشند/ برای رسیدگی به شکایت 1400 نفر فقط 24 ساعت فرصت داشتیم

          سخنگوی هیأت نظارت بر انتخابات شوراهای شهر و روستا درباره نامه ۵۲ نماینده در اعتراض به تخلفات این هیأت در تأیید صلاحیت‌ها گفت: هیأت نظارت نمی‌تواند بی‌نقص باشد و خلأ قانونی در این باره وجود دارد که باید هرچه سریع‌تر به آن رسیدگی شود.

          علی نیکزاد

          طی نامه‌ای به وزیر کشور صورت گرفت

          اعتراض رئیس ستاد «رئیسی» به عدم ارائه شفاف نتایج انتخابات به تفکیک صندوق‌ها

          علی نیکزاد رئیس ستاد انتخاباتی حجت‌الاسلام سیدابراهیم رئیسی در دوازدهمین دوره انتخابات ریاست جمهوری طی نامه‌ای به وزیر کشور از عدم ارائه شفاف نتایج به تفکیک صندوق‌های اخذ رأی انتقاد کرد.

          شعبه بانک

          فارس گزارش می‌دهد

          تعهد غیرمنصفانه‌ای که بانک مرکزی از سپرده‌گذاران ثامن‌الحجج گرفت+تصویر

          11 ماه از آغاز بازپرداخت سپرده‌های بالای 35 میلیون تومان مؤسسه ورشکسته ثامن الحجج می‌گذرد اما هیچ بازپرداختی نشده است و سپرده گذاران نیز با توجه به تعهد خود به بانک مرکزی، نمی‌توانند برای وصول طلب خود اعتراض و ادعا کنند.

          ارتش سوریه

          تداوم عملیات ارتش سوریه از قلمون تا تدمر؛ بیش از یک هزار کیلومتر آزاد شد

          ارتش سوریه اخیرا عملیات خود را از قلمون شرقی در ریف دمشق تا جنوب شهر تدمر واقع در ریف جنوب شرقی حمص ادامه داد و موفق شد مناطق زیادی را آزاد کند.

          +
          +
          + + +
          +
           
          +
           
          +
          + + +
          + برگزیده ها  +
          +
          + +

          دستگیری شکارچی تصاویر دختران

          نجات راننده محبوس‌شده در زیر اتوبوس

          پاسخ منفی مرودشتی‌ها به لیست امید

          نشست عکاسی خبری جنگ و بحران

          ترکیب تیم پرسپولیس مقابل لخویا

          بوکسوری که آتش‌نشان شد

          دستگیری قاتل فراری پس از 11سال

          رشد قیمت ارزها در بازار

          شوک انفجار منچستر به بازار طلا

          دومین سرطان شایع در میان مردان

          تکلیف صعود برق فردا مشخص می‌شود

          بازگشت پیکر130شهید به وطن

          کاهش‌زمان تردد در محدوده زوج‌وفرد

          اعتراض به نتیجه انتخابات شورای تبریز

          منتخبان شورای شهر اصفهان

          از هدفون دیگران استفاده نکنید

          کشتی قم صاحب مربی بانوان شد

          حمله به منزل شیخ عیسی قاسم

          اولین‌توصیه‌رهبر‌انقلاب‌به‌سید‌محمد‌خاتمی

          انفجار خونین در سالن کنسرت منچستر

          کمپین 16 میلیونی و کابینه دوم روحانی

          هند خرید نفت از خاورمیانه را قطع می‌کند

          ووشوکار نهاوندی به قتل رسید

          شانس پایین گاز ایران برای ورود به بازار اروپا

          خودروی امنیتی همراه با محافظ پهپادی

          آغاز صدور بیمه‌ آتش سوزی بازنشستگان

          جدیدترین تصاویر از قمر زحل

          سخاوت آمریکاییها به ایرانیان!

          جزئیات ورود قاچاق۳۰هزار کولرگازی«جنرال»

          چرا امروز روزنامه ایران 2بار منتشر شد؟

          کشف عکس واقعی «رئیسعلی دلواری»

          دانشور در آسیا طلایی شد

          فیلم/ روحانی‌در حرم‌امام راحل(ره)

          درخواست افزایش تعرفه کاغذ رد شد

          سرطان مثانه در کمین سیگاری‌ها

          آغاز آموزش حجاج از ماه رمضان

          این دانش‌آموزان ترک تحصیل می‌کنند+فیلم

          بزرگ‌ترین اشتباه وزیر راه و شهرسازی

          در شرق عربستان چه خبراست؟

          رئیس جمهور امروز به مشهد می‌رود

          مربی بدنساز بارسلونا در گسترش فولاد تبریز

          افزایش ۱۸۲ ریالی نرخ رسمی یورو+جدول

          شلوغ‌ترین‌وسایل‌حمل‌و‌نقل‌در‌جهان+تصاویر

          تصاویر زیبا ازآسمان پکن بعد از باران

          آشتی‌کنان علیرضا دبیر و عباس جدیدی

          روایت فجایع آمریکایی‌ها ضد سرخ‌پوستان

          ولایتی:باید شهریه‌ها را کاهش دهیم

          ادامه تحصن فداییان آیت‌الله عیسی قاسم

          گوشت در وضعیت قرمز

          صوت/ جدیدترین تلاوت سبزعلی

          توصیه‌های غذایی اسلام برای ماه رمضان

          روحانی دولت جوان تشکیل دهد

          عاقبت اصلاحاتی که طراحش آمریکاست

          تظاهرات‌صدها‌آمریکایی-اسرائیلی‌ضد‌ترامپ

          درآمد هدفمندی یارانه‌ها شفاف نیست

          عکس/ مهمان ناخوانده دیدار استقلال-العین

          رضایی بهترین بازیکن دیدار استقلال - العین

          پست دایی در سالروز درگذشت ناصرخان

          ولایت مطلقه فقیه در اندیشه رهبر انقلاب

          راهبردهای آمریکا در راستای تجزیه لیبی

          + +
          + +
          +
          + + + +
          +
           
          +
           
          +
          + + +
          آخرين اخبار
          +
          + +
          ۲۲:۱۴
          ۲۲:۱۲
          ۲۲:۱۱
          ۲۲:۱۱
          ۲۲:۰۹
          ۲۲:۰۰
          ۲۱:۵۸
          ۲۱:۵۷
          ۲۱:۵۴
          ۲۱:۴۸
          ۲۱:۴۸
          ۲۱:۴۷
          ۲۱:۴۷
          ۲۱:۴۶
          ۲۱:۴۰
          ۲۱:۳۸
          ۲۱:۳۶
          ۲۱:۳۵
          ۲۱:۳۳
          ۲۱:۳۲
          +
          + +
          +
          +
          +
          +

          منچستر

          حمله تروریستی در منچستر ۲۲ کشته برجا گذاشت/ تبلیغات انتخاباتی احزاب سیاسی متوقف شد/ داعش حمله منچستر را برعهده گرفت اما آمریکا تایید نمی‌کند

          وقوع انفجاری در محل برگزاری کنسرتی درشهر منچستر انگلیس، تا کنون ۲۲ کشته شامل چندین کودک و ۱۲۰ زخمی بر جای گذاشته است و در پی آن تبلیغات انتخاباتی احزاب سیاسی متوقف و دولت تشکیل جلسه فوری داد.

          پرسپولیس - لخویا

          مرحله یک هشتم نهایی لیگ قهرمانان آسیا

          توقف پرسپولیس بدون هوادار مقابل لخویا/ طارمی دوباره پنالتی خراب کرد!

          دیدار دو تیم پرسپولیس ایران و لخویا قطر در مرحله یک هشتم لیگ قهرمانان آسیا با تساوی بدون گل به پایان رسید.

          محمد صادق کوشکی

          کوشکی در گفت‌وگو با فارس:

          اعلام آرای تفکیکی ریاست جمهوری در این دولت مثل قراردادهای نفتی، سرنوشت کرسنت و قرارداد خرید هواپیماهای مرجوعی محرمانه است

          عضو هیأت علمی دانشگاه تهران گفت: عدم اعلام آراء تفکیکی ریاست جمهوری در این دولت مثل قراردادهای نفتی، سرنوشت کرسنت و قرارداد خرید هواپیماهای مرجوعی محرمانه است.

          سیدحسن حسینی‌شاهرودی

          عضو هیأت رئیسه کمیسیون اقتصادی مجلس:

          قرار بود تا پایان اردیبهشت‌ تکلیف کاسپین روشن‌ شود/ بانک مرکزی مقصر است

          عضو هیأت رئیسه کمیسیون اقتصادی مجلس گفت: وعده داده بودند تا پایان اردیبهشت‌‌ماه تکلیف کاسپین روشن‌ شود و بانک مرکزی از منظر مجلس مقصر است.

          وزارت کشور02

          فارس از اوضاع نابسامان شمارش آراء گزارش می‌دهد

          عملکرد مساله‌دار وزارت کشور و فرمانداری صدای اصلاح‌طلبان را هم در آورد/ سرخو: شمارش آرای من از قلم افتاده

          کاندیدای اصلاح طلب شورای شهر تهران طی نامه‌ای به فرماندار تهران خواستار بازشماری آرای ماخوذه با حضور نماینده این کاندیدا شده است.

          دلار، ارز، بازار سکه و ارز

          رشد قیمت ارزها در بازار/ دلار ۳۷۵۷ تومان+ جدول

          در بازار تهران امروز قیمت دلار 22 تومان، یورو 17 تومان، پوند 8 تومان و درهم 4 تومان افزایش یافت.

          روحانی

          روحانی به منظور زیارت بارگاه مطهر امام رضا(ع) وارد مشهد شد

          رئیس جمهور برای زیارت بارگاه مطهر هشتمین اختر تابناک آسمان ولایت و امامت حضرت علی بن موسی الرضا(ع) وارد مشهد مقدس شد.

          کیومرث عباسی قصری

          سی و ششمین «شب شاعر» برگزار شد

          پاسداشت برای «شاعرِ مردم»/ «قصر شیرین» استعاره‌ای برای شعر عباسی‌قصری

          سی و ششمین آیین «شب شاعر» پاسداشت کیومرث عباسی قصری در حالی برگزار شد که در این مراسم گفته شد که او شاعر مردم است.

          موسوی‌نژاد موسوی‌ نژاد

          موسوی‌نژاد در گفت‌وگو با فارس:

          عدم ارائه نتایج انتخابات به تفکیک استان‌ها یکی از تخلفات سازماندهی‌شده دولت در انتخابات است

          نماینده دوره نهم مجلس شورای اسلامی گفت: عدم ارائه نتایج انتخابات به تفکیک استان‌ها یکی از تخلفات سازماندهی‌شده دولت در بحث انتخابات است و اگر وزارت کشور مدعی است که انتخابات را به خوبی برگزار کرده، چرا از ارائه نتایج آمار به تفکیک استان‌ها استنکاف می‌ورزد.

          موسسه اعتباری کاسپین

          ظهر امروز صورت گرفت

          دومین اجتماع مالباختگان مؤسسه کاسپین در مشهد/ بانک مرکزی پاسخگو نیست

          دومین اجتماع مالباختگان مؤسسه مالی کاسپین امروز در حالی برگزار شد که سپرده‌گذاران از بی‌اعتنایی بانک مرکزی و عدم پاسخگویی مسؤولان به خواست خود گله‌مند هستند.

          علیرضا سلیمی نماینده مجلس دهم

          سلیمی در گفت‌وگو با فارس:

          وزارت کشور سریعاً آمار تفکیکی آرای انتخابات ریاست‌جمهوری در استان‌ها را اعلام کند/ لزوم جلوگیری از ایجاد شبهه در اذهان عمومی

          نماینده محلات و دلیجان در مجلس گفت: وزارت کشور سریعاً آمار تفکیکی آرای انتخابات ریاست‌جمهوری در استان‌ها را اعلام و از ایجاد شبهه در اذهان عمومی جلوگیری کند.

          قاسم خورشیدی سخنگوی ستاد مبارزه قاچاق

          سخنگوی ستاد مبارزه با قاچاق به فارس خبر داد

          جزئیات ورود قاچاق ۳۰ هزار کولرگازی «جنرال»

          سخنگوی ستاد مبارزه با قاچاق کالا و ارز جزئیات ورود ۳۰ هزار کولر گازی «جنرال» قاچاق از طریق گمرک را تشریح کرد.

          متین منتظمی

          منتظمی در گفت وگو با فارس:

          روحانی نمی‌تواند رقیب 16 میلیونی را نادیده بگیرد/بررسی کنند چرا در 8 استان از جمله زادگاه رئیس جمهور شکست خوردند؟

          دبیرکل اتحادیه جامعه اسلامی دانشجویان گفت: دولت دوازدهم بعد از انتخابات با 16 میلیون نفری مواجه شد که رئیسی را به عنوان نامزد خود انتخاب کردند به همین دلیل دیگر نمی‌تواند این رقیب قدرتمند را نادیده بگیرد و باید به مطالبات آنها پاسخ دهد.

          123انتخابات

          شبکه دانشگاهیان انقلاب اسلامی در بیانیه‌ای خطاب به مردم

          دولت قدر ملت را بداند و در خدمتگزاری به آنان ذره‌ای دریغ نکند

          شبکه دانشگاهیان انقلاب اسلامی در بیانیه‌ای خطاب به مردم ایران تاکید کرد: از دولت انتظار داریم حال که ملت عزیز ایران به تکلیف خود در عمل نمودند، قدر این ملت را بداند و در خدمتگزاری به آنان ذره‌ای دریغ نکند.

          بورس

          فارس از آن سوی تالار شیشه‌ای گزارش می‌دهد

          درجا زدن شاخص کل بورس روی 81 هزار و 124 واحد/ بازار سهام در بی‌رمقی کامل

          معامله‌گران بورس تهران امروز شاهد معاملاتی کم عمق و رخوت‌آور در کلیت معاملات بازار سهام بودند که فیلم‌های وسترن و شهرهای متروکه در غرب وحشی را به ذهن‌ها متبادر می‌کرد.

          فرهاد تجری

          تجری در پاسخ به فارس:

          نمایندگان حق دارند معترض روند نظارت انتخابات شوراها باشند/ برای رسیدگی به شکایت 1400 نفر فقط 24 ساعت فرصت داشتیم

          سخنگوی هیأت نظارت بر انتخابات شوراهای شهر و روستا درباره نامه ۵۲ نماینده در اعتراض به تخلفات این هیأت در تأیید صلاحیت‌ها گفت: هیأت نظارت نمی‌تواند بی‌نقص باشد و خلأ قانونی در این باره وجود دارد که باید هرچه سریع‌تر به آن رسیدگی شود.

          علی نیکزاد

          طی نامه‌ای به وزیر کشور صورت گرفت

          اعتراض رئیس ستاد «رئیسی» به عدم ارائه شفاف نتایج انتخابات به تفکیک صندوق‌ها

          علی نیکزاد رئیس ستاد انتخاباتی حجت‌الاسلام سیدابراهیم رئیسی در دوازدهمین دوره انتخابات ریاست جمهوری طی نامه‌ای به وزیر کشور از عدم ارائه شفاف نتایج به تفکیک صندوق‌های اخذ رأی انتقاد کرد.

          شعبه بانک

          فارس گزارش می‌دهد

          تعهد غیرمنصفانه‌ای که بانک مرکزی از سپرده‌گذاران ثامن‌الحجج گرفت+تصویر

          11 ماه از آغاز بازپرداخت سپرده‌های بالای 35 میلیون تومان مؤسسه ورشکسته ثامن الحجج می‌گذرد اما هیچ بازپرداختی نشده است و سپرده گذاران نیز با توجه به تعهد خود به بانک مرکزی، نمی‌توانند برای وصول طلب خود اعتراض و ادعا کنند.

          ارتش سوریه

          تداوم عملیات ارتش سوریه از قلمون تا تدمر؛ بیش از یک هزار کیلومتر آزاد شد

          ارتش سوریه اخیرا عملیات خود را از قلمون شرقی در ریف دمشق تا جنوب شهر تدمر واقع در ریف جنوب شرقی حمص ادامه داد و موفق شد مناطق زیادی را آزاد کند.

          +

          تمام اخبار برگزيده روز

          +
          + + + +
          +
          +
          +
          + + +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          +
          + + +
          +
          + +
          +
          + + +
          +
          + سایتهای دیگر +
          +
          +
          +
          + + + + +
          +
          + + + + + + + + + + + + + + diff --git a/test/testdata/fef78097af2366d041a66839250c48877990c79b.json b/test/testdata/fef78097af2366d041a66839250c48877990c79b.json new file mode 100644 index 00000000..bbe9f2ad --- /dev/null +++ b/test/testdata/fef78097af2366d041a66839250c48877990c79b.json @@ -0,0 +1,14 @@ +{ + "encoding": "utf-8", + "headers": { + "Cache-Control": "private", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Type": "text/html; charset=utf-8", + "Date": "Tue, 23 May 2017 18:06:45 GMT", + "Server": "nginx", + "Transfer-Encoding": "chunked" + }, + "status_code": 200, + "url": "http://www.farsnews.com/" +} \ No newline at end of file diff --git a/test/testdata/ff497ff23b56962fbe5dcc556dbe8cde49f9050b.html b/test/testdata/ff497ff23b56962fbe5dcc556dbe8cde49f9050b.html new file mode 100644 index 00000000..d850378b --- /dev/null +++ b/test/testdata/ff497ff23b56962fbe5dcc556dbe8cde49f9050b.html @@ -0,0 +1,94 @@ + + + + Glow worms in Wollemi National Park survived Gospers Mountain bushfire - ABC News + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

          Glow worms in Wollemi National Park survived Gospers Mountain bushfire

          ,
          Glow worms inside a tunnel
          The glow worms do not like direct light shone on them, or loud noises.(ABC News: Mridula Amin)
          Share

          It's like something out of a sci-fi movie — an ancient species of bug, glowing on the roof of an abandoned railway tunnel, deep in a remote forest.

          And locals could not be happier to see them.

          The colony of glow worms in the Wollemi National Park, just a couple of hours north-west of Sydney, was thought to have been destroyed when a bushfire ravaged the area last summer.

          The area has been off-limits due to bushfires and heavy rain before the coronavirus pandemic meant NSW's national parks were closed.

          Local tourism operator Kristie Kearney was among the first people to return to the tunnel and was relieved to find the creatures safe inside.

          "It's nature's Milky Way. It is a celestial experience," she said.

          A woman
          Kristie Kearney has grown up exploring the Wollemi National Park, and has visited the glow worms for decades.(ABC News: Mridula Amin)
          Glow worms
          The insects control their glowing mechanism using their nervous system.(ABC News: Mridula Amin)

          "It's as if you are looking out into the night sky, no moon, and all you see is just these millions of stars on the ceiling."

          Ms Kearney grew up in the area and watched in horror as the Gospers Mountain "mega-blaze" — Australia's largest ever bushfire from a single ignition point — raged through the park.

          "To be able to see them from one end of the tunnel to the other end was really quite incredible and it was something I hadn't seen in many, many years," she said.

          Thomas Ebersoll said he drove up to see if the tunnel had burned before the blaze blocked his access, eventually trapping him at his property.

          "You don't see fire, you just see smoke and the odd flare-up burning through that smoke, lighting up the smoke from inside.

          A man poses for a photo
          Thomas Ebersoll said he thought the iconic glow worms were lost when he saw smoke and flare-ups raging near the tunnel.(ABC News: Mridula Amin)
          A man outside a pub
          Thomas Ebersoll would drive up the valley to see if the fire had reached the glow worms.(ABC News: Mridula Amin)

          "I thought, they are gone, the glow worms are gone," he said.

          The University of Queensland's glow worm expert David Merritt said the railway tunnel, with a stream running through the old tracks, had provided the ideal refuge from the fires.

          Glow worms are unique to Australia and New Zealand and they date back millions of years.

          There are eight known species in Australia and only a handful of places with easy access to the habitats the insects like and where they can be easily seen, such as the Glow Worm Caves at the Gold Coast's Tambourine Mountain and Victoria's Melba Gully.

          A woman walks up some stairs
          The tunnel can be easily accessed after a 1-kilometre hike from the Glow Worm Tunnel car park.(ABC News: Mridula Amin)
          A woman stands at the ends of a tunnel
          The last train to run in the tunnel was in the 1930s.(ABC News: Mridula Amin)

          In Lithgow, you can hike one-and-a-half hours to witness the spectacle of "living light", or you can take an easy 1-kilometre walk from the Glow Worm Tunnel car park — neither require a guide.

          Dr Merritt says there is still a lot to learn about the creatures, which are, in fact, fly larvae.

          "It's a very unusual mechanism they use to glow, there aren't that many bioluminescent insects, " he said.

          A woman walks towards a tunnel
          The tunnel is 400 metres long and a stream runs through it, making it cool and slippery.(ABC News: Mridula Amin)
          ,
          Share

          Emergency Links

          Stay safe and informed with ABC's checklists & survival kits.

          Plan for BushfiresPlan for a Heatwave

          Just In

          + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/testdata/ff497ff23b56962fbe5dcc556dbe8cde49f9050b.json b/test/testdata/ff497ff23b56962fbe5dcc556dbe8cde49f9050b.json new file mode 100644 index 00000000..b54b2a17 --- /dev/null +++ b/test/testdata/ff497ff23b56962fbe5dcc556dbe8cde49f9050b.json @@ -0,0 +1,31 @@ +{ + "encoding": "utf-8", + "headers": { + "Access-Control-Allow-Origin": "http://nucwed.aus.aunty.abc.net.au", + "Application": "news-web", + "Branch": "master-news-web", + "Build": "39", + "Cache-Control": "public, max-age=57", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Length": "30208", + "Content-Security-Policy": "upgrade-insecure-requests;", + "Content-Type": "text/html; charset=utf-8", + "Date": "Fri, 25 Sep 2020 15:18:56 GMT", + "ETag": "W/\"2a5bb-qG3sUZ3xOrgYE8bmPUWOl5Xo2HE-gzip\"", + "Environment": "production", + "Expires": "Fri, 25 Sep 2020 15:19:53 GMT", + "Product": "presentation-layer", + "Referrer-Policy": "no-referrer-when-downgrade", + "Server": "Apache/2.4.46 (Unix)", + "Set-Cookie": "AWSALB=hSbkvCSVm/XwEtRKve5oXPvoAn1lx2FChKDAZxShSsHwiU9dIgKljbWxLEQrrr4GRPVFB5T3TI73BXfZNAQzpX3JpQYp424zH37bYq4eE/QzgbWfZ3tpiOnqRtWV; Expires=Fri, 02 Oct 2020 15:18:56 GMT; Path=/, AWSALBCORS=hSbkvCSVm/XwEtRKve5oXPvoAn1lx2FChKDAZxShSsHwiU9dIgKljbWxLEQrrr4GRPVFB5T3TI73BXfZNAQzpX3JpQYp424zH37bYq4eE/QzgbWfZ3tpiOnqRtWV; Expires=Fri, 02 Oct 2020 15:18:56 GMT; Path=/; SameSite=None; Secure, ABCGuestID=82.178.158.102.145361601047136653; expires=Mon, 31-Dec-2038 23:59:59 GMT; path=/; domain=.abc.net.au, ABC_LD=int; path=/; domain=.abc.net.au, ABC_FF=desktop; expires=Fri, 25-Sep-2020 17:18:56 GMT; path=/", + "Vary": "Accept-Encoding, Origin, Cookie, User-Agent, User-Agent", + "X-Content-Type-Options": "nosniff", + "X-DNS-Prefetch-Control": "off", + "X-Download-Options": "noopen", + "X-Frame-Options": "ALLOW-FROM http://nucwed.aus.aunty.abc.net.au", + "X-XSS-Protection": "1; mode=block" + }, + "status_code": 200, + "url": "https://www.abc.net.au/news/2020-09-06/glow-worms-in-wollemi-national-park-survived-summer-bushfire/12634762" +} \ No newline at end of file diff --git a/test/testdata/ff90691c7a4630b98c128002edc97c825324028f.html b/test/testdata/ff90691c7a4630b98c128002edc97c825324028f.html new file mode 100644 index 00000000..6035eff0 --- /dev/null +++ b/test/testdata/ff90691c7a4630b98c128002edc97c825324028f.html @@ -0,0 +1,438 @@ + + + + Wayback Machine + Internet Archive Wayback Machine + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + +
          + + + + +
          +
          +

          + The Wayback Machine is an initiative of the + Internet Archive, + a 501(c)(3) non-profit, building a digital library of + Internet sites and other cultural artifacts in digital form. +
          Other projects include + Open Library & + archive-it.org. +

          +

          + Your use of the Wayback Machine is subject to the Internet Archive's + Terms of Use. +

          +
          +
          + + diff --git a/test/testdata/ff90691c7a4630b98c128002edc97c825324028f.json b/test/testdata/ff90691c7a4630b98c128002edc97c825324028f.json new file mode 100644 index 00000000..839ebbc1 --- /dev/null +++ b/test/testdata/ff90691c7a4630b98c128002edc97c825324028f.json @@ -0,0 +1,16 @@ +{ + "encoding": "utf-8", + "headers": { + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Type": "text/html; charset=utf-8", + "Date": "Wed, 24 May 2017 04:23:49 GMT", + "Server": "Tengine/2.1.0", + "Transfer-Encoding": "chunked", + "X-Archive-Playback": "0", + "X-Page-Cache": "HIT", + "X-location": "All" + }, + "status_code": 200, + "url": "https://web.archive.org/" +} \ No newline at end of file diff --git a/test/urls_authors_test.py b/test/urls_authors_test.py index 55c6949a..a3ad2c8e 100644 --- a/test/urls_authors_test.py +++ b/test/urls_authors_test.py @@ -1,116 +1,157 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- +from pytest import mark +from regex import compile as regex_compile, VERBOSE, IGNORECASE -"""Test urls_authors.BYLINE_PATTERN.""" +from lib.urls_authors import byline_to_names, BYLINE_PATTERN, \ + BYLINE_TAG_FINDITER +from test.urls_test import urls_scr +BYLINE_PATTERN_REGEX = regex_compile( + fr'^{BYLINE_PATTERN}$', + IGNORECASE | VERBOSE) -from regex import compile as regex_compile, VERBOSE, IGNORECASE -from unittest import main, expectedFailure, TestCase -from lib.urls_authors import byline_to_names, BYLINE_PATTERN - -BYLINE_PATTERN_REGEX = regex_compile( - '^' + BYLINE_PATTERN + '$', - IGNORECASE | VERBOSE -) - - -class RegexTest(TestCase): - - """BYLINE_PATTERN should pass the following tests.""" - - def test_one_author(self): - """http://www.defense.gov/News/NewsArticle.aspx?ID=18509""" - text = 'By Jim Garamone' - self.assertRegex(text, BYLINE_PATTERN_REGEX) - - def test_cap_names_joined_by_and(self): - """Test two authors with and. - - Example: - https://www.eff.org/deeplinks/2014/06/ - sudan-tech-sanctions-harm-innovation-development-us-government-and- - corporations-must-act - - Note the two consecutive spaces. - - """ - text = 'By Kimberly Carlson and Jillian York' - self.assertRegex(text, BYLINE_PATTERN_REGEX) - - def test_four_authors(self): - """Test four authors, last one with and. - - http://arstechnica.com/science/2007/09/ - the-pseudoscience-behind-homeopathy/ - - """ - text = 'by John Timmer, Matt Ford, Chris Lee, and Jonathan Gitlin Sept' - self.assertRegex(text, BYLINE_PATTERN_REGEX) - - @expectedFailure - def test_four_authors_with_for(self): - """Test four authors, having a "for" at the end. - - http://arstechnica.com/science/2007/09/ - the-pseudoscience-behind-homeopathy/ - - """ - text = ( - 'By Sara Malm and Annette Witheridge and ' - 'Ian Drury for the Daily Mail and Daniel Bates' - ) - self.assertRegex(text, BYLINE_PATTERN_REGEX) - - -class BylineToNames(TestCase): - - """Test byline_to_names function.""" - - def test_two_author_seperated_by_comma(self): - byline = '\n By Roger Highfield, Science Editor \n' - names = byline_to_names(byline) - self.assertEqual(len(names), 1) - self.assertEqual(names[0][0], 'Roger') - - def test_in_in_byline(self): - byline = ( - ' By Erika Solomon in Beirut and Borzou Daragahi,' - ' Middle East correspondent' - ) - names = byline_to_names(byline) - self.assertEqual(len(names), 2) - self.assertEqual(names[0][0], 'Erika') - self.assertEqual(names[1][0], 'Borzou') - - def test_byline_ends_with_comma(self): - byline = 'by \n Tony Smith, \n' - names = byline_to_names(byline) - self.assertEqual(len(names), 1) - self.assertEqual(names[0][0], 'Tony') - - def test_semicolon_seperated_names_and_for(self): - byline = ( - 'Sara Malm;Annette Witheridge;Ian Drury for the Daily Mail;' - 'Daniel Bates' - ) - names = byline_to_names(byline) - self.assertEqual(len(names), 4) - self.assertEqual(names[2][0], 'Ian') - self.assertEqual(names[2][1], 'Drury') - - def test_newline_after_and(self): - byline = '\nIan Sample and \nStuart Clark in Darmstadt' - names = byline_to_names(byline) - self.assertEqual(len(names), 2) - self.assertEqual(names[1][1], 'Clark') - - def test_the_triggers_nofirst_fulllast(self): - # https://www.nytimes.com/2016/01/08/opinion/a-shameful-round-up-of-refugees.html?_r=0 - first, last = byline_to_names('THE EDITORIAL BOARD')[0] - self.assertEqual(first, '') - self.assertEqual(last, 'The Editorial Board') +def test_byline_pattern_one_author(): + """http://www.defense.gov/News/NewsArticle.aspx?ID=18509""" + assert BYLINE_PATTERN_REGEX.search('By Jim Garamone') -if __name__ == '__main__': - main() +def test_byline_pattern_cap_names_joined_by_and(): + """Test two authors with and. + + Example: + https://www.eff.org/deeplinks/2014/06/ + sudan-tech-sanctions-harm-innovation-development-us-government-and- + corporations-must-act + + Note the two consecutive spaces. + + """ + assert BYLINE_PATTERN_REGEX.search('By Kimberly Carlson and Jillian York') + + +def test_byline_pattern_four_authors(): + """Test four authors, last one with and. + + http://arstechnica.com/science/2007/09/ + the-pseudoscience-behind-homeopathy/ + + """ + assert BYLINE_PATTERN_REGEX.search( + 'by John Timmer, Matt Ford, Chris Lee, and Jonathan Gitlin Sept') + + +@mark.xfail +def test_byline_pattern_four_authors_with_for(): + """Test four authors, having a "for" at the end. + + http://arstechnica.com/science/2007/09/ + the-pseudoscience-behind-homeopathy/ + + """ + assert BYLINE_PATTERN_REGEX.search( + 'By Sara Malm and Annette Witheridge and ' + 'Ian Drury for the Daily Mail and Daniel Bates') + + +def test_byline_to_names_two_author_seperated_by_comma(): + names = byline_to_names('\n By Roger Highfield, Science Editor \n') + assert len(names) == 1 + assert names[0][0] == 'Roger' + + +def test_byline_to_names_in_in_byline(): + byline = ( + ' By Erika Solomon in Beirut and Borzou Daragahi,' + ' Middle East correspondent' + ) + names = byline_to_names(byline) + assert len(names) == 2 + assert names[0][0] == 'Erika' + assert names[1][0] == 'Borzou' + + +def test_byline_to_names_byline_ends_with_comma(): + names = byline_to_names('by \n Tony Smith, \n') + assert len(names) == 1 + assert names[0][0] == 'Tony' + + +def test_byline_to_names_semicolon_seperated_names_and_for(): + names = byline_to_names( + 'Sara Malm;Annette Witheridge;Ian Drury for the Daily Mail;' + 'Daniel Bates') + assert len(names) == 4 + assert names[2][0] == 'Ian' + assert names[2][1] == 'Drury' + + +def test_byline_to_names_newline_after_and(): + names = byline_to_names('\nIan Sample and \nStuart Clark in Darmstadt') + assert len(names) == 2 + assert names[1][1] == 'Clark' + + +def test_byline_to_names_schema_author(): + # https://www.abc.net.au/news/2020-09-06/glow-worms-in-wollemi-national-park-survived-summer-bushfire/12634762 + assert next(BYLINE_TAG_FINDITER( + ''))['result'] == 'Kathleen Ferguson' + + +def test_authors_meta_tag_with_no_quote(): # 28 + # + assert ( + "{{cite web | last=Truitt | first=Brian " + "| title='Star Wars': Disney+ switches up controversial " + "Han Solo/Greedo scene | website=USA TODAY | date=2019-11-12 " + "| url=https://www.usatoday.com/story/entertainment/movies/2019/11/12/star-wars-disney-plus-changes-controversial-han-solo-greedo-scene/2576097001/ " + "| access-date=" + ) == urls_scr( + 'https://www.usatoday.com/story/entertainment/movies/2019/11/12/star-wars-disney-plus-changes-controversial-han-solo-greedo-scene/2576097001/' + )[1][2:-12] + + +def test_uppercase_sitename_in_authors(): # 28 + # note: must use the specific testdata stored at + # https://gist.github.com/5j9/ec831edb740363191e21c4f500cb9a09#file-usatoday_toolforge-html-L86 + assert ( + "{{cite web | last=Truitt | first=Brian " + "| title=The infamous 'Han shot first' scene in 'Star Wars' has changed yet again on Disney+ " + "| website=USA TODAY | date=2019-11-12 " + "| url=https://www.usatoday.com/story/entertainment/movies/2019/11/12/2576097001/ " + "| access-date=" + ) == urls_scr( + 'https://www.usatoday.com/story/entertainment/movies/2019/11/12/2576097001/' + )[1][2:-12] + + +def test_byline_ending_with_semicolon(): + # https://pubmed.ncbi.nlm.nih.gov/32687126/ + # + assert byline_to_names( + 'Ojewola RW;Tijani KH;Fatuga AL;Onyeze CI;Okeke CJ;' + ) == [ # it used to raise error however first are last are still swapped + ('Ojewola', 'RW'), + ('Tijani', 'KH'), + ('Fatuga', 'AL'), + ('Onyeze', 'CI'), + ('Okeke', 'CJ') + ] diff --git a/test/urls_test.py b/test/urls_test.py index 9b714c3f..e4b54b9b 100644 --- a/test/urls_test.py +++ b/test/urls_test.py @@ -1,995 +1,989 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- - -"""Test urls.py module.""" - - -from unittest import main, TestCase, skip - -from lib.urls import urls_sfn_cit_ref - - -class BostonTest(TestCase): - - def test_bg1(self): - """boston.com, dateformat '%B %d, %Y'""" - self.assertIn( - '* {{cite web ' - '| last=Griffith ' - '| first=Bill ' - '| title=Hot Rod Stamps; Google on Road; A GM Prospectus ' - '| website=Boston.com ' - '| date=June 29, 2014 ' - '| url=https://www.boston.com/cars/news-and-reviews/2014/06/29/' - 'hot-rod-stamps-google-on-road-a-gm-prospectus ' - '| ref=harv ' - '| access-date=', - urls_sfn_cit_ref( - 'http://www.boston.com/cars/news-and-reviews/2014/06/28/' - 'hot-rod-stamps-google-road-prospectus/hylbVi9qonAwBIH10CwiDP/' - 'story.html', - '%B %d, %Y', - )[1] - ) - - def test_bg2(self): - """bostonglobe.com""" - i = ( - 'http://www.bostonglobe.com/metro/2014/06/03/' - 'walsh-meets-with-college-leaders-off-campus-housing/' - 'lsxtLSGJMD86Gbkjay3D6J/story.html' - ) - o = urls_sfn_cit_ref(i) - ct = ( - '* {{cite web ' - '| last=Saltzman ' - '| first=Jonathan ' - '| last2=Farragher ' - '| first2=Thomas ' - '| title=Walsh meets with college leaders on off-campus housing ' - '| website=BostonGlobe.com ' - '| date=2014-06-03 ' - '| url=https://www.bostonglobe.com/metro/2014/06/03/' - 'walsh-meets-with-college-leaders-off-campus-housing/' - 'lsxtLSGJMD86Gbkjay3D6J/story.html ' - '| ref=harv ' - '| access-date=' - ) - self.assertIn(ct, o[1]) - - def test_bg3(self): - """bostonmagazine.com. Author tags return unrelated authors.""" - i = ( - 'http://www.bostonmagazine.com/news/blog/2013/08/21/' - 'juliette-kayyem-jumps-in-for-guv/' - ) - o = urls_sfn_cit_ref(i) - ct = ( - '* {{cite web ' - '| last=Bernstein ' - '| first=David S. ' - '| title=Juliette Kayyem Is Running for Governor of Massachusetts ' - '| website=Boston Magazine ' - '| date=2013-08-21 ' - '| url=http://www.bostonmagazine.com/news/blog/2013/08/21/' - 'juliette-kayyem-jumps-in-for-guv/ ' - '| ref=harv ' - '| access-date=' - ) - self.assertIn(ct, o[1]) - - -class WashingtonpostTest(TestCase): - - def test_wp1(self): - """`1 author, 2005, the pubdate is different from last edit date""" - o = urls_sfn_cit_ref( - 'http://www.washingtonpost.com/wp-dyn/content/article/2005/09/02/' - 'AR2005090200822.html' - ) - self.assertIn('{{sfn | Sachs | 2005}}', o[0]) - self.assertIn( - '* {{cite web ' - '| last=Sachs ' - '| first=Andrea ' - '| title=March of the Migration ' - '| website=Washington Post ' - '| date=2005-09-04 ' - '| url=http://www.washingtonpost.com/wp-dyn/content/article/' - '2005/09/02/AR2005090200822.html ' - '| ref=harv ' - '| access-date=', - o[1], - ) - - -class HuffingtonpostTest(TestCase): - - def test_hp1(self): - """`1 author, 2013""" - o = urls_sfn_cit_ref( - 'http://www.huffingtonpost.ca/annelise-sorg/' - 'blackfish-killer-whale-seaworld_b_3686306.html' - ) - self.assertEqual('{{sfn | Sorg | 2013}}', o[0]) - self.assertIn( - '* {{cite web ' - '| last=Sorg ' - '| first=Annelise ' - '| title=When Killer Whales Kill: Why the movie' - ' "Blackfish" Should Sink Captive Whale Programs ' - '| website=The Huffington Post ' - '| date=2013-08-01 ' - '| url=http://www.huffingtonpost.ca/annelise-sorg/' - 'blackfish-killer-whale-seaworld_b_3686306.html ' - '| ref=harv ' - '| access-date=', - o[1], - ) - - def test_hp2(self): - """`class:author` returns wrong result. Disallow `\n` in fullnames.""" - i = ( - 'http://www.huffingtonpost.com/jeremy-rifkin/' - 'obamas-climate-change-plan_b_5427656.html' - ) - o = urls_sfn_cit_ref(i) - e2 = ( - "* {{cite web " - "| last=Rifkin " - "| first=Jeremy " - "| title=Beyond Obama's Plan: " - "A New Economic Vision for Addressing Climate Change " - "| website=The Huffington Post " - "| date=2014-06-02 " - "| url=http://www.huffingtonpost.com/jeremy-rifkin/" - "obamas-climate-change-plan_b_5427656.html " - "| ref=harv " - "| access-date=" - ) - self.assertEqual('{{sfn | Rifkin | 2014}}', o[0]) - self.assertIn(e2, o[1]) - - -class DilyTelegraphTest(TestCase): - - def test_dt1(self): - """`1 author, 2005""" - i = ( - 'http://www.telegraph.co.uk/news/health/3334755/' - 'We-could-see-the-whales-eyes-mouth...-' - 'the-barnacles-on-its-back.html' - ) - o = urls_sfn_cit_ref(i) - e2 = ( - "* {{cite web " - "| last=Fogle " - "| first=Ben " - "| title=We could see the whale's eyes, mouth... " - "the barnacles on its back " - "| website=Telegraph.co.uk " - "| date=2005-12-22 " - "| url=http://www.telegraph.co.uk/news/health/3334755/" - "We-could-see-the-whales-eyes-mouth...-" - "the-barnacles-on-its-back.html " - "| ref=harv " - "| access-date=" - ) - self.assertEqual('{{sfn | Fogle | 2005}}', o[0]) - self.assertIn(e2, o[1]) - - def test_dt2(self): - """1 author, 2003""" - i = ( - 'http://www.telegraph.co.uk/news/science/science-news/3313298/' - 'Marine-collapse-linked-to-whale-decline.html' - ) - o = urls_sfn_cit_ref(i) - e2 = ( - "* {{cite web " - "| last=Highfield " - "| first=Roger " - "| title=Marine 'collapse' linked to whale decline " - "| website=Telegraph.co.uk " - "| date=2003-09-29 " - "| url=http://www.telegraph.co.uk/news/science/science-news/" - "3313298/Marine-collapse-linked-to-whale-decline.html " - "| ref=harv " - "| access-date=" - ) - self.assertEqual('{{sfn | Highfield | 2003}}', o[0]) - self.assertIn(e2, o[1]) - - def test_dt3(self): - """1 author, 2011""" - i = ( - 'http://www.telegraph.co.uk/news/8323909/' - 'The-sperm-whale-works-in-extraordinary-ways.html' - ) - o = urls_sfn_cit_ref(i) - e2 = ( - "* {{cite web " - "| last=Whitehead " - "| first=Hal " - "| title=The sperm whale works in extraordinary ways " - "| website=Telegraph.co.uk " - "| date=2011-02-15 " - "| url=http://www.telegraph.co.uk/news/science/8323909/" - "The-sperm-whale-works-in-extraordinary-ways.html " - "| ref=harv " - "| access-date=" - ) - self.assertEqual('{{sfn | Whitehead | 2011}}', o[0]) - self.assertIn(e2, o[1]) - - -class DilyMailTest(TestCase): - - def test_dm1(self): - """4 authors""" - o = urls_sfn_cit_ref( - 'http://www.dailymail.co.uk/news/article-2633025/' - 'London-cleric-convicted-NYC-terrorism-trial.html' - ) - self.assertEqual( - '{{sfn | Malm | Witheridge | Drury | Bates | 2014}}', o[0] - ) - self.assertIn( - '* {{cite web ' - '| last=Malm ' - '| first=Sara ' - '| last2=Witheridge ' - '| first2=Annette ' - '| last3=Drury ' - '| first3=Ian ' - '| last4=Bates ' - '| first4=Daniel ' - '| title=Abu Hamza found guilty in US court of helping' - ' Al-Qaeda terrorists ' - '| website=Daily Mail Online ' - '| date=2014-05-19 ' - '| url=http://www.dailymail.co.uk/news/article-2633025/' - 'London-cleric-convicted-NYC-terrorism-trial.html ' - '| ref=harv ' - '| access-date=', - o[1], - ) - - def test_dm2(self): - """`for` in byline.""" - self.assertIn( - '* {{cite web ' - '| last=Gower ' - '| first=Eleanor ' - "| title=Kim Kardashian's meltdown at nude magazine cover" - " three years before full frontal photoshoot " - '| website=Daily Mail Online ' - '| date=2014-11-14 ' - '| url=http://www.dailymail.co.uk/tvshowbiz/article-2834145/' - 'I-m-never-taking-clothes-s-Vogue-Throwback-2011-video-shows-Kim-' - 'Kardashian-s-meltdown-nude-magazine-cover.html ' - '| ref=harv ' - '| access-date=', - urls_sfn_cit_ref( - 'http://www.dailymail.co.uk/tvshowbiz/article-2834145/' - 'I-m-never-taking-clothes-s-Vogue-Throwback-2011-video-' - 'shows-Kim-Kardashian-s-meltdown-nude-magazine-cover.html' - )[1], - ) - - -class BbcTest(TestCase): - - def test_bbc1(self): - """no authors""" - i = 'https://www.bbc.com/news/world-asia-27653361' - o = urls_sfn_cit_ref(i) - ct = ( - "* {{cite web " - "| title=US 'received Qatar assurances' on Afghan prisoner deal " - "| website=BBC News " - "| date=2014-06-01 " - "| url=http://www.bbc.com/news/world-asia-27653361 " - "| ref={{sfnref | BBC News | 2014}} " - "| access-date=" - ) - self.assertIn(ct, o[1]) - - def test_bbc2(self): - """1 author""" - self.assertIn( - '* {{cite web ' - '| last=Gage ' - '| first=Suzi ' - '| title=Sea otter return boosts ailing seagrass in California ' - '| website=BBC News ' - '| date=2013-08-26 ' - '| url=http://www.bbc.com/news/science-environment-23814524 ' - '| ref=harv ' - '| access-date=', - urls_sfn_cit_ref( - 'http://www.bbc.com/news/science-environment-23814524' - )[1], - ) - - def test_bbc3(self): - """https version of bbc2 (differs a lot!)""" - i = 'https://www.bbc.com/news/science-environment-23814524' - o = urls_sfn_cit_ref(i) - ct = ( - '* {{cite web ' - '| last=Gage ' - '| first=Suzi ' - '| title=Sea otter return boosts ailing seagrass in California ' - '| website=BBC News ' - '| date=2013-08-26 ' - '| url=http://www.bbc.com/news/science-environment-23814524 ' - '| ref=harv ' - '| access-date=' - ) - self.assertIn(ct, o[1]) - - def test_bbc4(self): - """news.bbc.co.uk, 1 author""" - self.assertIn( - "* {{cite web " - "| last=Jones " - "| first=Meirion " - "| title=Malaria advice 'risks lives' " - "| website=BBC NEWS " - "| date=2006-07-13 " - "| url=" - "http://news.bbc.co.uk/2/hi/programmes/newsnight/5178122.stm " - "| ref=harv " - "| access-date=", - urls_sfn_cit_ref( - 'http://news.bbc.co.uk/2/hi/programmes/newsnight/5178122.stm' - )[1], - ) - - def test_bbc5(self): - """news.bbc.co.uk, 1 author""" - self.assertIn( - "* {{cite web " - "| last=Madslien " - "| first=Jorn " - "| title=Inside the Bentley factory " - "| website=BBC NEWS " - "| date=2002-12-24 " - "| url=http://news.bbc.co.uk/2/hi/business/2570109.stm " - "| ref=harv " - "| access-date=", - urls_sfn_cit_ref( - 'http://news.bbc.co.uk/2/hi/business/2570109.stm' - )[1], - ) - - def test_bbc6(self): - """bbc.com, 1 author""" - i = 'http://www.bbc.com/news/science-environment-26267918' - o = urls_sfn_cit_ref(i) - ct = ( - "* {{cite web " - "| last=Amos " - "| first=Jonathan " - "| title=European Space Agency picks Plato planet-hunting mission " - "| website=BBC News " - "| date=2014-02-20 " - "| url=http://www.bbc.com/news/science-environment-26267918 " - "| ref=harv " - "| access-date=" - ) - self.assertIn(ct, o[1]) - - -class NytTest(TestCase): - - def test_nyt1(self): - """newstylct, 1 author""" - i = ( - 'http://www.nytimes.com/2014/05/30/business/international/' - 'on-the-internet-the-right-to-forget-vs-the-right-to-know.html?' - 'hp&_r=0' - ) - o = urls_sfn_cit_ref(i) - ct = ( - '* {{cite web ' - '| last=Hakim ' - '| first=Danny ' - '| title=Right to Be Forgotten? Not That Easy ' - '| website=The New York Times ' - '| date=2014-05-30 ' - '| url=https://www.nytimes.com/2014/05/30/business/international/' - 'on-the-internet-the-right-to-forget-vs-the-right-to-know.html ' - '| ref=harv ' - '| access-date=' - ) - self.assertIn(ct, o[1]) - - def test_nyt2(self): - """newstylct, 2 authors""" - ct = ( - '* {{cite web ' - '| last=Belson ' - '| first=Ken ' - '| last2=Sandomir ' - '| first2=Richard ' - '| title=$2 Billion for Clippers? In Time, ' - 'It May Be a Steal for Steve Ballmer ' - '| website=The New York Times ' - '| date=2014-05-30 ' - '| url=https://www.nytimes.com/2014/05/31/sports/basketball/' - 'steven-a-ballmers-2-billion-play-for-clippers-is-a-big-bet-on-' - 'the-nba.html ' - '| ref=harv ' - '| access-date=' - ) - self.assertIn( - ct, - urls_sfn_cit_ref( - 'https://www.nytimes.com/2014/05/31/sports/basketball/' - 'steven-a-ballmers-2-billion-play-for-clippers-is-a-big-bet-' - 'on-the-nba.html?hp' - )[1], - ) - - def test_nyt3(self): - """oldstylct, 1 author""" - i = 'http://www.nytimes.com/2007/12/25/world/africa/25kenya.html' - o = urls_sfn_cit_ref(i) - ct = ( - '* {{cite web ' - '| last=Gettleman ' - '| first=Jeffrey ' - '| title=Election Rules Complicate Kenya Race ' - '| website=The New York Times ' - '| date=2007-12-25 ' - '| url=https://www.nytimes.com/2007/12/25/world/africa/' - '25kenya.html ' - '| ref=harv ' - '| access-date=' - ) - self.assertIn(ct, o[1]) - - def test_nyt4(self): - """newstylct, 2 authors, only byline""" - i = ( - 'http://dealbook.nytimes.com/2014/05/30/' - 'insider-trading-inquiry-includes-mickelson-and-icahn/' - ) - o = urls_sfn_cit_ref(i) - ct = ( - '* {{cite web ' - '| last=Goldstein ' - '| first=Matthew ' - '| last2=Protess ' - '| first2=Ben ' - '| title=Investor, Bettor, Golfer: ' - 'Insider Trading Inquiry Includes Mickelson, Icahn and William T. ' - 'Walters ' - '| website=DealBook ' - '| date=2014-06-12 ' - '| url=https://dealbook.nytimes.com/2014/05/30/' - 'insider-trading-inquiry-includes-mickelson-and-icahn/ ' - '| ref=harv ' - '| access-date=' - ) - self.assertIn(ct, o[1]) - - def test_nyt5(self): - """special case for date format (not in usual meta tags)""" - i = ( - 'https://www.nytimes.com/2007/06/13/world/americas/' - '13iht-whale.1.6123654.html' - ) - o = urls_sfn_cit_ref(i) - ct = ( - '* {{cite web ' - '| title=19th-century harpoon gives clue on whales ' - '| website=The New York Times ' - '| date=2007-06-13 ' - '| url=https://www.nytimes.com/2007/06/13/world/americas/' - '13iht-whale.1.6123654.html ' - '| ref={{sfnref | The New York Times | 2007}} ' - '| access-date=' - ) - self.assertIn(ct, o[1]) - - def test_nyt6(self): - """lastname=O'Connor""" - i = ( - 'http://www.nytimes.com/2003/10/09/us/' - 'adding-weight-to-suspicion-sonar-is-linked-to-whale-deaths.html' - ) - o = urls_sfn_cit_ref(i) - ct = ( - "* {{cite web " - "| last=O'Connor " - "| first=Anahad " - "| title=Adding Weight to Suspicion, " - "Sonar Is Linked to Whale Deaths " - "| website=The New York Times " - "| date=2003-10-09 " - "| url=https://www.nytimes.com/2003/10/09/us/" - "adding-weight-to-suspicion-sonar-is-linked-to-whale-deaths.html " - "| ref=harv " - "| access-date=" - ) - self.assertIn(ct, o[1]) - - -class TGDaily(TestCase): - - def test_tgd2(self): - """Hard to find author and date.""" - i = ( - 'http://www.tgdaily.com/web/' - '100381-apple-might-buy-beats-for-32-billion' - ) - o = urls_sfn_cit_ref(i) - ct = ( - '* {{cite web ' - '| title=Apple might buy Beats for $3.2 billion ' - '| website=TG Daily ' - '| date=2014-05-09 ' - '| url=http://www.tgdaily.com/web/' - '100381-apple-might-buy-beats-for-32-billion ' - '| ref={{sfnref | TG Daily | 2014}} ' - '| access-date=' - ) - self.assertIn(ct, o[1]) - - def test_tgd3(self): - """"Staff" in author name.""" - i = ( - 'http://www.tgdaily.com/space-features/' - '82906-sma-reveals-giant-star-cluster-in-the-making' - ) - o = urls_sfn_cit_ref(i) - ct = ( - '* {{cite web ' - '| title=SMA reveals giant star cluster in the making ' - '| website=TG Daily ' - '| date=2013-12-17 ' - '| url=http://www.tgdaily.com/space-features/' - '82906-sma-reveals-giant-star-cluster-in-the-making ' - '| ref={{sfnref | TG Daily | 2013}} ' - '| access-date=' - ) - self.assertIn(ct, o[1]) - - -@skip -class NotWorking(TestCase): - def test_tgd1(self): - """ABCNews. Wrong author: | last=News | first=ABC.""" - i = 'http://abcnews.go.com/blogs/headlines/2006/12/saddam_executed/' - o = urls_sfn_cit_ref(i) - ct = ( - '* {{cite web ' - '| last=Ross ' - '| first=Brian ' - '| title=Saddam Executed; An Era Comes to an End ' - '| website=ABC News Blogs ' - '| date=2006-12-30 ' - '| url=http://abcnews.go.com/blogs/headlines/2006/12/' - 'saddam_executed/ ' - '| ref=harv ' - '| access-date=' - ) - self.assertIn(ct, o[1]) - - def test_oth12(self): - """Times of India, author could not be detected.""" - i = ( - 'http://timesofindia.indiatimes.com/city/pune/' - 'UK-allows-working-visas-for-Indian-students/' - 'articleshow/1163528927.cms?' - ) - o = urls_sfn_cit_ref(i) - sfn = "{{sfn | Kashyap | 2001}}" - self.assertIn(sfn, o[0]) - - -class Others(TestCase): - - def test_oth1(self): - """Get title by hometitle comparison.""" - i = 'http://www.ensani.ir/fa/content/326173/default.aspx' - o = urls_sfn_cit_ref(i) - ct = ( - '* {{cite web ' - '| last=جلیلیان ' - '| first=شهرام ' - '| last2=نیا ' - '| first2=امیر علی ' - '| title=ورود کاسی ها به میان رودان و پیامدهای آن ' - '| website=پرتال جامع علوم انسانی ' - '| date=2014-05-20 ' - '| url=http://www.ensani.ir/fa/content/326173/default.aspx ' - '| language=fa ' - '| ref=harv ' - '| access-date=' - ) - self.assertIn(ct, o[1]) - - def test_text_search(self): - """Match byline on soup.text.""" - self.assertIn( - '* {{cite web ' - '| last=Carlson ' - '| first=Kimberly ' - '| last2=York ' - '| first2=Jillian C. ' - '| title=Sudan Tech Sanctions Harm Innovation and Development: ' - 'US Government and Corporations Must Act ' - '| website=Electronic Frontier Foundation ' - '| date=2014-06-26 ' - '| url=https://www.eff.org/deeplinks/2014/06/' - 'sudan-tech-sanctions-harm-innovation-development-us-' - 'government-and-corporations-must-act ' - '| ref=harv ' - '| access-date=', - urls_sfn_cit_ref( - 'https://www.eff.org/deeplinks/2014/06/' - 'sudan-tech-sanctions-harm-innovation-development-us-' - 'government-and-corporations-must-act' - )[1], - ) - - # Disable because relies on class="author" which has been disabled due - # to hight error rate. - @skip - def test_oth3(self): - """4 authors.""" - i = ( - 'https://arstechnica.com/science/2007/09/' - 'the-pseudoscience-behind-homeopathy/' - ) - o = urls_sfn_cit_ref(i) - ct = ( - '* {{cite web ' - '| last=Timmer ' - '| first=John ' - '| last2=Ford ' - '| first2=Matt ' - '| last3=Lee ' - '| first3=Chris ' - '| last4=Gitlin ' - '| first4=Jonathan ' - '| title=Diluting the scientific method: Ars looks at homeopathy ' - '| website=Ars Technica ' - '| date=2007-09-12 ' - '| url=https://arstechnica.com/science/2007/09/' - 'the-pseudoscience-behind-homeopathy/ ' - '| ref=harv ' - '| access-date=' - ) - self.assertIn(ct, o[1]) - - def test_oth4(self): - """rel="author" tag contains invalid information.""" - self.assertIn( - "* {{cite web " - "| last=Ghose " - "| first=Tia " - "| title='Revolutionary' Physics:" - " Do Sterile Neutrinos Lurk in the Universe? " - "| website=Live Science " - "| date=2014-07-01 " - "| url=http://www.livescience.com/" - "46619-sterile-neutrino-experiment-beginning.html " - "| ref=harv " - "| access-date=", - urls_sfn_cit_ref( - 'http://www.livescience.com/' - '46619-sterile-neutrino-experiment-beginning.html?' - 'cmpid=514645_20140702_27078936' - )[1], - ) - - def test_oth5(self): - """Getting the date is tricky here.""" - o = urls_sfn_cit_ref('http://www.magiran.com/npview.asp?ID=1410487') - self.assertIn('{{sfn | نوري | 2007}}', o[0]) - self.assertIn( - '* {{cite web ' - '| last=نوري ' - '| first=آزاده شهمير ' - '| title=روزنامه سرمايه86/3/1: دكتر طاهر صباحي، محقق و مجموعه دار' - ' فرش: بازار جهاني با توليد فرش هنري نصيب ايران مي شود ' - '| website=magiran.com ' - '| date=2007-05-22 ' - '| url=http://www.magiran.com/npview.asp?ID=1410487 ' - '| language=fa ' - '| ref=harv ' - '| access-date=', - o[1], - ) - - def test_oth6(self): - """Detection of website name.""" - o = urls_sfn_cit_ref( - 'http://www.farsnews.com/newstext.php?nn=13930418000036' - ) - self.assertIn("{{sfn | ''خبرگزاری فارس'' | 2014}}", o[0]) - # Fars news is using 'خبرگزاری فارس' as og:author which is wrong - # and thats why its name is not italicized in sfn. - self.assertIn( - '* {{cite web ' - '| title=آیت\u200cالله محمدی گیلانی دارفانی را وداع گفت ' - '| website=خبرگزاری فارس ' - '| date=2014-07-09 ' - '| url=http://www.farsnews.com/newstext.php?nn=13930418000036 ' - '| language=fa ' - '| ref={{sfnref | خبرگزاری فارس | 2014}} ' - '| access-date=', - o[1], - ) - - def test_oth7(self): - """Contains a By Topic line and also the byline contains ' | '.""" - self.assertIn( - '* {{cite web ' - '| last=Chandler ' - '| first=David L. ' - '| title=Traffic lights: There’s a better way ' - '| website=MIT News ' - '| date=2014-07-07 ' - '| url=http://news.mit.edu/2014/' - 'traffic-lights-theres-a-better-way-0707 ' - '| ref=harv ' - '| access-date=', - urls_sfn_cit_ref( - 'http://news.mit.edu/2014/' - 'traffic-lights-theres-a-better-way-0707' - )[1], - ) - - def test_oth8(self): - """Two authors from guardian that are mentions in other tags, too.""" - i = ( - 'http://www.theguardian.com/world/2014/jul/14/' - 'israel-drone-launched-gaza-ashdod' - ) - o = urls_sfn_cit_ref(i) - ct = ( - '* {{cite web ' - '| last=Beaumont ' - '| first=Peter ' - '| last2=Crowcroft ' - '| first2=Orlando ' - '| title=Israel says it has shot down drone launched from Gaza ' - '| website=the Guardian ' - '| date=2014-07-14 ' - '| url=http://www.theguardian.com/world/2014/jul/14/' - 'israel-drone-launched-gaza-ashdod ' - '| ref=harv ' - '| access-date=' - ) - self.assertIn(ct, o[1]) - - def test_oth10(self): - """The Times. (Authors found by "byline" css selector)""" - self.assertIn( - '* {{cite web ' - '| last=Lagan ' - '| first=Bernard ' - '| last2=Charter ' - '| first2=David ' - '| title=' - 'Woman who lost brother on MH370 mourns relatives on board MH17 ' - '| website=The Times & The Sunday Times ' - '| date=2014-07-18 ' - '| url=https://www.thetimes.co.uk/article/' - 'woman-who-lost-brother-on-mh370-mourns-relatives-on-board-' - 'mh17-r07q5rwppl0 ' - '| ref=harv ' - '| access-date=', - urls_sfn_cit_ref( - 'http://www.thetimes.co.uk/tto/news/world/' - 'australia-newzealand/article4151214.ece' - )[1] - ) - - def test_oth11(self): - """Business News Daily.""" - i = ( - 'http://www.businessnewsdaily.com/6762-male-female-entrepreneurs' - '.html?cmpid=514642_20140715_27858876' - ) - o = urls_sfn_cit_ref(i) - ct = ( - '* {{cite web ' - '| last=Helmrich ' - '| first=Brittney ' - '| title=Male vs. Female Entrepreneurs: How Are They Different? ' - '| website=Business News Daily ' - '| date=2014-07-10 ' - '| url=http://www.businessnewsdaily.com/6762-male-female-' - 'entrepreneurs.html ' - '| ref=harv ' - '| access-date=' - ) - self.assertIn(ct, o[1]) - - def test_oth12(self): - """thebulletin.org""" - i = 'http://thebulletin.org/evidence-shows-iron-dome-not-working7318' - o = urls_sfn_cit_ref(i) - ct = ( - '* {{cite web ' - '| last=Postol ' - '| first=Theodore A. ' - '| title=The evidence that shows Iron Dome is not working ' - '| website=Bulletin of the Atomic Scientists ' - '| date=2014-07-19 ' - '| url=http://thebulletin.org/' - 'evidence-shows-iron-dome-not-working7318 ' - '| ref=harv ' - '| access-date=' - ) - self.assertIn(ct, o[1]) - - def test_reverse_name(self): - """Author is `Martin, Tracy`. Tracy should be the first name.""" - self.assertIn( - '* {{cite web ' - '| last=Martin ' - '| first=Tracy ' - '| title=Dynamometers Explained ' - '| website=HighBeam Research ' - '| date=2014-07-01 ' - '| url=http://www.highbeam.com/doc/1P3-3372742961.html ' - '| ref=harv ' - '| access-date=', - urls_sfn_cit_ref( - 'http://www.highbeam.com/doc/1P3-3372742961.html' - )[1], - ) - - def test_oth14(self): - """thebulletin.org""" - i = ( - 'http://www.independent.co.uk/news/business/' - 'the-investment-column-tt-group-1103208.html' - ) - o = urls_sfn_cit_ref(i) - ct = ( - '* {{cite web ' - '| title=The Investment column: TT Group ' - '| website=The Independent ' - '| date=1999-06-29 ' - '| url=http://www.independent.co.uk/news/business/' - 'the-investment-column-tt-group-1103208.html ' - '| ref={{sfnref | The Independent | 1999}} ' - '| access-date=' - ) - self.assertIn(ct, o[1]) - - def test_oth15(self): - """Contains """ - self.assertIn( - '* {{cite web ' - '| title=برجام شرایط بین‌المللی ایران را کاملا متحول کرد ' - '| website=ایسنا ' - '| date=2017-01-25 ' - '| url=http://www.isna.ir/news/95110603890/ ' - '| language=fa ' - '| ref={{sfnref | ایسنا | 2017}} ' - '| access-date=', - urls_sfn_cit_ref( - 'http://www.isna.ir/news/95110603890/' - '%D8%A8%D8%B1%D8%AC%D8%A7%D9%85-%D8%B4%D8%B1%D8%A7%DB%8C%D8%B7' - '-%D8%A8%DB%8C%D9%86-%D8%A7%D9%84%D9%85%D9%84%D9%84%DB%8C-' - '%D8%A7%DB%8C%D8%B1%D8%A7%D9%86-%D8%B1%D8%A7-' - '%DA%A9%D8%A7%D9%85%D9%84%D8%A7-%D9%85%D8%AA%D8%AD%D9%88%D9%84' - '-%DA%A9%D8%B1%D8%AF' - )[1], - ) - - def test_invalid_name(self): - """Test that URL does not fail with InvalidNameError.""" - self.assertIn( - '* {{cite web | title=انتخابات 96 به روایت آمار ' - '| website=پایگاه اطلاع رسانی شبکه خبر صدا' - ' و سیمای جمهوری اسلامی ایران |' - ' date=2017-05-24 | url=http://www.irinn.ir/fa/news/499654 ' - '| language=fa | ref={{sfnref |' - ' پایگاه اطلاع رسانی شبکه خبر' - ' صدا و سیمای جمهوری اسلامی ایران | 2017}} |' - ' access-date=', - urls_sfn_cit_ref( - 'http://www.irinn.ir/fa/news/499654/' - '%D8%A7%D9%86%D8%AA%D8%AE%D8%A7%D8%A8%D8%A7%D8%AA-96-' - '%D8%A8%D9%87-%D8%B1%D9%88%D8%A7%DB%8C%D8%AA-' - '%D8%A2%D9%85%D8%A7%D8%B1' - )[1], - ) - - def test_pages_from_html_meta(self): - """Test extracting pages from html meta tags.""" - self.assertIn( - '* {{cite journal ' - '| last=جلیلیان ' - '| first=شهرام ' - '| title=نهاد دایگانی در دورۀ ساسانیان ' - '| journal=تحقیقات تاریخ اجتماعی ' - '| volume=2 ' - '| issue=1 ' - '| date=2012-09-17 ' - '| issn=2383-0484 ' - '| pages=53–74 ' - '| url=http://socialhistory.ihcs.ac.ir/article_319_84.html ' - '| language=fa ' - '| ref=harv ' - '| access-date=', - urls_sfn_cit_ref( - 'http://socialhistory.ihcs.ac.ir/article_319_84.html' - )[1], - ) - - def test_empty_meta_author_content(self): - """Test that the output will not be malformed because empty meta.""" - self.assertIn( - "* {{cite web " - "| title=UAE's Enoc pays Iran $4 billion in oil dues " - "| website=Al Jazeera " - "| date=2017-05-29 " - "| url=http://www.aljazeera.com/news/2017/05/uae-enoc-pays-iran-4-" - "billion-oil-dues-170529171315570.html " - "| ref={{sfnref | Al Jazeera | 2017}} " - "| access-date=", - urls_sfn_cit_ref( - 'http://www.aljazeera.com/news/2017/05/' - 'uae-enoc-pays-iran-4-billion-oil-dues-170529171315570.html' - )[1], - ) - - def test_citation_author_reverse_order(self): - """Test correct detection of citation_author. - - first name and last name are in reverse order. - - """ - self.assertIn( - '* {{cite web ' - '| last=Hartman ' - '| first=JudithAnn R. ' - '| last2=Nelson ' - '| first2=Eric A. ' - '| title=Automaticity in Computation and Student Success in ' - 'Introductory Physical Science Courses ' - '| website=arXiv.org e-Print archive ' - '| date=2016-08-17 ' - '| url=https://arxiv.org/abs/1608.05006?utm_medium=email&' - 'utm_source=other&utm_campaign=opencourse.GdeNrll1EeSROyIACtiVvg.' - 'announcements%257Eopencourse.GdeNrll1EeSROyIACtiVvg.' - '4xDVKzx5EeeJjRJrkGD1dA ' - '| ref=harv ' - '| access-date=', - urls_sfn_cit_ref( - 'https://arxiv.org/abs/1608.05006?utm_medium=email&utm_source=' - 'other&utm_campaign=opencourse.GdeNrll1EeSROyIACtiVvg.' - 'announcements%257Eopencourse.GdeNrll1EeSROyIACtiVvg.' - '4xDVKzx5EeeJjRJrkGD1dA' - )[1], - ) - - def test_single_line_meta_tags(self): - """Issue #9.""" - self.assertIn( - "* {{cite web | last=Shoichet | first=Catherine E. " - "| title=Spill spews tons of coal ash into North Carolina's " - "Dan River | website=CNN | date=2014-02-09 " - "| url=http://www.cnn.com/2014/02/09/us/north-carolina-coal-ash" - "-spill/index.html | ref=harv | access-date=", - urls_sfn_cit_ref( - 'https://edition.cnn.com/' - '2014/02/09/us/north-carolina-coal-ash-spill/' - )[1], - ) - - -if __name__ == '__main__': - main() +# noinspection PyPackageRequirements +from pytest import mark + +from lib.urls import url_to_dict +from lib.commons import dict_to_sfn_cit_ref + + +def urls_scr(*args): + return dict_to_sfn_cit_ref(url_to_dict(*args)) + + +def test_bostonglobe1(): + """boston.com, dateformat '%B %d, %Y'""" + assert ( + '* {{cite web ' + '| last=Griffith ' + '| first=Bill ' + '| title=Hot Rod Stamps; Google on Road; A GM Prospectus ' + '| website=Boston.com ' + '| date=June 29, 2014 ' + '| url=https://www.boston.com/cars/news-and-reviews/2014/06/29/' + 'hot-rod-stamps-google-on-road-a-gm-prospectus ' + '| access-date=' + ) in urls_scr( + 'http://www.boston.com/cars/news-and-reviews/2014/06/28/' + 'hot-rod-stamps-google-road-prospectus/hylbVi9qonAwBIH10CwiDP/' + 'story.html', + '%B %d, %Y', + )[1] + + +def test_bostonglobe2(): + """bostonglobe.com""" + i = ( + 'http://www.bostonglobe.com/metro/2014/06/03/' + 'walsh-meets-with-college-leaders-off-campus-housing/' + 'lsxtLSGJMD86Gbkjay3D6J/story.html' + ) + o = urls_scr(i) + ct = ( + '* {{cite web ' + '| last=Saltzman ' + '| first=Jonathan ' + '| last2=Farragher ' + '| first2=Thomas ' + '| title=Walsh meets with college leaders on off-campus housing ' + '| website=BostonGlobe.com ' + '| date=2014-06-03 ' + '| url=https://www.bostonglobe.com/metro/2014/06/03/' + 'walsh-meets-with-college-leaders-off-campus-housing/' + 'lsxtLSGJMD86Gbkjay3D6J/story.html ' + '| access-date=' + ) + assert ct in o[1] + + +def test_bostonglobe3(): + """bostonmagazine.com. Author tags return unrelated authors.""" + i = ( + 'http://www.bostonmagazine.com/news/blog/2013/08/21/' + 'juliette-kayyem-jumps-in-for-guv/' + ) + o = urls_scr(i) + ct = ( + '* {{cite web ' + '| last=Bernstein ' + '| first=David S. ' + '| title=Juliette Kayyem Is Running for Governor of Massachusetts ' + '| website=Boston Magazine ' + '| date=2013-08-21 ' + '| url=http://www.bostonmagazine.com/news/blog/2013/08/21/' + 'juliette-kayyem-jumps-in-for-guv/ ' + '| access-date=' + ) + assert ct in o[1] + + +def test_washingtonpost1(): + """`1 author, 2005, the pubdate is different from last edit date""" + o = urls_scr( + 'http://www.washingtonpost.com/wp-dyn/content/article/2005/09/02/' + 'AR2005090200822.html' + ) + assert '{{sfn | Sachs | 2005}}' in o[0] + assert ( + '* {{cite web ' + '| last=Sachs ' + '| first=Andrea ' + '| title=March of the Migration ' + '| website=Washington Post ' + '| date=2005-09-04 ' + '| url=http://www.washingtonpost.com/wp-dyn/content/article/' + '2005/09/02/AR2005090200822.html ' + '| access-date=') in o[1] + + +def test_huffingtonpost1(): + """`1 author, 2013""" + o = urls_scr( + 'http://www.huffingtonpost.ca/annelise-sorg/' + 'blackfish-killer-whale-seaworld_b_3686306.html' + ) + assert '{{sfn | Sorg | 2013}}' == o[0] + assert ( + '* {{cite web ' + '| last=Sorg ' + '| first=Annelise ' + '| title=When Killer Whales Kill: Why the movie' + ' "Blackfish" Should Sink Captive Whale Programs ' + '| website=The Huffington Post ' + '| date=2013-08-01 ' + '| url=http://www.huffingtonpost.ca/annelise-sorg/' + 'blackfish-killer-whale-seaworld_b_3686306.html ' + '| access-date=') in o[1] + + +def test_huffingtonpost2(): + """`class:author` returns wrong result. Disallow `\n` in fullnames.""" + i = ( + 'http://www.huffingtonpost.com/jeremy-rifkin/' + 'obamas-climate-change-plan_b_5427656.html' + ) + o = urls_scr(i) + e2 = ( + "* {{cite web " + "| last=Rifkin " + "| first=Jeremy " + "| title=Beyond Obama's Plan: " + "A New Economic Vision for Addressing Climate Change " + "| website=The Huffington Post " + "| date=2014-06-02 " + "| url=http://www.huffingtonpost.com/jeremy-rifkin/" + "obamas-climate-change-plan_b_5427656.html " + "| access-date=" + ) + assert '{{sfn | Rifkin | 2014}}' == o[0] + assert e2 in o[1] + + +def test_dilytelegraph1(): + """`1 author, 2005""" + i = ( + 'http://www.telegraph.co.uk/news/health/3334755/' + 'We-could-see-the-whales-eyes-mouth...-' + 'the-barnacles-on-its-back.html' + ) + o = urls_scr(i) + e2 = ( + "* {{cite web " + "| last=Fogle " + "| first=Ben " + "| title=We could see the whale's eyes, mouth... " + "the barnacles on its back " + "| website=Telegraph.co.uk " + "| date=2005-12-22 " + "| url=http://www.telegraph.co.uk/news/health/3334755/" + "We-could-see-the-whales-eyes-mouth...-" + "the-barnacles-on-its-back.html " + "| access-date=" + ) + assert '{{sfn | Fogle | 2005}}' == o[0] + assert e2 in o[1] + + +def test_dilytelegraph2(): + """1 author, 2003""" + i = ( + 'http://www.telegraph.co.uk/news/science/science-news/3313298/' + 'Marine-collapse-linked-to-whale-decline.html' + ) + o = urls_scr(i) + e2 = ( + "* {{cite web " + "| last=Highfield " + "| first=Roger " + "| title=Marine 'collapse' linked to whale decline " + "| website=Telegraph.co.uk " + "| date=2003-09-29 " + "| url=http://www.telegraph.co.uk/news/science/science-news/" + "3313298/Marine-collapse-linked-to-whale-decline.html " + "| access-date=" + ) + assert '{{sfn | Highfield | 2003}}' == o[0] + assert e2 in o[1] + + +def test_dilytelegraph3(): + """1 author, 2011""" + i = ( + 'http://www.telegraph.co.uk/news/8323909/' + 'The-sperm-whale-works-in-extraordinary-ways.html' + ) + o = urls_scr(i) + e2 = ( + "* {{cite web " + "| last=Whitehead " + "| first=Hal " + "| title=The sperm whale works in extraordinary ways " + "| website=Telegraph.co.uk " + "| date=2011-02-15 " + "| url=http://www.telegraph.co.uk/news/science/8323909/" + "The-sperm-whale-works-in-extraordinary-ways.html " + "| access-date=" + ) + assert '{{sfn | Whitehead | 2011}}' == o[0] + assert e2 in o[1] + + +def test_dilymail1(): + """4 authors""" + o = urls_scr( + 'http://www.dailymail.co.uk/news/article-2633025/' + 'London-cleric-convicted-NYC-terrorism-trial.html' + ) + assert '{{sfn | Malm | Witheridge | Drury | Bates | 2014}}' == o[0] + assert ( + '* {{cite web ' + '| last=Malm ' + '| first=Sara ' + '| last2=Witheridge ' + '| first2=Annette ' + '| last3=Drury ' + '| first3=Ian ' + '| last4=Bates ' + '| first4=Daniel ' + '| title=Abu Hamza found guilty in US court of helping' + ' Al-Qaeda terrorists ' + '| website=Daily Mail Online ' + '| date=2014-05-19 ' + '| url=http://www.dailymail.co.uk/news/article-2633025/' + 'London-cleric-convicted-NYC-terrorism-trial.html ' + '| access-date=') in o[1] + + +def test_dilymail2(): + """`for` in byline.""" + assert ( + '* {{cite web ' + '| last=Gower ' + '| first=Eleanor ' + "| title=Kim Kardashian's meltdown at nude magazine cover" + " three years before full frontal photoshoot " + '| website=Daily Mail Online ' + '| date=2014-11-14 ' + '| url=http://www.dailymail.co.uk/tvshowbiz/article-2834145/' + 'I-m-never-taking-clothes-s-Vogue-Throwback-2011-video-shows-Kim-' + 'Kardashian-s-meltdown-nude-magazine-cover.html ' + '| access-date=' + ) in urls_scr( + 'http://www.dailymail.co.uk/tvshowbiz/article-2834145/' + 'I-m-never-taking-clothes-s-Vogue-Throwback-2011-video-' + 'shows-Kim-Kardashian-s-meltdown-nude-magazine-cover.html' + )[1] + + +def test_bbc1(): + """no authors""" + i = 'https://www.bbc.com/news/world-asia-27653361' + o = urls_scr(i) + ct = ( + "* {{cite web " + "| title=US 'received Qatar assurances' on Afghan prisoner deal " + "| website=BBC News " + "| date=2014-06-01 " + "| url=http://www.bbc.com/news/world-asia-27653361 " + "| ref={{sfnref | BBC News | 2014}} " + "| access-date=" + ) + assert ct in o[1] + + +def test_bbc2(): + """1 author""" + assert ( + '* {{cite web ' + '| last=Gage ' + '| first=Suzi ' + '| title=Sea otter return boosts ailing seagrass in California ' + '| website=BBC News ' + '| date=2013-08-26 ' + '| url=http://www.bbc.com/news/science-environment-23814524 ' + '| access-date=' + ) in urls_scr('http://www.bbc.com/news/science-environment-23814524')[1] + + +def test_bbc3(): + """https version of bbc2 (differs a lot!)""" + i = 'https://www.bbc.com/news/science-environment-23814524' + o = urls_scr(i) + ct = ( + '* {{cite web ' + '| last=Gage ' + '| first=Suzi ' + '| title=Sea otter return boosts ailing seagrass in California ' + '| website=BBC News ' + '| date=2013-08-26 ' + '| url=http://www.bbc.com/news/science-environment-23814524 ' + '| access-date=' + ) + assert ct in o[1] + + +def test_bbc4(): + """news.bbc.co.uk, 1 author""" + assert ( + "* {{cite web " + "| last=Jones " + "| first=Meirion " + "| title=Malaria advice 'risks lives' " + "| website=BBC NEWS " + "| date=2006-07-13 " + "| url=" + "http://news.bbc.co.uk/2/hi/programmes/newsnight/5178122.stm " + "| access-date=" + ) in urls_scr( + 'http://news.bbc.co.uk/2/hi/programmes/newsnight/5178122.stm')[1] + + +def test_bbc5(): + """news.bbc.co.uk, 1 author""" + assert ( + "* {{cite web " + "| last=Madslien " + "| first=Jorn " + "| title=Inside the Bentley factory " + "| website=BBC NEWS " + "| date=2002-12-24 " + "| url=http://news.bbc.co.uk/2/hi/business/2570109.stm " + "| access-date=" + ) in urls_scr('http://news.bbc.co.uk/2/hi/business/2570109.stm')[1] + + +def test_bbc6(): + """bbc.com, 1 author""" + i = 'http://www.bbc.com/news/science-environment-26267918' + o = urls_scr(i) + ct = ( + "* {{cite web " + "| last=Amos " + "| first=Jonathan " + "| title=European Space Agency picks Plato planet-hunting mission " + "| website=BBC News " + "| date=2014-02-20 " + "| url=http://www.bbc.com/news/science-environment-26267918 " + "| access-date=" + ) + assert ct in o[1] + + +def test_nyt1(): + """newstylct, 1 author""" + i = ( + 'http://www.nytimes.com/2014/05/30/business/international/' + 'on-the-internet-the-right-to-forget-vs-the-right-to-know.html?' + 'hp&_r=0' + ) + o = urls_scr(i) + ct = ( + '* {{cite web ' + '| last=Hakim ' + '| first=Danny ' + '| title=Right to Be Forgotten? Not That Easy ' + '| website=The New York Times ' + '| date=2014-05-30 ' + '| url=https://www.nytimes.com/2014/05/30/business/international/' + 'on-the-internet-the-right-to-forget-vs-the-right-to-know.html ' + '| access-date=' + ) + assert ct in o[1] + + +def test_nyt2(): + """newstylct, 2 authors""" + ct = ( + '* {{cite web ' + '| last=Belson ' + '| first=Ken ' + '| last2=Sandomir ' + '| first2=Richard ' + '| title=$2 Billion for Clippers? In Time, ' + 'It May Be a Steal for Steve Ballmer ' + '| website=The New York Times ' + '| date=2014-05-30 ' + '| url=https://www.nytimes.com/2014/05/31/sports/basketball/' + 'steven-a-ballmers-2-billion-play-for-clippers-is-a-big-bet-on-' + 'the-nba.html ' + '| access-date=' + ) + assert ct in urls_scr( + 'https://www.nytimes.com/2014/05/31/sports/basketball/' + 'steven-a-ballmers-2-billion-play-for-clippers-is-a-big-bet-' + 'on-the-nba.html?hp' + )[1] + + +def test_nyt3(): + """oldstylct, 1 author""" + i = 'http://www.nytimes.com/2007/12/25/world/africa/25kenya.html' + o = urls_scr(i) + ct = ( + '* {{cite web ' + '| last=Gettleman ' + '| first=Jeffrey ' + '| title=Election Rules Complicate Kenya Race ' + '| website=The New York Times ' + '| date=2007-12-25 ' + '| url=https://www.nytimes.com/2007/12/25/world/africa/' + '25kenya.html ' + '| access-date=' + ) + assert ct in o[1] + + +def test_nyt4(): + """newstylct, 2 authors, only byline""" + i = ( + 'http://dealbook.nytimes.com/2014/05/30/' + 'insider-trading-inquiry-includes-mickelson-and-icahn/' + ) + o = urls_scr(i) + ct = ( + '* {{cite web ' + '| last=Goldstein ' + '| first=Matthew ' + '| last2=Protess ' + '| first2=Ben ' + '| title=Investor, Bettor, Golfer: ' + 'Insider Trading Inquiry Includes Mickelson, Icahn and William T. ' + 'Walters ' + '| website=DealBook ' + '| date=2014-06-12 ' + '| url=https://dealbook.nytimes.com/2014/05/30/' + 'insider-trading-inquiry-includes-mickelson-and-icahn/ ' + '| access-date=' + ) + assert ct in o[1] + + +def test_nyt5(): + """special case for date format (not in usual meta tags)""" + i = ( + 'https://www.nytimes.com/2007/06/13/world/americas/' + '13iht-whale.1.6123654.html' + ) + o = urls_scr(i) + ct = ( + '* {{cite web ' + '| title=19th-century harpoon gives clue on whales ' + '| website=The New York Times ' + '| date=2007-06-13 ' + '| url=https://www.nytimes.com/2007/06/13/world/americas/' + '13iht-whale.1.6123654.html ' + '| ref={{sfnref | The New York Times | 2007}} ' + '| access-date=' + ) + assert ct in o[1] + + +def test_nyt6(): + """lastname=O'Connor""" + i = ( + 'http://www.nytimes.com/2003/10/09/us/' + 'adding-weight-to-suspicion-sonar-is-linked-to-whale-deaths.html' + ) + o = urls_scr(i) + ct = ( + "* {{cite web " + "| last=O'Connor " + "| first=Anahad " + "| title=Adding Weight to Suspicion, " + "Sonar Is Linked to Whale Deaths " + "| website=The New York Times " + "| date=2003-10-09 " + "| url=https://www.nytimes.com/2003/10/09/us/" + "adding-weight-to-suspicion-sonar-is-linked-to-whale-deaths.html " + "| access-date=" + ) + assert ct in o[1] + + +def test_tgdaily1(): + """Hard to find author and date.""" + i = ( + 'http://www.tgdaily.com/web/' + '100381-apple-might-buy-beats-for-32-billion' + ) + o = urls_scr(i) + ct = ( + '* {{cite web ' + '| title=Apple might buy Beats for $3.2 billion ' + '| website=TG Daily ' + '| date=2014-05-09 ' + '| url=http://www.tgdaily.com/web/' + '100381-apple-might-buy-beats-for-32-billion ' + '| ref={{sfnref | TG Daily | 2014}} ' + '| access-date=' + ) + assert ct in o[1] + + +def test_tgdaily2(): + """"Staff" in author name.""" + i = ( + 'http://www.tgdaily.com/space-features/' + '82906-sma-reveals-giant-star-cluster-in-the-making' + ) + o = urls_scr(i) + ct = ( + '* {{cite web ' + '| title=SMA reveals giant star cluster in the making ' + '| website=TG Daily ' + '| date=2013-12-17 ' + '| url=http://www.tgdaily.com/space-features/' + '82906-sma-reveals-giant-star-cluster-in-the-making ' + '| ref={{sfnref | TG Daily | 2013}} ' + '| access-date=' + ) + assert ct in o[1] + + +@mark.skip +def test_tgdaily3(): + """ABCNews. Wrong author: | last=News | first=ABC.""" + i = 'http://abcnews.go.com/blogs/headlines/2006/12/saddam_executed/' + o = urls_scr(i) + ct = ( + '* {{cite web ' + '| last=Ross ' + '| first=Brian ' + '| title=Saddam Executed; An Era Comes to an End ' + '| website=ABC News Blogs ' + '| date=2006-12-30 ' + '| url=http://abcnews.go.com/blogs/headlines/2006/12/' + 'saddam_executed/ ' + '| access-date=' + ) + assert ct in o[1] + + +def test_oth2(): + """Times of India, author could not be detected.""" + i = ( + 'http://timesofindia.indiatimes.com/city/pune/' + 'UK-allows-working-visas-for-Indian-students/' + 'articleshow/1163528927.cms?' + ) + o = urls_scr(i) + sfn = "{{sfn | Kashyap | 2001}}" + assert sfn in o[0] + + +def test_text_search(): + """Match byline on soup.text.""" + assert ( + '* {{cite web ' + '| last=Carlson ' + '| first=Kimberly ' + '| last2=York ' + '| first2=Jillian C. ' + '| title=Sudan Tech Sanctions Harm Innovation and Development: ' + 'US Government and Corporations Must Act ' + '| website=Electronic Frontier Foundation ' + '| date=2014-06-26 ' + '| url=https://www.eff.org/deeplinks/2014/06/' + 'sudan-tech-sanctions-harm-innovation-development-us-' + 'government-and-corporations-must-act ' + '| access-date=' + ) in urls_scr( + 'https://www.eff.org/deeplinks/2014/06/' + 'sudan-tech-sanctions-harm-innovation-development-us-' + 'government-and-corporations-must-act' + )[1] + + +# Disable because relies on class="author" which has been disabled due +# to hight error rate. +@mark.skip +def test_oth3(): + """4 authors.""" + i = ( + 'https://arstechnica.com/science/2007/09/' + 'the-pseudoscience-behind-homeopathy/' + ) + o = urls_scr(i) + ct = ( + '* {{cite web ' + '| last=Timmer ' + '| first=John ' + '| last2=Ford ' + '| first2=Matt ' + '| last3=Lee ' + '| first3=Chris ' + '| last4=Gitlin ' + '| first4=Jonathan ' + '| title=Diluting the scientific method: Ars looks at homeopathy ' + '| website=Ars Technica ' + '| date=2007-09-12 ' + '| url=https://arstechnica.com/science/2007/09/' + 'the-pseudoscience-behind-homeopathy/ ' + '| access-date=' + ) + assert ct in o[1] + + +def test_oth4(): + """rel="author" tag contains invalid information.""" + assert ( + "* {{cite web " + "| last=Ghose " + "| first=Tia " + "| title='Revolutionary' Physics:" + " Do Sterile Neutrinos Lurk in the Universe? " + "| website=Live Science " + "| date=2014-07-01 " + "| url=http://www.livescience.com/" + "46619-sterile-neutrino-experiment-beginning.html " + "| access-date=" + ) in urls_scr( + 'http://www.livescience.com/' + '46619-sterile-neutrino-experiment-beginning.html?' + 'cmpid=514645_20140702_27078936' + )[1] + + +def test_oth5(): + """Getting the date is tricky here.""" + o = urls_scr('http://www.magiran.com/npview.asp?ID=1410487') + assert "{{sfn | ''Magiran'' | 2007}}" in o[0] + assert ( + '* {{cite web ' + # todo: could this be fixed for the new format of magiran? + # '| last=نوري ' + # '| first=آزاده شهمير ' + '| title=روزنامه سرمایه (1386/03/01): دکتر طاهر صباحی، محقق و ' + 'مجموعه دار فرش: بازار جهانی با تولید فرش هنری نصیب ایران می شود ' + '| website=Magiran ' + '| date=2007-05-22 ' + '| url=http://www.magiran.com/npview.asp?ID=1410487 ' + '| language=fa ' + '| ref={{sfnref | Magiran | 2007}} ' + '| access-date=') == o[1][:-12] + + +def test_oth6(): + """Detection of website name.""" + o = urls_scr( + 'http://www.farsnews.com/newstext.php?nn=13930418000036' + ) + assert "{{sfn | ''خبرگزاری فارس'' | 2014}}" in o[0] + # Fars news is using 'خبرگزاری فارس' as og:author which is wrong + # and thats why its name is not italicized in sfn. + assert ( + '* {{cite web ' + '| title=آیت\u200cالله محمدی گیلانی دارفانی را وداع گفت ' + '| website=خبرگزاری فارس ' + '| date=2014-07-09 ' + '| url=http://www.farsnews.com/newstext.php?nn=13930418000036 ' + '| language=fa ' + '| ref={{sfnref | خبرگزاری فارس | 2014}} ' + '| access-date=') in o[1] + + +def test_oth7(): + """Contains a By Topic line and also the byline contains ' | '.""" + assert ( + '* {{cite web ' + '| last=Chandler ' + '| first=David L. ' + '| title=Traffic lights: There’s a better way ' + '| website=MIT News ' + '| date=2014-07-07 ' + '| url=http://news.mit.edu/2014/' + 'traffic-lights-theres-a-better-way-0707 ' + '| access-date=') in urls_scr( + 'http://news.mit.edu/2014/' + 'traffic-lights-theres-a-better-way-0707' + )[1] + + +def test_oth8(): + """Two authors from guardian that are mentions in other tags, too.""" + i = ( + 'http://www.theguardian.com/world/2014/jul/14/' + 'israel-drone-launched-gaza-ashdod' + ) + o = urls_scr(i) + ct = ( + '* {{cite web ' + '| last=Beaumont ' + '| first=Peter ' + '| last2=Crowcroft ' + '| first2=Orlando ' + '| title=Israel says it has shot down drone launched from Gaza ' + '| website=the Guardian ' + '| date=2014-07-14 ' + '| url=http://www.theguardian.com/world/2014/jul/14/' + 'israel-drone-launched-gaza-ashdod ' + '| access-date=' + ) + assert ct in o[1] + + +def test_oth10(): + """The Times. (Authors found by "byline" css selector)""" + assert ( + '* {{cite web ' + '| last=Lagan ' + '| first=Bernard ' + '| last2=Charter ' + '| first2=David ' + '| title=' + 'Woman who lost brother on MH370 mourns relatives on board MH17 ' + '| website=The Times & The Sunday Times ' + '| date=2014-07-18 ' + '| url=https://www.thetimes.co.uk/article/' + 'woman-who-lost-brother-on-mh370-mourns-relatives-on-board-' + 'mh17-r07q5rwppl0 ' + '| access-date=' + ) in urls_scr( + 'http://www.thetimes.co.uk/tto/news/world/' + 'australia-newzealand/article4151214.ece' + )[1] + + +def test_oth11(): + """Business News Daily.""" + i = ( + 'http://www.businessnewsdaily.com/6762-male-female-entrepreneurs' + '.html?cmpid=514642_20140715_27858876' + ) + o = urls_scr(i) + ct = ( + '* {{cite web ' + '| last=Helmrich ' + '| first=Brittney ' + '| title=Male vs. Female Entrepreneurs: How Are They Different? ' + '| website=Business News Daily ' + '| date=2014-07-10 ' + '| url=http://www.businessnewsdaily.com/6762-male-female-' + 'entrepreneurs.html ' + '| access-date=' + ) + assert ct in o[1] + + +def test_oth12(): + """thebulletin.org""" + i = 'http://thebulletin.org/evidence-shows-iron-dome-not-working7318' + o = urls_scr(i) + ct = ( + '* {{cite web ' + '| last=Postol ' + '| first=Theodore A. ' + '| title=The evidence that shows Iron Dome is not working ' + '| website=Bulletin of the Atomic Scientists ' + '| date=2014-07-19 ' + '| url=http://thebulletin.org/' + 'evidence-shows-iron-dome-not-working7318 ' + '| access-date=' + ) + assert ct in o[1] + + +def test_reverse_name(): + """Author is `Martin, Tracy`. Tracy should be the first name.""" + assert ( + '* {{cite web ' + '| last=Martin ' + '| first=Tracy ' + '| title=Dynamometers Explained ' + '| website=HighBeam Research ' + '| date=2014-07-01 ' + '| url=http://www.highbeam.com/doc/1P3-3372742961.html ' + '| access-date=' + ) in urls_scr( + 'http://www.highbeam.com/doc/1P3-3372742961.html' + )[1] + + +def test_oth14(): + """thebulletin.org""" + i = ( + 'http://www.independent.co.uk/news/business/' + 'the-investment-column-tt-group-1103208.html' + ) + o = urls_scr(i) + ct = ( + '* {{cite web ' + '| title=The Investment column: TT Group ' + '| website=The Independent ' + '| date=1999-06-29 ' + '| url=http://www.independent.co.uk/news/business/' + 'the-investment-column-tt-group-1103208.html ' + '| ref={{sfnref | The Independent | 1999}} ' + '| access-date=' + ) + assert ct in o[1] + + +def test_oth15(): + """Contains """ + assert ( + '* {{cite web ' + '| title=برجام شرایط بین‌المللی ایران را کاملا متحول کرد ' + '| website=ایسنا ' + '| date=2017-01-25 ' + '| url=http://www.isna.ir/news/95110603890/ ' + '| language=fa ' + '| ref={{sfnref | ایسنا | 2017}} ' + '| access-date=' + ) in urls_scr( + 'http://www.isna.ir/news/95110603890/' + '%D8%A8%D8%B1%D8%AC%D8%A7%D9%85-%D8%B4%D8%B1%D8%A7%DB%8C%D8%B7' + '-%D8%A8%DB%8C%D9%86-%D8%A7%D9%84%D9%85%D9%84%D9%84%DB%8C-' + '%D8%A7%DB%8C%D8%B1%D8%A7%D9%86-%D8%B1%D8%A7-' + '%DA%A9%D8%A7%D9%85%D9%84%D8%A7-%D9%85%D8%AA%D8%AD%D9%88%D9%84' + '-%DA%A9%D8%B1%D8%AF' + )[1] + + +def test_invalid_name(): + """Test that URL does not fail with InvalidNameError.""" + assert ( + '* {{cite web | title=انتخابات 96 به روایت آمار ' + '| website=پایگاه اطلاع رسانی شبکه خبر صدا' + ' و سیمای جمهوری اسلامی ایران |' + ' date=2017-05-24 | url=http://www.irinn.ir/fa/news/499654 ' + '| language=fa | ref={{sfnref |' + ' پایگاه اطلاع رسانی شبکه خبر' + ' صدا و سیمای جمهوری اسلامی ایران | 2017}} |' + ' access-date=' + ) in urls_scr( + 'http://www.irinn.ir/fa/news/499654/' + '%D8%A7%D9%86%D8%AA%D8%AE%D8%A7%D8%A8%D8%A7%D8%AA-96-' + '%D8%A8%D9%87-%D8%B1%D9%88%D8%A7%DB%8C%D8%AA-' + '%D8%A2%D9%85%D8%A7%D8%B1' + )[1] + + +def test_pages_from_html_meta(): + """Test extracting pages from html meta tags.""" + assert ( + '* {{cite journal ' + '| last=جلیلیان ' + '| first=شهرام ' + '| title=نهاد دایگانی در دورۀ ساسانیان ' + '| journal=تحقیقات تاریخ اجتماعی ' + '| volume=2 ' + '| issue=1 ' + '| date=2012-09-17 ' + '| issn=2383-0484 ' + '| pages=53–74 ' + '| url=http://socialhistory.ihcs.ac.ir/article_319_84.html ' + '| language=fa ' + '| access-date=') in urls_scr( + 'http://socialhistory.ihcs.ac.ir/article_319_84.html')[1] + + +def test_empty_meta_author_content(): + """Test that the output will not be malformed because empty meta.""" + assert ( + "* {{cite web " + "| title=UAE's Enoc pays Iran $4 billion in oil dues " + "| website=Al Jazeera " + "| date=2017-05-29 " + "| url=http://www.aljazeera.com/news/2017/05/uae-enoc-pays-iran-4-" + "billion-oil-dues-170529171315570.html " + "| ref={{sfnref | Al Jazeera | 2017}} " + "| access-date=") in urls_scr( + 'http://www.aljazeera.com/news/2017/05/' + 'uae-enoc-pays-iran-4-billion-oil-dues-170529171315570.html')[1] + + +def test_citation_author_reverse_order(): + """Test correct detection of citation_author. + + first name and last name are in reverse order. + + """ + assert ( + '* {{cite web ' + '| last=Hartman ' + '| first=JudithAnn R. ' + '| last2=Nelson ' + '| first2=Eric A. ' + '| title=Automaticity in Computation and Student Success in ' + 'Introductory Physical Science Courses ' + '| website=arXiv.org e-Print archive ' + '| date=2016-08-17 ' + '| url=https://arxiv.org/abs/1608.05006?utm_medium=email&' + 'utm_source=other&utm_campaign=opencourse.GdeNrll1EeSROyIACtiVvg.' + 'announcements%257Eopencourse.GdeNrll1EeSROyIACtiVvg.' + '4xDVKzx5EeeJjRJrkGD1dA ' + '| access-date=') in urls_scr( + 'https://arxiv.org/abs/1608.05006?utm_medium=email&utm_source=' + 'other&utm_campaign=opencourse.GdeNrll1EeSROyIACtiVvg.' + 'announcements%257Eopencourse.GdeNrll1EeSROyIACtiVvg.' + '4xDVKzx5EeeJjRJrkGD1dA')[1] + + +def test_single_line_meta_tags(): + """Issue #9.""" + assert ( + "* {{cite web | last=Shoichet | first=Catherine E. " + "| title=Spill spews tons of coal ash into North Carolina's " + "Dan River | website=CNN | date=2014-02-09 " + "| url=http://www.cnn.com/2014/02/09/us/north-carolina-coal-ash" + "-spill/index.html | access-date=") in urls_scr( + 'https://edition.cnn.com/' + '2014/02/09/us/north-carolina-coal-ash-spill/')[1] + + +def test_abc_author(): + assert ( + '* {{cite web | last=Ferguson | first=Kathleen ' + '| title=Glow worms in Wollemi National Park survived Gospers ' + 'Mountain bushfire - ABC News ' + '| website=ABC (Australian Broadcasting Corporation) ' + '| date=2020-09-06 | url=https://www.abc.net.au/news/2020-09-06/' + 'glow-worms-in-wollemi-national-park-survived-summer-bushfire/' + '12634762 | access-date=') in urls_scr( + 'https://www.abc.net.au/news/2020-09-06/' + 'glow-worms-in-wollemi-national-park-survived-summer-bushfire/' + '12634762')[1] + + +def test_indaily(): + assert ( + "* {{cite web | last=Siebert | first=Bension " + "| title=Epidemics expert questions Marshall's schools advice " + "| website=InDaily | date=2020-03-19 " + "| url=https://indaily.com.au/news/2020/03/19/epidemics-expert-contradicts-marshalls-schools-advice/ " + "| access-date=") in urls_scr( + 'https://indaily.com.au/news/2020/03/19/epidemics-expert-contradicts-marshalls-schools-advice/' + )[1] + + +def test_language_not_de_csbc(): + assert ( + "{{cite web " + "| last=Martin " + "| first=Emmie " + "| title=In San Francisco, households earning $117,000 qualify as ‘low income’ " + "| website=CNBC " + "| date=2018-06-28 " + "| url=https://www.cnbc.com/2018/06/28/families-earning-117000-qualify-as-low-income-in-san-francisco.html " + "| access-date=" + ) == urls_scr( + 'https://www.cnbc.com/2018/06/28/families-earning-117000-qualify-as-low-income-in-san-francisco.html' + )[1][2:-12] + + +def test_language_not_zh(): + assert ( + "{{cite web " + "| last=Jonscher " + "| first=Samantha " + "| title=Malcolm Abbott's domestic violence past shows 'urgent action' required to support First Nations - ABC News " + "| website=ABC (Australian Broadcasting Corporation) " + "| date=2022-05-14 " + "| url=https://www.abc.net.au/news/2022-05-15/malcolm-abbott-domestic-violence-prevention-fails/101059440 " + "| access-date=" + ) == urls_scr( + 'https://www.abc.net.au/news/2022-05-15/malcolm-abbott-domestic-violence-prevention-fails/101059440' + )[1][2:-12] + + +def test_home_site_name(): + # this url does contain site name, but its homepage does + assert ( + "* {{cite web | title=Black Convicts | website=University of Tasmania " + "| url=https://www.utas.edu.au/library/companion_to_tasmanian_history/B/Black%20Convicts.htm " + "| ref={{sfnref | University of Tasmania}} | access-date=" + ) == urls_scr( + 'https://www.utas.edu.au/library/companion_to_tasmanian_history/B/Black%20Convicts.htm' + )[1][:-12] + + +def test_use_doi_if_available(): + assert urls_scr('https://pubmed.ncbi.nlm.nih.gov/32687126/')[1] == ( + '* {{cite journal | last=Ojewola | first=RufusWale | last2=Tijani | ' + 'first2=KehindeHabeeb | last3=Fatuga | first3=AdedejiLukman | ' + 'last4=Onyeze | first4=ChigozieInnocent | last5=Okeke | ' + 'first5=ChikeJohn | title=Management of a giant prostatic ' + 'enlargement: Case report and review of the literature | ' + 'journal=Nigerian Postgraduate Medical Journal | publisher=Medknow | ' + 'volume=27 | issue=3 | year=2020 | issn=1117-1936 | ' + 'doi=10.4103/npmj.npmj_69_20 | page=242}}' + ) diff --git a/test/waybackmachine_test.py b/test/waybackmachine_test.py index bbdfc962..f6e87d44 100644 --- a/test/waybackmachine_test.py +++ b/test/waybackmachine_test.py @@ -1,112 +1,101 @@ -#! /usr/bin/python -# -*- coding: utf-8 -*- +from lib.waybackmachine import url_to_dict +from lib.commons import dict_to_sfn_cit_ref -"""Test urls.py module.""" +def waybackmachine_scr(*args): + return dict_to_sfn_cit_ref(url_to_dict(*args)) -from unittest import main, TestCase -from lib.waybackmachine import waybackmachine_sfn_cit_ref +def test_live_og_link(): + """dead-link=no""" + assert ( + '* {{cite web ' + '| last=Stuart ' + '| first=Hunter ' + '| title=LOOK: Bieber Fan Had $100K Worth Of Plastic Surgery To ' + 'Look Like His Idol ' + '| website=The Huffington Post ' + '| date=2013-10-19 ' + '| url=http://www.huffingtonpost.com/2013/10/19/' + 'plastic-surgery-justin-bieber-100k_n_4128563.html?' + 'utm_hp_ref=mostpopular ' + '| archive-url=http://web.archive.org/web/20131021230444/' + 'http://www.huffingtonpost.com/2013/10/19/' + 'plastic-surgery-justin-bieber-100k_n_4128563.html?' + 'utm_hp_ref=mostpopular ' + '| archive-date=2013-10-21 ' + '| url-status=live ' + '| access-date=' + ) == waybackmachine_scr( + 'http://web.archive.org/web/20131021230444/' + 'http://www.huffingtonpost.com/2013/10/19/' + 'plastic-surgery-justin-bieber-100k_n_4128563.html?' + 'utm_hp_ref=mostpopular' + )[1][:-12] -class WaybackmachineResponse(TestCase): +def test_dead_url(): + """url-status=dead""" + assert ( + '* {{cite web ' + '| title=London Development Centre: Support, time, recovery (STR) ' + 'workers ' + '| website=londondevelopmentcentre.org ' + '| date=2007-02-12 ' + '| url=http://www.londondevelopmentcentre.org/page.php?s=1&p=2462 ' + '| archive-url=https://web.archive.org/web/20070429193849id_/' + 'http://www.londondevelopmentcentre.org/page.php?s=1&p=2462 ' + '| archive-date=2007-04-29 ' + '| url-status=dead ' + '| ref={{sfnref | londondevelopmentcentre.org | 2007}} ' + '| access-date=' + ) == waybackmachine_scr( + 'https://web.archive.org/web/20070429193849id_/http://www.londondevelopmentcentre.org/page.php?s=1&p=2462' + )[1][:-12] - def test_live_og_link(self): - """dead-link=no""" - self.assertIn( - '* {{cite web ' - '| last=Stuart ' - '| first=Hunter ' - '| title=LOOK: Bieber Fan Had $100K Worth Of Plastic Surgery To ' - 'Look Like His Idol ' - '| website=The Huffington Post ' - '| date=2013-10-19 ' - '| url=http://www.huffingtonpost.com/2013/10/19/' - 'plastic-surgery-justin-bieber-100k_n_4128563.html?' - 'utm_hp_ref=mostpopular ' - '| archive-url=http://web.archive.org/web/20131021230444/' - 'http://www.huffingtonpost.com/2013/10/19/' - 'plastic-surgery-justin-bieber-100k_n_4128563.html?' - 'utm_hp_ref=mostpopular ' - '| archive-date=2013-10-21 ' - '| dead-url=no ' - '| ref=harv ' - '| access-date=', - waybackmachine_sfn_cit_ref( - 'http://web.archive.org/web/20131021230444/' - 'http://www.huffingtonpost.com/2013/10/19/' - 'plastic-surgery-justin-bieber-100k_n_4128563.html?' - 'utm_hp_ref=mostpopular' - )[1], - ) - def test_dead_url(self): - """dead-url=yes""" - self.assertIn( - '* {{cite web ' - '| title=London Development Centre: Support, time, recovery (STR) ' - 'workers ' - '| website=londondevelopmentcentre.org ' - '| date=2007-04-29 ' - '| url=http://www.londondevelopmentcentre.org/page.php?s=1&p=2462 ' - '| archive-url=https://web.archive.org/web/20070429193849id_/' - 'http://www.londondevelopmentcentre.org/page.php?s=1&p=2462 ' - '| archive-date=2007-04-29 ' - '| dead-url=yes ' - '| ref={{sfnref | londondevelopmentcentre.org | 2007}} ' - '| access-date=', - waybackmachine_sfn_cit_ref( - 'https://web.archive.org/web/20070429193849id_/' - 'http://www.londondevelopmentcentre.org/page.php?s=1&p=2462' - )[1] - ) - - def test_webless_url(self): - """The 'web/ component of the url can be omitted sometimes.""" - o = waybackmachine_sfn_cit_ref( - 'https://web.archive.org/web/20170119050001/' - 'http://www.isna.ir/news/95102918901/' - '%D8%B1%D9%88%D8%A7%D9%86%DA%86%DB%8C-%D8%AF%D8%B1-' - '%D8%A7%D8%B1%D8%AA%D8%A8%D8%A7%D8%B7-%D8%A8%D8%A7-' - '%D9%85%D9%88%D8%A7%D8%B6%D8%B9-' - '%D9%86%D8%A7%D9%85%D9%86%D8%A7%D8%B3%D8%A8-' - '%D8%A7%D8%AE%DB%8C%D8%B1-' - '%D9%85%D9%82%D8%A7%D9%85%D8%A7%D8%AA-' - '%D8%A7%D9%86%DA%AF%D9%84%DB%8C%D8%B3%DB%8C-' - '%D8%AF%D8%B1-%D9%85%D9%88%D8%B1%D8%AF' - ) - ct = ( - '* {{cite web ' - '| title=روانچی: در ارتباط با مواضع نامناسب ' - 'اخیر مقامات انگلیسی در مورد ایران گفت‌وگو خواهیم کرد |' - ' website=ایسنا |' - ' date=2017-01-18 ' - '| url=http://www.isna.ir/news/95102918901/' - '%D8%B1%D9%88%D8%A7%D9%86%DA%86%DB%8C-%D8%AF%D8%B1-' - '%D8%A7%D8%B1%D8%AA%D8%A8%D8%A7%D8%B7-%D8%A8%D8%A7-' - '%D9%85%D9%88%D8%A7%D8%B6%D8%B9-' - '%D9%86%D8%A7%D9%85%D9%86%D8%A7%D8%B3%D8%A8-' - '%D8%A7%D8%AE%DB%8C%D8%B1-%D9%85%D9%82%D8%A7%D9%85%D8%A7%D8%AA-' - '%D8%A7%D9%86%DA%AF%D9%84%DB%8C%D8%B3%DB%8C-%D8%AF%D8%B1-' - '%D9%85%D9%88%D8%B1%D8%AF ' - '| archive-url=https://web.archive.org/web/20170119050001/' - 'http://www.isna.ir/news/95102918901/' - '%D8%B1%D9%88%D8%A7%D9%86%DA%86%DB%8C-%D8%AF%D8%B1-' - '%D8%A7%D8%B1%D8%AA%D8%A8%D8%A7%D8%B7-%D8%A8%D8%A7-' - '%D9%85%D9%88%D8%A7%D8%B6%D8%B9-' - '%D9%86%D8%A7%D9%85%D9%86%D8%A7%D8%B3%D8%A8-' - '%D8%A7%D8%AE%DB%8C%D8%B1-' - '%D9%85%D9%82%D8%A7%D9%85%D8%A7%D8%AA-' - '%D8%A7%D9%86%DA%AF%D9%84%DB%8C%D8%B3%DB%8C-%D8%AF%D8%B1-' - '%D9%85%D9%88%D8%B1%D8%AF ' - '| archive-date=2017-01-19 ' - '| dead-url=no ' - '| language=fa ' - '| ref={{sfnref | ایسنا | 2017}} ' - '| access-date=' - ) - self.assertIn(ct, o[1]) - - -if __name__ == '__main__': - main() +def test_webless_url(): + """The 'web/ component of the url can be omitted sometimes.""" + o = waybackmachine_scr( + 'https://web.archive.org/web/20170119050001/' + 'http://www.isna.ir/news/95102918901/' + '%D8%B1%D9%88%D8%A7%D9%86%DA%86%DB%8C-%D8%AF%D8%B1-' + '%D8%A7%D8%B1%D8%AA%D8%A8%D8%A7%D8%B7-%D8%A8%D8%A7-' + '%D9%85%D9%88%D8%A7%D8%B6%D8%B9-' + '%D9%86%D8%A7%D9%85%D9%86%D8%A7%D8%B3%D8%A8-' + '%D8%A7%D8%AE%DB%8C%D8%B1-' + '%D9%85%D9%82%D8%A7%D9%85%D8%A7%D8%AA-' + '%D8%A7%D9%86%DA%AF%D9%84%DB%8C%D8%B3%DB%8C-' + '%D8%AF%D8%B1-%D9%85%D9%88%D8%B1%D8%AF' + ) + ct = ( + '* {{cite web ' + '| title=روانچی: در ارتباط با مواضع نامناسب ' + 'اخیر مقامات انگلیسی در مورد ایران گفت‌وگو خواهیم کرد |' + ' website=ایسنا |' + ' date=2017-01-18 ' + '| url=http://www.isna.ir/news/95102918901/' + '%D8%B1%D9%88%D8%A7%D9%86%DA%86%DB%8C-%D8%AF%D8%B1-' + '%D8%A7%D8%B1%D8%AA%D8%A8%D8%A7%D8%B7-%D8%A8%D8%A7-' + '%D9%85%D9%88%D8%A7%D8%B6%D8%B9-' + '%D9%86%D8%A7%D9%85%D9%86%D8%A7%D8%B3%D8%A8-' + '%D8%A7%D8%AE%DB%8C%D8%B1-%D9%85%D9%82%D8%A7%D9%85%D8%A7%D8%AA-' + '%D8%A7%D9%86%DA%AF%D9%84%DB%8C%D8%B3%DB%8C-%D8%AF%D8%B1-' + '%D9%85%D9%88%D8%B1%D8%AF ' + '| archive-url=https://web.archive.org/web/20170119050001/' + 'http://www.isna.ir/news/95102918901/' + '%D8%B1%D9%88%D8%A7%D9%86%DA%86%DB%8C-%D8%AF%D8%B1-' + '%D8%A7%D8%B1%D8%AA%D8%A8%D8%A7%D8%B7-%D8%A8%D8%A7-' + '%D9%85%D9%88%D8%A7%D8%B6%D8%B9-' + '%D9%86%D8%A7%D9%85%D9%86%D8%A7%D8%B3%D8%A8-' + '%D8%A7%D8%AE%DB%8C%D8%B1-' + '%D9%85%D9%82%D8%A7%D9%85%D8%A7%D8%AA-' + '%D8%A7%D9%86%DA%AF%D9%84%DB%8C%D8%B3%DB%8C-%D8%AF%D8%B1-' + '%D9%85%D9%88%D8%B1%D8%AF ' + '| archive-date=2017-01-19 ' + '| url-status=live ' + '| language=fa ' + '| ref={{sfnref | ایسنا | 2017}} ' + '| access-date=' + ) + assert ct in o[1] diff --git a/test_requirements.txt b/test_requirements.txt new file mode 100644 index 00000000..f2df533d --- /dev/null +++ b/test_requirements.txt @@ -0,0 +1,2 @@ +path +environs