diff --git a/.gitignore b/.gitignore
index e22ac668..b56e5055 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
*.py[cod]
*.log
-.*
+.docs
__pycache__/
+config.py
diff --git a/README.md b/README.md
index a1e635fb..46edf88c 100644
--- a/README.md
+++ b/README.md
@@ -1,19 +1,32 @@
# Citer
-A citation generator tool for Wikipedia. Currently accessible from:
-http://tools.wmflabs.org/citer/ (the English version)
-http://tools.wmflabs.org/yadfa/ (the Persian version)
+A citation generator tool for Wikipedia. Currently accessible from:\
+[https://citer.toolforge.org/](https://citer.toolforge.org/) (the English version)\
+[https://yadfa.toolforge.org/](https://yadfa.toolforge.org/) (the Persian version)
## What does it do?
-Citer is specially useful for generating citations from Google Books URLs, DOIs (Any Digital object Identifiers) and ISBNs (International Standard Book Numbers).
-Additionally URL of many major news websites are supported, including:
-The New York Times, BBC, Daily Mail, Daily Mirror, The Daily Telegraph, The Huffington Post, The Washington Post, The Boston Globe, Bloomberg Businessweek, Financial Times, and The Times of India. Sepecial support for the URLs of the [Wayback Machine](https://en.wikipedia.org/wiki/Wayback_Machine) is also implemented.
-
-Some other tested and supported Persian web-sites:
-* http://www.noormags.com (نورمگز)
+Citer is especially useful for generating citations from Google Books URLs, DOIs (Any Digital object Identifiers) and ISBNs (International Standard Book Numbers).
+Additionally, URLs of many major news websites are supported, including:
+
+* The New York Times
+* BBC
+* Daily Mail
+* Daily Mirror
+* The Daily Telegraph
+* The Huffington Post
+* The Washington Post
+* The Boston Globe
+* Bloomberg Businessweek
+* Financial Times
+* The Times of India
+
+Special support for the URLs of the [Wayback Machine](https://en.wikipedia.org/wiki/Wayback_Machine) is also implemented.
+
+Some other tested and supported Persian websites:
+* http://www.noormags.ir (نورمگز)
* http://www.noorlib.ir (کتابخانه دیجیتال نور)
-* http://www.adinebook.com (آدینهبوک)
+* http://www.ketab.ir (خانه كتاب)
* http://socialhistory.ihcs.ac.ir/ (تحقیقات تاریخ اجتماعی)
@@ -21,15 +34,20 @@ Some other tested and supported Persian web-sites:
To run Citer on your local computer:
-1. Install Python 3.6+.
-2. Clone the project.
-3. Install the dependencies using `pip install -r requirements.txt`.
-3. Make sure that `flup` is __not__ installed in your environment.
-4. Run Citer by calling `main.py`.
+1. Install Python 3.9+
+2. Clone the project
+3. Install the dependencies using `pip install --user -r requirements.txt`
+4. Copy `config.py.example` to `config.py` (You might want to get an NCBI API key and add it to the config file if you're going to use its services)
+5. Run `python3 app.py`
-If everything goes fine, the main page will be accessible from:
- http://127.0.0.1:5000/
+If there are no warnings or error messages (and no HTML is displayed), the main page will be accessible from:\
+ [http://localhost:5000/](http://localhost:5000/)
+If you experience any problems or have questions, please open an issue on this repo.
## Language Setting
-The default language is English and can be change to Persian using the setting in config.py file.
+The default language is English and can be changed to Persian using the setting in the config.py file.
+
+
+## Known issues
+* The bookmarklet does not work on archive.org (issue #26) or any other website that does not allow opening external links. One needs to use Citer directly in such cases.
diff --git a/app.py b/app.py
index 93ec3d28..912d030f 100644
--- a/app.py
+++ b/app.py
@@ -1,58 +1,76 @@
-#! /usr/bin/python
-# -*- coding: utf-8 -*-
-
from collections import defaultdict
from html import unescape
from logging import getLogger, Formatter, WARNING, INFO
from logging.handlers import RotatingFileHandler
+from os.path import dirname, abspath
from urllib.parse import parse_qs, urlparse, unquote
from wsgiref.headers import Headers
-from requests import ConnectionError as RequestsConnectionError
+from requests import ConnectionError as RequestsConnectionError, \
+ JSONDecodeError
from config import LANG
-from lib.adinebook import adinehbook_sfn_cit_ref
-from lib.commons import uninum2en, sfn_cit_ref_to_json
-from lib.doi import doi_sfn_cit_ref, DOI_SEARCH
-from lib.googlebooks import googlebooks_sfn_cit_ref
-from lib.isbn_oclc import (
- ISBN_10OR13_SEARCH, IsbnError, isbn_sfn_cit_ref, oclc_sfn_cit_ref)
-from lib.noorlib import noorlib_sfn_cit_ref
-from lib.noormags import noormags_sfn_cit_ref
-from lib.pubmed import pmcid_sfn_cit_ref, pmid_sfn_cit_ref
-from lib.urls import urls_sfn_cit_ref
-from lib.waybackmachine import waybackmachine_sfn_cit_ref
+from lib.ketabir import url_to_dict as ketabir_url_to_dict
+from lib.commons import uninum2en, scr_to_json, ISBN_10OR13_SEARCH, \
+ dict_to_sfn_cit_ref, ReturnError
+from lib.doi import doi_to_dict, DOI_SEARCH
+from lib.googlebooks import url_to_dict as google_books_dict
+from lib.isbn_oclc import IsbnError, isbn_to_dict, oclc_dict
+from lib.jstor import url_to_dict as jstor_url_to_dict
+from lib.noorlib import url_to_dict as noorlib_url_to_dict
+from lib.noormags import url_to_dict as noormags_url_to_dict
+from lib.pubmed import pmcid_dict, pmid_dict
+from lib.urls import url_to_dict as urls_url_to_dict
+from lib.waybackmachine import url_to_dict as archive_url_to_dict
if LANG == 'en':
from lib.html.en import (
- DEFAULT_SFN_CIT_REF,
- UNDEFINED_INPUT_SFN_CIT_REF,
- HTTPERROR_SFN_CIT_REF,
- OTHER_EXCEPTION_SFN_CIT_REF,
- sfn_cit_ref_to_html,
+ DEFAULT_SCR,
+ UNDEFINED_INPUT_SCR,
+ HTTPERROR_SCR,
+ OTHER_EXCEPTION_SCR,
+ scr_to_html,
CSS,
CSS_HEADERS,
JS,
JS_HEADERS)
else:
from lib.html.fa import (
- DEFAULT_SFN_CIT_REF,
- UNDEFINED_INPUT_SFN_CIT_REF,
- HTTPERROR_SFN_CIT_REF,
- OTHER_EXCEPTION_SFN_CIT_REF,
- sfn_cit_ref_to_html,
+ DEFAULT_SCR,
+ UNDEFINED_INPUT_SCR,
+ HTTPERROR_SCR,
+ OTHER_EXCEPTION_SCR,
+ scr_to_html,
CSS,
CSS_HEADERS)
+def google_encrypted_dict(url, parsed_url, date_format) -> dict:
+ if parsed_url[2][:7] in {'/books', '/books/'}:
+ # sample urls:
+ # https://encrypted.google.com/books?id=6upvonUt0O8C
+ # https://www.google.com/books?id=bwfoCAAAQBAJ&pg=PA32
+ # https://www.google.com/books/edition/_/bwfoCAAAQBAJ?gbpv=1&pg=PA32
+ return google_books_dict(parsed_url, date_format)
+ return urls_url_to_dict(url, date_format)
+
+
TLDLESS_NETLOC_RESOLVER = {
- 'adinebook': adinehbook_sfn_cit_ref,
- 'adinehbook': adinehbook_sfn_cit_ref,
- 'noorlib': noorlib_sfn_cit_ref,
- 'noormags': noormags_sfn_cit_ref,
- 'web.archive': waybackmachine_sfn_cit_ref,
- 'web-beta.archive': waybackmachine_sfn_cit_ref,
- 'books.google.co': googlebooks_sfn_cit_ref,
- 'books.google': googlebooks_sfn_cit_ref,
+ 'ketab': ketabir_url_to_dict,
+
+ 'noorlib': noorlib_url_to_dict,
+ 'noormags': noormags_url_to_dict,
+
+ 'web.archive': archive_url_to_dict,
+ 'web-beta.archive': archive_url_to_dict,
+
+ 'books.google.co': google_books_dict,
+ 'books.google.com': google_books_dict,
+ 'books.google': google_books_dict,
+
+ 'google': google_encrypted_dict,
+ 'encrypted.google': google_encrypted_dict,
+
+ 'jstor': jstor_url_to_dict,
}.get
RESPONSE_HEADERS = Headers([('Content-Type', 'text/html; charset=UTF-8')])
@@ -65,13 +83,14 @@
def get_root_logger():
custom_logger = getLogger()
custom_logger.setLevel(INFO)
+ srcdir = dirname(abspath(__file__))
handler = RotatingFileHandler(
- filename='citer.log',
+ filename=f'{srcdir}/citer.log',
mode='a',
maxBytes=20000,
backupCount=0,
encoding='utf-8',
- delay=0)
+ )
handler.setLevel(INFO)
handler.setFormatter(
Formatter('\n%(asctime)s\n%(levelname)s\n%(message)s\n'))
@@ -82,7 +101,7 @@ def get_root_logger():
LOGGER = get_root_logger()
-def url_doi_isbn_to_sfn_cit_ref(user_input, date_format) -> tuple:
+def input_to_dict(user_input, date_format, /) -> dict:
en_user_input = unquote(uninum2en(user_input))
# Checking the user input for dot is important because
# the use of dotless domains is prohibited.
@@ -90,110 +109,121 @@ def url_doi_isbn_to_sfn_cit_ref(user_input, date_format) -> tuple:
if '.' in en_user_input:
# Try predefined URLs
# Todo: The following code could be done in threads.
- if not user_input.startswith('http'):
+ if not (url_input := user_input.startswith('http')):
url = 'http://' + user_input
else:
url = user_input
+ parsed_url = urlparse(url)
# TLD stands for top-level domain
- tldless_netloc = urlparse(url)[1].rpartition('.')[0]
- resolver = TLDLESS_NETLOC_RESOLVER(
+ tldless_netloc = parsed_url[1].rpartition('.')[0]
+ # todo: make lazy?
+ if (to_dict := TLDLESS_NETLOC_RESOLVER(
tldless_netloc[4:] if tldless_netloc.startswith('www.')
- else tldless_netloc)
- if resolver:
- return resolver(url, date_format)
+ else tldless_netloc
+ )) is not None:
+ if to_dict is google_books_dict:
+ return to_dict(parsed_url, date_format)
+ elif to_dict is google_encrypted_dict:
+ return to_dict(url, parsed_url, date_format)
+ return to_dict(url, date_format)
+
# DOIs contain dots
- m = DOI_SEARCH(unescape(en_user_input))
- if m:
- return doi_sfn_cit_ref(m.group(1), True, date_format)
- return urls_sfn_cit_ref(url, date_format)
+ if (m := DOI_SEARCH(unescape(en_user_input))) is not None:
+ try:
+ return doi_to_dict(m[0], True, date_format)
+ except JSONDecodeError:
+ if url_input is False:
+ raise
+ # continue with urls_scr
+
+ return urls_url_to_dict(url, date_format)
else:
# We can check user inputs containing dots for ISBNs, but probably is
- # error prone.
- m = ISBN_10OR13_SEARCH(en_user_input)
- if m:
+ # error-prone.
+ if (m := ISBN_10OR13_SEARCH(en_user_input)) is not None:
try:
- return isbn_sfn_cit_ref(m.group(), True, date_format)
+ return isbn_to_dict(m[0], True, date_format)
except IsbnError:
pass
- return UNDEFINED_INPUT_SFN_CIT_REF
-
+ return UNDEFINED_INPUT_SCR
-def app(environ, start_response):
- query_dict_get = parse_qs(environ['QUERY_STRING']).get
+def app(environ: dict, start_response: callable) -> tuple:
path_info = environ['PATH_INFO']
if '/static/' in path_info:
- if path_info.endswith('.css'):
+ if path_info[-4:] == '.css':
start_response('200 OK', CSS_HEADERS)
- return [CSS]
+ return CSS,
else:
# path_info.endswith('.js') and config.lang == 'en'
start_response('200 OK', JS_HEADERS)
- return [JS]
+ return JS,
+ query_dict_get = parse_qs(environ['QUERY_STRING']).get
date_format = query_dict_get('dateformat', [''])[0].strip()
-
input_type = query_dict_get('input_type', [''])[0]
# Warning: input is not escaped!
- user_input = query_dict_get('user_input', [''])[0].strip()
- if not user_input:
- response_body = sfn_cit_ref_to_html(
- DEFAULT_SFN_CIT_REF, date_format, input_type
+ if not (user_input := query_dict_get('user_input', [''])[0].strip()):
+ response_body = scr_to_html(
+ DEFAULT_SCR, date_format, input_type
).encode()
RESPONSE_HEADERS['Content-Length'] = str(len(response_body))
start_response('200 OK', RESPONSE_HEADERS.items())
- return [response_body]
+ return response_body,
output_format = query_dict_get('output_format', [''])[0] # apiquery
- resolver = input_type_to_resolver[input_type]
+ to_dict = input_type_to_resolver[input_type]
# noinspection PyBroadException
try:
- response = resolver(user_input, date_format)
+ d = to_dict(user_input, date_format)
except RequestsConnectionError:
status = '500 ConnectionError'
LOGGER.exception(user_input)
if output_format == 'json':
- response_body = sfn_cit_ref_to_json(HTTPERROR_SFN_CIT_REF)
+ response_body = scr_to_json(HTTPERROR_SCR)
else:
- response_body = sfn_cit_ref_to_html(
- HTTPERROR_SFN_CIT_REF, date_format, input_type)
- except Exception:
+ response_body = scr_to_html(
+ HTTPERROR_SCR, date_format, input_type)
+ except Exception as e:
status = '500 Internal Server Error'
- LOGGER.exception(user_input)
+
+ if isinstance(e, ReturnError):
+ scr = e.args
+ else:
+ LOGGER.exception(user_input)
+ scr = OTHER_EXCEPTION_SCR
+
if output_format == 'json':
- response_body = sfn_cit_ref_to_json(OTHER_EXCEPTION_SFN_CIT_REF)
+ response_body = scr_to_json(scr)
else:
- response_body = sfn_cit_ref_to_html(
- OTHER_EXCEPTION_SFN_CIT_REF, date_format, input_type)
+ response_body = scr_to_html(scr, date_format, input_type)
else:
status = '200 OK'
+ scr = dict_to_sfn_cit_ref(d)
if output_format == 'json':
- response_body = sfn_cit_ref_to_json(response)
+ response_body = scr_to_json(scr)
else:
- response_body = sfn_cit_ref_to_html(
- response, date_format, input_type)
+ response_body = scr_to_html(scr, date_format, input_type)
response_body = response_body.encode()
RESPONSE_HEADERS['Content-Length'] = str(len(response_body))
start_response(status, RESPONSE_HEADERS.items())
- return [response_body]
+ return response_body,
input_type_to_resolver = defaultdict(
- lambda: url_doi_isbn_to_sfn_cit_ref, {
- 'url-doi-isbn': url_doi_isbn_to_sfn_cit_ref,
- 'pmid': pmid_sfn_cit_ref,
- 'pmcid': pmcid_sfn_cit_ref,
- 'oclc': oclc_sfn_cit_ref})
+ lambda: input_to_dict, {
+ 'url-doi-isbn': input_to_dict, # todo: can be removed?
+ 'pmid': pmid_dict,
+ 'pmcid': pmcid_dict,
+ 'oclc': oclc_dict})
if __name__ == '__main__':
# note that app.py is not run as '__main__' in kubernetes
- try:
- from flup.server.fcgi import WSGIServer
- WSGIServer(app).run()
- except ImportError: # on local computer
- from wsgiref.simple_server import make_server
- httpd = make_server('localhost', 5000, app)
- httpd.serve_forever()
+ # only for local computer
+ from wsgiref.simple_server import make_server
+ httpd = make_server('localhost', 5000, app)
+ print('serving on http://localhost:5000')
+ httpd.serve_forever()
diff --git a/citer-cli b/citer-cli
new file mode 100755
index 00000000..19cf29a7
--- /dev/null
+++ b/citer-cli
@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+import argparse
+import html
+
+import app
+
+def main():
+ ap = argparse.ArgumentParser()
+ types = list(app.input_type_to_resolver.keys())
+ ap.add_argument('-t', '--type', choices=types, metavar='TYPE', default='url-doi-isbn', help=str.join(' or ', map(repr, types)))
+ ap.add_argument('-d', '--date-format', metavar='DATE-FMT', default='%Y-%m-%d')
+ ap.add_argument('ident', metavar='URL-OR-ID')
+ options = ap.parse_args()
+ resolver = app.input_type_to_resolver[options.type]
+ d = resolver(options.ident, options.date_format)
+ scr = app.dict_to_sfn_cit_ref(d)
+ for item in scr:
+ print(html.unescape(item), end='\n\n')
+
+if __name__ == '__main__':
+ main()
diff --git a/config.py b/config.py.example
similarity index 90%
rename from config.py
rename to config.py.example
index 7d0c3e7f..41f77881 100644
--- a/config.py
+++ b/config.py.example
@@ -1,4 +1,5 @@
LANG = 'en'
+STATIC_PATH = './static/' + LANG
USER_AGENT = 'https://github.com/5j9/citer'
SPOOFED_USER_AGENT = ''
diff --git a/dev/googlebooksdomains.py b/dev/googlebooksdomains.py
new file mode 100644
index 00000000..8eaab5d1
--- /dev/null
+++ b/dev/googlebooksdomains.py
@@ -0,0 +1,13 @@
+from requests import get
+from re import findall, MULTILINE
+
+github_content = get('https://github.com/SebastianJ/fiber-freeze/raw/master/data/https_urls.txt').content
+github_domains = set(findall(rb'(?<=//)books\.google\.[^/\n]*', github_content, MULTILINE))
+assert len(github_domains) == 16
+
+# Most referenced domains on the English Wikipedia (2015-05-15) (T96927)
+# https://phabricator.wikimedia.org/P587
+phab_content = get('https://phab.wmfusercontent.org/file/data/nw6aboiuwxgb4mytb45u/PHID-FILE-perbg6gmtj55dgenca5h/Most_referenced_domains_on_the_English_Wikipedia_%282015-05-15%29_%28T96927%29').content
+phab_domains = set(findall(rb'(?<=//)books\.google\.[^/\n]*', phab_content, MULTILINE))
+
+assert 'books.google.co' not in phab_domains | github_domains
diff --git a/install.py b/install.py
index b90260fa..33af4791 100644
--- a/install.py
+++ b/install.py
@@ -23,18 +23,15 @@ def set_file_permissions():
def copy_config():
committer_date = check_output([
- 'git', '-C', HOME + '/www/python/src', 'log', '-1', '--format=%cI'
- ]).partition(b'T')[0].replace(b'-', b'.')
+ 'git', '-C', HOME + '/www/python/src', 'log', '-1', '--format=%cd',
+ '--date=short']).rstrip().replace(b'-', b'.')
with open(HOME + '/.citer_config', 'rb') as home_config:
with open(HOME + '/www/python/src/config.py', 'wb') as src_config:
- src_config.write(
- sub(
- b"(USER_AGENT = '.*)'\n",
- br"\1 v" + committer_date + b"'\n",
- home_config.read(),
- 1,
- )
- )
+ src_config.write(sub(
+ b"(USER_AGENT = '.*)'\n",
+ br"\1 v" + committer_date + b"'\n",
+ home_config.read(),
+ 1))
def main():
diff --git a/lib/adinebook.py b/lib/adinebook.py
deleted file mode 100644
index d8462166..00000000
--- a/lib/adinebook.py
+++ /dev/null
@@ -1,109 +0,0 @@
-#! /usr/bin/python
-# -*- coding: utf-8 -*-
-
-"""All things that are specifically related to adinebook website"""
-
-from collections import defaultdict
-import logging
-
-from langid import classify
-from regex import compile as regex_compile
-from requests import RequestException
-
-from lib.commons import first_last, dict_to_sfn_cit_ref, fetch
-
-ISBN_SEARCH = regex_compile(
- r' \s*+'
- r'(?:(?\d\d?)? (?[^،]*+)، )?(?\d{4})'
-).search
-PUBLISHER_SEARCH = regex_compile(
- r'ناشر:(?:<[^>]++>\s*)++([^<\n]++)'
-).search
-TITLE_SEARCH = regex_compile(
- r' tuple:
- """Return the response namedtuple."""
- dictionary = url2dictionary(url)
- dictionary['date_format'] = date_format
- if 'language' not in dictionary:
- # Assume that language is either fa or en.
- # Todo: give warning about this assumption?
- dictionary['language'] = \
- classify(dictionary['title'])[0]
- return dict_to_sfn_cit_ref(dictionary)
-
-
-def isbn2url(isbn: str):
- """Convert isbn to AdinebookURL. Return the url as string."""
- # Apparently adinebook uses 10 digit codes (without hyphens) for its
- # book-urls. If it's an isbn13 then the first 3 digits are excluded
- isbn = isbn.replace('-', '').replace(' ', '')
- if len(isbn) == 13:
- isbn = isbn[3:]
- url = 'http://www.adinebook.com/gp/product/' + isbn
- return url
-
-
-def url2dictionary(adinebook_url: str):
- """Get adinebook_url and return the result as a dict."""
- try:
- # Try to see if adinebook is available,
- # ottobib should continoue its work in isbn.py if it is not.
- r = fetch(adinebook_url)
- adinebook_html = r.content.decode('utf-8')
- except RequestException:
- logger.exception(adinebook_url)
- return
- if 'صفحه مورد نظر پبدا نشد.' in adinebook_html:
- return
- else:
- d = defaultdict(lambda: None, cite_type='book')
- d['title'] = TITLE_SEARCH(adinebook_html)[1]
- # initiating name lists:
- others = []
- authors = []
- editors = []
- translators = []
- # building lists:
- for name in AUTHORS_SEARCH(adinebook_html)[1].strip().split('،'):
- if '(به اهتمام)' in name:
- authors.append(first_last(name.partition('(به اهتمام)')[0]))
- elif '(ویراستار)' in name:
- editors.append(first_last(name.partition('(ویراستار)')[0]))
- elif '(مترجم)' in name:
- translators.append(first_last(name.partition('(مترجم)')[0]))
- elif '(' in name:
- others.append(('', name))
- else:
- authors.append(first_last(name))
- if authors:
- d['authors'] = authors
- if others:
- d['others'] = others
- if editors:
- d['editors'] = editors
- if translators:
- d['translators'] = translators
- m = PUBLISHER_SEARCH(adinebook_html)
- if m:
- d['publisher'] = m.group(1)
- m = DATE_SEARCH(adinebook_html)
- if m:
- d['month'] = m.group('month')
- d['year'] = m.group('year')
- m = ISBN_SEARCH(adinebook_html)
- if m:
- d['isbn'] = m.group(1)
- return d
-
-
-logger = logging.getLogger(__name__)
diff --git a/lib/bibtex.py b/lib/bibtex.py
index 90336ad5..890c72ad 100644
--- a/lib/bibtex.py
+++ b/lib/bibtex.py
@@ -1,6 +1,3 @@
-#! /usr/bin/python
-# -*- coding: utf-8 -*-
-
"""This module is used for parsing BibTeX entries.
The goal of this code is to parse BibTeX entries from a number of known sites
@@ -8,26 +5,26 @@
incorrectly or incompletely as TeX system is very complex and this module is
not intended to parse TeX.
-Some of the known issues:
+Known issues:
* Currently it does not detect special symbols and many TeX escape
sequences (more information: http://www.bibtex.org/SpecialSymbols/)
- * String concatinatins are not recognized. (e.g. "str1" # "str2")
+ * String concatenations are not recognized. (e.g. "str1" # "str2")
* Abbreviations are not supported (e.g. @string { foo = "Mrs. Foo" })
"""
from collections import defaultdict
-import regex as regex
+from regex import compile as rc
from lib.commons import first_last
# To remove Texts like {APA} from input.
-WORDS_IN_BRACES_SUB = regex.compile(r'(? defaultdict:
@@ -43,12 +40,10 @@ def parse(bibtex):
bibtex = special_sequence_cleanup(bibtex)
d = search_for_tag(bibtex)
# cite_type: book, journal, incollection, etc.
- m = TYPE_SEARCH(bibtex)
- if m:
- d['cite_type'] = m.group(1).strip().lower()
+ if (m := TYPE_SEARCH(bibtex)) is not None:
+ d['cite_type'] = m[1].strip().lower()
# author
- author = d['author']
- if author:
+ if author := d['author']:
d['authors'] = names = []
names_append = names.append
for author in author.split(' and '):
@@ -59,8 +54,7 @@ def parse(bibtex):
names_append(first_last(author))
del d['author']
# editor, not tested, just a copy of author
- editor = d['editor']
- if editor:
+ if editor := d['editor']:
d['editors'] = names = []
names_append = names.append
for editor in editor.split(' and '):
@@ -70,8 +64,7 @@ def parse(bibtex):
continue
names_append(first_last(editor))
del d['editor']
- pages = d['pages']
- if pages:
+ if pages := d['pages']:
d['page'] = \
pages.replace(' ', '').replace('--', '–').replace('-', '–')
return d
diff --git a/lib/commons.py b/lib/commons.py
index 689ffa12..587435e6 100644
--- a/lib/commons.py
+++ b/lib/commons.py
@@ -1,11 +1,6 @@
-#! /usr/bin/python
-# -*- coding: utf-8 -*-
-
-"""Common variables, functions, and classes used in string conversions, etc."""
-
from calendar import month_abbr, month_name
-from datetime import datetime
-from datetime import date as datetime_date
+from datetime import datetime, date as datetime_date
+from functools import partial
from json import dumps as json_dumps
from isbnlib import mask as isbn_mask, NotValidISBNError
@@ -21,21 +16,39 @@
from lib.generator_fa import sfn_cit_ref
+# The regex is from:
+# http://stackoverflow.com/questions/27910/finding-a-doi-in-a-document-or-page
+DOI_SEARCH = regex_compile(
+ r'''
+ \b
+ 10\.[0-9]{4,}+
+ (?:\.[0-9]++)*+
+ /[^"&\'\s]++
+ \b
+ ''',
+ VERBOSE,
+).search
+
+
b_TO_NUM = {name.lower(): num for num, name in enumerate(month_abbr) if num}
B_TO_NUM = {name.lower(): num for num, name in enumerate(month_name) if num}
-# jB_TO_NUM contains entries for both ی and ي
jB_TO_NUM = {
'فروردین': 1,
+ 'فروردين': 1,
'اردیبهشت': 2,
+ 'ارديبهشت': 2,
'خرداد': 3,
'تیر': 4,
+ 'تير': 4,
'مرداد': 5,
'شهریور': 6,
+ 'شهريور': 6,
'مهر': 7,
'آبان': 8,
'آذر': 9,
'دی': 10,
+ 'دي': 10,
'بهمن': 11,
'اسفند': 12}
@@ -57,10 +70,7 @@
(?:(?:(?:Sept|Nov|Dec)em)|Octo)ber))
''')
# فروردین|اردیبهشت|خرداد...
-jB = (
- '(?>(?'
- + '|'.join([jm for jm in jB_TO_NUM]).replace('ی', '[یي]')
- + '))')
+jB = f"(?>(?{'|'.join([jm for jm in jB_TO_NUM]).replace('ی', '[یي]')}))"
# Month abbreviations:
b = r'(?>(?Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)).?'
# Month numbers 0?1-12
@@ -74,23 +84,49 @@
# Gregorian year pattern 1900-2099
Y = r'(?(?:19|20)\d\d)'
ANYDATE_PATTERN = (
- '(?:(?:' + B + '|' + b + r')\ ' + d + r',?\ ' + Y
- + '|' + d + r'\ (?:' + B + '|' + b + r')\ ' + Y
- + '|' + Y + '(?[-/])' + zm + '(?P=sep)' + zd
- + '|' + r'(?\d\d?)\ ' + jB + r'\ (?\d\d\d\d)'
- + r'|\b' + Y + zm + zd
- + ')')
+ fr'(?:(?:{B}|{b})\ {d},?\ {Y}|{d}\ (?:{B}|{b})\ {Y}|{Y}(?[-/]){zm}'
+ fr'(?P=sep){zd}|(?\d\d?)\ {jB}\ (?\d\d\d\d))')
ANYDATE_SEARCH = regex_compile(ANYDATE_PATTERN, VERBOSE).search
DIGITS_FINDALL = regex_compile(r'\d').findall
MC_SUB = regex_compile(r'MC(\w)', IGNORECASE).sub
+LAST_FIRST = partial(regex_compile(r'[,،]').split, maxsplit=1)
AGENT_HEADER = {
'User-Agent': USER_AGENT,
# Not required but recommended by
# https://meta.wikimedia.org/wiki/User-Agent_policy
- 'Api-User-Agent': NCBI_TOOL + '/' + NCBI_EMAIL}
-SPOOFED_AGENT_HEADER = {'User-Agent': SPOOFED_USER_AGENT}
+ 'Api-User-Agent': f'{NCBI_TOOL}/{NCBI_EMAIL}'}
+SPOOFED_AGENT_HEADER = {
+ 'User-Agent': SPOOFED_USER_AGENT,
+ 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+}
+REQUEST = partial(Session().request, timeout=10)
+
+# original regex from:
+# https://www.debuggex.com/r/0Npla56ipD5aeTr9
+# https://www.debuggex.com/r/2s3Wld3CVCR1wKoZ
+ISBN_10OR13_SEARCH = regex_compile(
+ r'97[89]([ -]?+)(?=\d{1,5}\1?+\d{1,7}\1?+\d{1,6}\1?+\d)(?:\d\1*){9}\d'
+ r'|(?=\d{1,5}([ -]?+)\d{1,7}\2?+\d{1,6}\2?+\d)(?:\d\2*+){9}[\dX]'
+).search
+
+ISBN10_SEARCH = regex_compile(
+ r'(?=\d{1,5}([ -]?+)\d{1,7}\1?+\d{1,6}\1?+\d)(?:\d\1*+){9}[\dX]'
+).search
+
+ISBN13_SEARCH = regex_compile(
+ r'97[89]([ -]?+)(?=\d{1,5}\1?+\d{1,7}\1?+\d{1,6}\1?+\d)(?:\d\1*+){9}\d'
+).search
+
+
+# original regex from: http://stackoverflow.com/a/14260708/2705757
+# ISBN_REGEX = regex_compile(
+# r'(?=[-0-9 ]{17}|[-0-9X ]{13}|[0-9X]{10})(?:97[89][- ]?)'
+# r'?[0-9]{1,5}[- ]?(?:[0-9]+[- ]?){2}[0-9X]'
+# )
+
+FOUR_DIGIT_NUM = regex_compile(r'\d\d\d\d').search
class InvalidNameError(ValueError):
@@ -100,28 +136,27 @@ class InvalidNameError(ValueError):
class NumberInNameError(InvalidNameError):
- """Raise when a RawName() contains digits.."""
+ """Raise when a RawName() contains digits."""
-def fetch(url, spoof=False, **kwargs):
- with Session() as session:
- return session.request(
- 'get', url, timeout=10,
- headers=SPOOFED_AGENT_HEADER if spoof else AGENT_HEADER,
- **kwargs)
+class ReturnError(RuntimeError):
+ """Raise to display message to end user.
-def dict_to_sfn_cit_ref(dictionary) -> tuple:
- """Return (sfn, cite, ref) strings.
-
- dictionary should be ready before calling this function.
- The dictionary will be cleaned up (empty values will be removed) and
- all values will be encoded using encode_for_template() function.
- ISBN (if exist) will be hyphenated.
+ Pass sfn, cit, and ref fields as positional args.
"""
- value_encode(dictionary)
- isbn = dictionary['isbn']
- if isbn:
+
+
+def request(url, spoof=False, method='get', **kwargs):
+ headers = SPOOFED_AGENT_HEADER if spoof else AGENT_HEADER
+ if 'headers' in kwargs:
+ headers |= kwargs.pop('headers')
+ return REQUEST(method, url, headers=headers, **kwargs)
+
+
+def dict_to_sfn_cit_ref(dictionary) -> tuple:
+ # Return (sfn, cite, ref) strings.
+ if isbn := dictionary.get('isbn'):
try:
dictionary['isbn'] = isbn_mask(isbn)
except NotValidISBNError:
@@ -130,7 +165,7 @@ def dict_to_sfn_cit_ref(dictionary) -> tuple:
return sfn_cit_ref(dictionary)
-def sfn_cit_ref_to_json(response) -> str:
+def scr_to_json(response) -> str:
"""Generate api JSON response containing sfn, cite and ref."""
return json_dumps({
'reference_tag': response.ref,
@@ -153,7 +188,10 @@ def first_last(fullname, separator=None) -> tuple:
>>> first_last('DeBolt, V.', ',')
('V.', 'DeBolt')
- >>> first_last('BBC', None)
+ The function is more strict if the separator is None:
+
+ >>> first_last('BBC', None) # InvalidNameError
+ >>> first_last('BBC', ',')
('', 'BBC')
"""
fullname = fullname.strip()
@@ -169,29 +207,26 @@ def first_last(fullname, separator=None) -> tuple:
fullname = fullname[:-4]
else:
suffix = None
- if not separator:
- if ',' in fullname:
- separator = ','
- elif '،' in fullname:
- separator = '،'
- if separator:
+ if separator is None:
+ try:
+ lastname, firstname = LAST_FIRST(fullname)
+ except ValueError: # not enough values to unpack, use whitespace
+ sname = fullname.split()
+ if len(sname) == 1: # single word first-last with None separator
+ raise InvalidNameError
+ lastname = sname.pop()
+ firstname = ' '.join(sname)
+ else:
if separator in fullname:
lastname, _, firstname = fullname.partition(separator)
else:
lastname, firstname = fullname, ''
- else:
- sname = fullname.split()
- lastname = sname.pop()
- firstname = ' '.join(sname)
firstname = firstname.strip()
if (firstname.isupper() and lastname.isupper()) or \
(firstname.islower() and lastname.islower()):
firstname = firstname.title()
lastname = lastname.title()
- lastname = MC_SUB(
- lambda mtch: 'Mc' + mtch.group(1).upper(),
- lastname,
- )
+ lastname = MC_SUB(lambda m: 'Mc' + m[1].upper(), lastname)
if suffix:
firstname += suffix.title()
return firstname, lastname
@@ -227,72 +262,29 @@ def find_any_date(str_or_match) -> datetime.date or None:
groupdict = match.groupdict()
day = int(groupdict['d'])
year = int(groupdict['Y'])
- month = groupdict.get('jB')
today = datetime.today().date()
- if month:
+ get = groupdict.get
+
+ if (month := get('jB')) is not None:
date = jdate(year, jB_TO_NUM[month], day).togregorian()
if date <= today:
return date
return
- month = groupdict.get('B')
- if month:
+
+ if (month := get('B')) is not None:
date = datetime_date(year, B_TO_NUM[month.lower()], day)
if date <= today:
return date
return
- month = groupdict.get('b')
- if month:
+
+ if (month := get('b')) is not None:
date = datetime_date(year, b_TO_NUM[month.lower()], day)
if date <= today:
return date
return
- month = groupdict.get('m')
- if month:
+
+ if (month := get('m')) is not None:
date = datetime_date(year, int(month), day)
if date <= today:
return date
return
-
-
-def bidi_pop(string) -> str:
- """Makes sure all LRE, RLE, LRO, or RLO chars are terminated with PDF."""
- # Pop isolations
- isolates = [
- '\u2066', # LRI
- '\u2067', # RLI
- '\u2068', # FSI
- ]
- diff = sum(string.count(c) for c in isolates) - \
- string.count('\u2069') # PDI
- string += '\u2069' * diff
- # Pop embeddings and overrides
- diff = sum(
- string.count(c) for c in (
- '\u202A', # LRE
- '\u202B', # RLE
- '\u202D', # LRO
- '\u202E', # RLO
- )
- ) - string.count('\u202C') # PDF
- return string + '\u202C' * diff
-
-
-def value_encode(dictionary) -> None:
- """Cleanup dictionary values.
-
- * Remove any key with False bool value.
- * Replace special characters in dictionary values with their respective
- HTML entities.
- * Strip all values.
- """
- for k, v in dictionary.items():
- if isinstance(v, str):
- v = (
- bidi_pop(v.strip())
- .replace('|', '|')
- .replace('[', '[')
- .replace(']', ']')
- .replace('\r\n', ' ')
- .replace('\n', ' ')
- )
- dictionary[k] = v
diff --git a/lib/doi.py b/lib/doi.py
index dfc51ebe..03e09495 100644
--- a/lib/doi.py
+++ b/lib/doi.py
@@ -1,99 +1,77 @@
-#! /usr/bin/python
-# -*- coding: utf-8 -*-
-
"""Codes related to DOI inputs."""
from collections import defaultdict
-from datetime import date as datetime_date
-from urllib.parse import unquote
+from datetime import datetime
+from typing import Any
+from urllib.parse import unquote_plus
from html import unescape
from langid import classify
-from regex import compile as regex_compile, VERBOSE
-from lib.commons import dict_to_sfn_cit_ref, fetch
+from lib.commons import request, DOI_SEARCH
from config import LANG
-# The regex is from:
-# http://stackoverflow.com/questions/27910/finding-a-doi-in-a-document-or-page
-DOI_SEARCH = regex_compile(
- r'''
- \b(
- 10\.[0-9]{4,}+
- (?:\.[0-9]++)*+
- /[^"&\'\s]++
- )\b
- ''',
- VERBOSE,
-).search
-
-
-def doi_sfn_cit_ref(doi_or_url, pure=False, date_format='%Y-%m-%d') -> tuple:
- """Return the response namedtuple."""
+def doi_to_dict(doi_or_url, pure=False, date_format='%Y-%m-%d', /) -> dict:
if pure:
doi = doi_or_url
else:
# unescape '&', '<', and '>' in doi_or_url
# decode percent encodings
- decoded_url = unquote(unescape(doi_or_url))
- doi = DOI_SEARCH(decoded_url)[1]
+ decoded_url = unquote_plus(unescape(doi_or_url))
+ doi = DOI_SEARCH(decoded_url)[0]
dictionary = get_crossref_dict(doi)
dictionary['date_format'] = date_format
if LANG == 'fa':
dictionary['language'] = classify(dictionary['title'])[0]
- return dict_to_sfn_cit_ref(dictionary)
+ return dictionary
def get_crossref_dict(doi) -> defaultdict:
"""Return the parsed data of crossref.org for the given DOI."""
- # See https://github.com/CrossRef/rest-api-doc/blob/master/api_format.md
- # for documentation.
- # Force using the version 1 of the API to prevent breakage. See:
- # https://github.com/CrossRef/rest-api-doc/blob/master/rest_api.md#how-to-manage-api-versions
- j = fetch('http://api.crossref.org/v1/works/' + doi).json()
- assert j['status'] == 'ok'
- d = defaultdict(
- lambda: None, {k.lower(): v for k, v in j['message'].items()})
-
- d['cite_type'] = d.pop('type')
-
- for field in ('title', 'container-title', 'issn', 'isbn'):
- value = d[field]
- if value:
- d[field] = value[0]
-
- date = d['issued']['date-parts'][0]
- date_len = len(date)
- if date_len == 3:
- d['date'] = datetime_date(*date)
- elif date_len == 2:
- d['year'], d['month'] = str(date[0]), str(date[1])
- else:
- year = date[0]
- # date can be of the form [None]
- # https://github.com/CrossRef/rest-api-doc/issues/169
- if year:
- d['year'] = str(date[0])
-
- authors = d['author']
- if authors:
- d['authors'] = \
- [(name['given'], name['family']) for name in authors]
-
- editors = d['editor']
- if editors:
- d['editors'] = \
- [(name['given'], name['family']) for name in editors]
-
- translators = d['translator']
- if translators:
- d['translators'] = \
- [(name['given'], name['family']) for name in translators]
-
- page = d['page']
- if page:
+ # See https://citation.crosscite.org/docs.html for documentation.
+ j = request(
+ f'https://doi.org/{doi}',
+ headers={"Accept": "application/vnd.citationstyles.csl+json"}
+ ).json()
+
+ d : defaultdict[str, Any] = defaultdict(
+ lambda: None, {k.lower(): v for k, v in j.items()})
+
+ d['cite_type'] = d['type']
+
+ if (author := d['author']) is not None:
+ d['authors'] = [
+ (a['given'], a['family']) for a in author if 'given' in a
+ ]
+
+ if (issn := d['issn']) is not None:
+ d['issn'] = issn[0]
+
+ if (published := d['published']) is not None:
+ date = published['date-parts'][0]
+ if len(date) == 3:
+ d['date'] = datetime(*date)
+ else: # todo: better handle the case where len == 2
+ d['year'] = f'{date[0]}'
+
+ if (page := d['page']) is not None:
d['page'] = page.replace('-', '–')
+ if (isbn := d['isbn']) is not None:
+ d['isbn'] = isbn[0]
+
return d
+
+
+def extract_names(d: dict, from_key: str, to_key: str):
+ if (from_values := d[from_key]) is None:
+ return
+ to_values = d[to_key] = []
+ authors_append = to_values.append
+ for from_value in from_values:
+ try:
+ authors_append((from_value['given'], from_value['family']))
+ except KeyError:
+ pass
diff --git a/lib/generator_en.py b/lib/generator_en.py
index 0c025c4e..0f158546 100644
--- a/lib/generator_en.py
+++ b/lib/generator_en.py
@@ -1,22 +1,24 @@
-#! /usr/bin/python
-# -*- coding: utf-8 -*-
-
"""Codes required to create English Wikipedia citation templates."""
-import re
from datetime import date as datetime_date
+from functools import partial
from collections import defaultdict
from logging import getLogger
+from regex import compile as regex_compile
+
from lib.language import TO_TWO_LETTER_CODE
# Includes ShortDOIs (See: http://shortdoi.org/) and
# https://www.crossref.org/display-guidelines/
-DOI_URL_MATCH = re.compile(
- r'https?://(dx\.)?doi\.org/'
-).match
+DOI_URL_MATCH = regex_compile(r'https?://(dx\.)?doi\.org/').match
+DIGITS_TO_EN = str.maketrans('۰۱۲۳۴۵۶۷۸۹', '0123456789')
+
+refless = partial(regex_compile(
+ r'( \| ref=({{.*?}}|harv))(?P \| |}})'
+).sub, r'\g')
TYPE_TO_CITE = {
# BibTex types. Descriptions are from
@@ -35,6 +37,7 @@
'manual': 'book',
# An article from a journal or magazine.
'article': 'journal',
+ 'article-journal': 'journal',
# The same as INPROCEEDINGS, included for Scribe compatibility.
'conference': 'conference',
# An article in a conference proceedings.
@@ -65,6 +68,7 @@
'reference-entry': '',
'proceedings-article': 'conference',
'journal': 'journal',
+ 'jour': 'journal',
# https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id=22368089&retmode=json&tool=my_tool&email=my_email@example.com
'Journal Article': 'journal',
'component': '',
@@ -80,20 +84,21 @@
'book-series': 'book',
'edited-book': 'book',
'standard-series': '',
+ 'rprt': 'report',
}.get
def sfn_cit_ref(d: defaultdict) -> tuple:
- """Create citation templates according to the given dictionary."""
+ """Return sfn, citation, and ref."""
date_format = d['date_format']
- cite_type = TYPE_TO_CITE(d['cite_type'])
- if not cite_type:
+ if not (cite_type := TYPE_TO_CITE(d['cite_type'])):
logger.warning('Unknown citation type: %s, d: %s', cite_type, d)
cite_type = ''
- cit = '* {{cite ' + cite_type
+ cit = '* {{cite'
+ else:
+ cit = '* {{cite ' + cite_type
sfn = '{{sfn'
- authors = d['authors']
publisher = d['publisher']
website = d['website']
title = d['title']
@@ -103,7 +108,7 @@ def sfn_cit_ref(d: defaultdict) -> tuple:
else:
journal = d['journal']
- if authors:
+ if authors := d['authors']:
cit += names2para(authors, 'first', 'last', 'author')
# {{sfn}} only supports a maximum of four authors
for first, last in authors[:4]:
@@ -112,26 +117,23 @@ def sfn_cit_ref(d: defaultdict) -> tuple:
# the same order should be used in citation_template:
sfn += ' | ' + (
publisher or
- "''" + journal + "''" if journal else
- "''" + website + "''" if website else
+ f"''{journal}''" if journal else
+ f"''{website}''" if website else
title or 'Anon.'
)
- editors = d['editors']
- if editors:
+ if editors := d['editors']:
cit += names2para(editors, 'editor-first', 'editor-last', 'editor')
- translators = d['translators']
- if translators:
+ if translators := d['translators']:
for i, (first, last) in enumerate(translators):
- translators[i] = first, last + ' (مترجم)'
+ translators[i] = first, f'{last} (مترجم)'
# Todo: add a 'Translated by ' before name of translators?
others = d['others']
if others:
others.extend(d['translators'])
else:
d['others'] = d['translators']
- others = d['others']
- if others:
+ if others := d['others']:
cit += names1para(others, 'others')
if cite_type == 'book':
@@ -140,99 +142,89 @@ def sfn_cit_ref(d: defaultdict) -> tuple:
booktitle = None
if booktitle:
- cit += ' | title=' + booktitle
- if title:
- cit += ' | chapter=' + title
+ cit += f' | title={booktitle}'
+ if title:
+ cit += f' | chapter={title}'
elif title:
- cit += ' | title=' + title
+ cit += f' | title={title}'
if journal:
- cit += ' | journal=' + journal
+ cit += f' | journal={journal}'
elif website:
- cit += ' | website=' + website
+ cit += f' | website={website}'
- chapter = d['chapter']
- if chapter:
- cit += ' | chapter=' + chapter
+ if chapter := d['chapter']:
+ cit += f' | chapter={chapter}'
- publisher = d['publisher'] or d['organization']
- if publisher:
- cit += ' | publisher=' + publisher
+ if publisher := (d['publisher'] or d['organization']):
+ cit += f' | publisher={publisher}'
- address = d['address'] or d['publisher-location']
- if address:
- cit += ' | publication-place=' + address
+ if address := (d['address'] or d['publisher-location']):
+ cit += f' | publication-place={address}'
- edition = d['edition']
- if edition:
- cit += ' | edition=' + edition
+ if edition := d['edition']:
+ cit += f' | edition={edition}'
- series = d['series']
- if series:
- cit += ' | series=' + series
+ if series := d['series']:
+ cit += f' | series={series}'
- volume = d['volume']
- if volume:
- cit += ' | volume=' + volume
+ if volume := d['volume']:
+ cit += f' | volume={volume.translate(DIGITS_TO_EN)}'
- issue = d['issue'] or d['number']
- if issue:
- cit += ' | issue=' + issue
+ if issue := (d['issue'] or d['number']):
+ cit += f' | issue={issue}'
- date = d['date']
- if date:
+ if date := d['date']:
if not isinstance(date, str):
date = date.strftime(date_format)
- cit += ' | date=' + date
+ cit += f' | date={date}'
- year = d['year']
- if year:
+ if year := d['year']:
+ year = str(int(year)) # convert any non-Latin digits to English ones
if not date or year not in date:
- cit += ' | year=' + year
- sfn += ' | ' + year
+ cit += f' | year={year}'
+ sfn += f' | {year}'
- isbn = d['isbn']
- if isbn:
- cit += ' | isbn=' + isbn
+ if isbn := d['isbn']:
+ cit += f' | isbn={isbn}'
- issn = d['issn']
- if issn:
- cit += ' | issn=' + issn
+ if issn := d['issn']:
+ cit += f' | issn={issn}'
- pmid = d['pmid']
- if pmid:
- cit += ' | pmid=' + pmid
+ if pmid := d['pmid']:
+ cit += f' | pmid={pmid}'
- pmcid = d['pmcid']
- if pmcid:
- cit += ' | pmc=' + pmcid
+ if pmcid := d['pmcid']:
+ cit += f' | pmc={pmcid}'
- doi = d['doi']
- if doi:
- cit += ' | doi=' + doi
+ if doi := d['doi']:
+ cit += f' | doi={doi}'
- oclc = d['oclc']
- if oclc:
- cit += ' | oclc=' + oclc
+ if oclc := d['oclc']:
+ cit += f' | oclc={oclc}'
- pages = d['page']
- if pages:
+ if jstor := d['jstor']:
+ cit += f' | jstor={jstor}'
+ jstor_access = d['jstor-access']
+ if jstor_access:
+ cit += f' | jstor-access=free'
+
+ if pages := d['page']:
if '–' in pages:
- sfn += ' | pp=' + pages
+ sfn += f' | pp={pages}'
else:
- sfn += ' | p=' + pages
+ sfn += f' | p={pages}'
if cite_type == 'journal':
if pages:
if '–' in pages:
- cit += ' | pages=' + pages
+ cit += f' | pages={pages}'
else:
- cit += ' | page=' + pages
+ cit += f' | page={pages}'
- url = d['url']
- if url:
+ if url := d['url']:
# Don't add a DOI URL if we already have added a DOI.
if not doi or not DOI_URL_MATCH(url):
- cit += ' | url=' + url
+ cit += f' | url={url}'
else:
# To prevent addition of access date
url = None
@@ -240,58 +232,44 @@ def sfn_cit_ref(d: defaultdict) -> tuple:
if not pages and cite_type != 'web':
sfn += ' | p='
- archive_url = d['archive-url']
- if archive_url:
+ if archive_url := d['archive-url']:
cit += (
- ' | archive-url=' + archive_url +
- ' | archive-date=' + d['archive-date'].strftime(date_format) +
- ' | dead-url=' + d['dead-url']
- )
+ f' | archive-url={archive_url}'
+ f' | archive-date={d["archive-date"].strftime(date_format)}'
+ f' | url-status={d["url-status"]}')
- language = d['language']
- if language:
+ if language := d['language']:
language = TO_TWO_LETTER_CODE(language.lower(), language)
if language.lower() != 'en':
cit += ' | language=' + language
- # Todo: Template:Citation generates anchors for Harvard by default
- # references
- # whereas the Cite templates by default do not (although they can be
- # made to
- # do so).
- if authors:
- cit += ' | ref=harv'
- else:
+ if not authors:
# order should match sfn_template
- cit += ' | ref={{sfnref | ' +\
- (publisher or journal or website or title or 'Anon.')
+ cit += ' | ref={{sfnref | ' \
+ f'{publisher or journal or website or title or "Anon."}'
if year:
- cit += ' | ' + year
+ cit += f' | {year}'
cit += '}}'
if url:
- cit += ' | access-date=' + datetime_date.today().strftime(date_format)
+ cit += f' | access-date={datetime_date.today().strftime(date_format)}'
cit += '}}'
sfn += '}}'
# Finally create the ref tag.
name = sfn[8:-2].replace(' | ', ' ').replace("'", '')
- text = re.sub(
- r'( \| ref=({{.*?}}|harv))(?P \| |}})',
- r'\g',
- cit[2:],
- )
+ text = refless(cit[2:])
if ' p=' in name and ' | page=' not in text:
name = name.replace(' p=', ' p. ')
if pages:
- text = text[:-2] + ' | page=' + pages + '}}'
+ text = f'{text[:-2]} | page={pages}}}}}'
else:
- text = text[:-2] + ' | page=}}'
+ text = f'{text[:-2]} | page=}}}}'
elif ' pp=' in name:
name = name.replace(' pp=', ' pp. ')
if pages and ' | pages=' not in text:
- text = text[:-2] + ' | pages=' + pages + '}}'
- ref = '<ref name="' + name + '">' + text + '</ref>'
+ text = f'{text[:-2]} | pages={pages}}}}}'
+ ref = f'<ref name="{name}">{text}</ref>'
return sfn, cit, ref
@@ -303,38 +281,35 @@ def names2para(names, fn_parameter, ln_parameter, nofn_parameter=None):
c += 1
if c == 1:
if first or not nofn_parameter:
- s += ' | ' + ln_parameter + '=' + last
- s += ' | ' + fn_parameter + '=' + first
+ s += f' | {ln_parameter}={last} | {fn_parameter}={first}'
else:
- s += ' | ' + nofn_parameter + '=' + fullname(first, last)
+ s += f' | {nofn_parameter}={fullname(first, last)}'
else:
if first or not nofn_parameter:
- s += ' | ' + ln_parameter + str(c) + '=' + last
- s += ' | ' + fn_parameter + str(c) + '=' + first
+ s += f' | {ln_parameter}{c}={last} | {fn_parameter}{c}={first}'
else:
- s += ' | ' + nofn_parameter + str(c) + '=' + \
- fullname(first, last)
+ s += f' | {nofn_parameter}{c}={fullname(first, last)}'
return s
def names1para(translators, para):
"""Take list of names. Return the string to be appended to citation."""
- s = ' | ' + para + '='
+ s = f' | {para}='
c = 0
for first, last in translators:
c += 1
if c == 1:
s += fullname(first, last)
elif c == len(translators):
- s += ', and ' + fullname(first, last)
+ s += f', and {fullname(first, last)}'
else:
- s += ', ' + fullname(first, last)
+ s += f', {fullname(first, last)}'
return s
def fullname(first: str, last: str) -> str:
if first:
- return first + ' ' + last
+ return f'{first} {last}'
return last
diff --git a/lib/generator_fa.py b/lib/generator_fa.py
index abc023d1..be6439c9 100644
--- a/lib/generator_fa.py
+++ b/lib/generator_fa.py
@@ -1,6 +1,3 @@
-#! /usr/bin/python
-# -*- coding: utf-8 -*-
-
"""Codes required to create citation templates for wikifa."""
@@ -63,6 +60,7 @@
'journal': 'ژورنال',
# https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id=22368089&retmode=json&tool=my_tool&email=my_email@example.com
'Journal Article': 'ژورنال',
+ 'article-journal': 'ژورنال',
'component': '',
'book-chapter': 'کتاب',
'report-series': 'report',
@@ -76,6 +74,7 @@
'book-series': 'کتاب',
'edited-book': 'کتاب',
'standard-series': '',
+ 'jour': 'ژورنال',
}.get
# According to https://en.wikipedia.org/wiki/Help:Footnotes,
@@ -87,9 +86,8 @@
def sfn_cit_ref(d: defaultdict) -> tuple:
- """Create citation templates using the given dictionary."""
- cite_type = TYPE_TO_CITE(d['cite_type'])
- if not cite_type:
+ """Return sfn, citation, and ref."""
+ if not (cite_type := TYPE_TO_CITE(d['cite_type'])):
logger.warning('Unknown citation type: %s, d: %s', cite_type, d)
cite_type = ''
if cite_type in ('کتاب', 'ژورنال', 'وب'):
@@ -97,8 +95,7 @@ def sfn_cit_ref(d: defaultdict) -> tuple:
else:
return en_citations(d)
- authors = d['authors']
- if authors:
+ if authors := d['authors']:
cit += names2para(authors, 'نام', 'نام خانوادگی', 'نویسنده')
sfn = '<ref>{{پک'
for first, last in authors[:4]:
@@ -106,22 +103,17 @@ def sfn_cit_ref(d: defaultdict) -> tuple:
else:
sfn = '<ref>{{پک/بن'
- editors = d['editors']
- if editors:
+ if editors := d['editors']:
cit += names2para(
- editors, 'نام ویراستار', 'نام خانوادگی ویراستار', 'ویراستار'
- )
+ editors, 'نام ویراستار', 'نام خانوادگی ویراستار', 'ویراستار')
- translators = d['translators']
- if translators:
+ if translators := d['translators']:
cit += names1para(translators, 'ترجمه')
- others = d['others']
- if others:
+ if others := d['others']:
cit += names1para(others, 'دیگران')
- year = d['year']
- if year:
+ if year := d['year']:
sfn += ' | ' + year
if cite_type == 'book':
@@ -148,81 +140,67 @@ def sfn_cit_ref(d: defaultdict) -> tuple:
else:
website = d['website']
if website:
- cit += ' | وبگاه=' + website
+ cit += ' | وبگاه=' + website
- chapter = d['chapter']
- if chapter:
+ if chapter := d['chapter']:
cit += ' | فصل=' + chapter
- publisher = d['publisher'] or d['organization']
- if publisher:
+ if publisher := (d['publisher'] or d['organization']):
cit += ' | ناشر=' + publisher
- address = d['address'] or d['publisher-location']
- if address:
+ if address := (d['address'] or d['publisher-location']):
cit += ' | مکان=' + address
- edition = d['edition']
- if edition:
+ if edition := d['edition']:
cit += ' | ویرایش=' + edition
- series = d['series']
- if series:
+ if series := d['series']:
cit += ' | سری=' + series
- volume = d['volume']
- if volume:
+ if volume := d['volume']:
cit += ' | جلد=' + volume
- issue = d['issue'] or d['number']
- if issue:
+ if issue := (d['issue'] or d['number']):
cit += ' | شماره=' + issue
- ddate = d['date']
- if ddate:
+ if ddate := d['date']:
if isinstance(ddate, str):
cit += ' | تاریخ=' + ddate
else:
cit += ' | تاریخ=' + date.isoformat(ddate)
-
- if year:
+ elif year:
cit += ' | سال=' + year
- month = d['month']
- if month:
- cit += ' | ماه=' + month
-
- isbn = d['isbn']
- if isbn:
+ if isbn := d['isbn']:
cit += ' | شابک=' + isbn
- issn = d['issn']
- if issn:
+ if issn := d['issn']:
cit += ' | issn=' + issn
- pmid = d['pmid']
- if pmid:
+ if pmid := d['pmid']:
cit += ' | pmid=' + pmid
- pmcid = d['pmcid']
- if pmcid:
+ if pmcid := d['pmcid']:
cit += ' | pmc=' + pmcid
- doi = d['doi']
- if doi:
+ if doi := d['doi']:
cit += ' | doi=' + doi
- oclc = d['oclc']
- if oclc:
+ if oclc := d['oclc']:
cit += ' | oclc=' + oclc
+ if jstor := d['jstor']:
+ cit += f' | jstor={jstor}'
+ jstor_access = d['jstor-access']
+ if jstor_access:
+ cit += f' | jstor-access=free'
+
pages = d['page']
if cite_type == 'ژورنال':
if pages:
cit += ' | صفحه=' + pages
- url = d['url']
- if url:
+ if url := d['url']:
# Don't add a DOI URL if we already have added a DOI.
if not doi or not DOI_URL_MATCH(url):
cit += ' | پیوند=' + url
@@ -230,33 +208,29 @@ def sfn_cit_ref(d: defaultdict) -> tuple:
# To prevent addition of access date
url = None
- archive_url = d['archive-url']
- if archive_url:
+ if archive_url := d['archive-url']:
cit += (
- ' | پیوند بایگانی=' + archive_url +
- ' | تاریخ بایگانی=' + d['archive-date'].isoformat() +
- ' | پیوند مرده=' + ('آری' if d['dead-url'] == 'yes' else 'نه')
- )
+ f' | پیوند بایگانی={archive_url}'
+ f' | تاریخ بایگانی={d["archive-date"].isoformat()}'
+ f" | پیوند مرده={('آری' if d['url-status'] == 'yes' else 'نه')}")
- language = d['language']
- if language:
+ if language := d['language']:
language = TO_TWO_LETTER_CODE(language.lower(), language)
if cite_type == 'وب':
- cit += ' | کد زبان=' + language
+ cit += f' | کد زبان={language}'
else:
- cit += ' | زبان=' + language
- sfn += ' | زبان=' + language
+ cit += f' | زبان={language}'
+ sfn += f' | زبان={language}'
if pages:
- sfn += ' | ص=' + pages
+ sfn += f' | ص={pages}'
# Seed the random generator before adding today's date.
randseed(cit)
ref_name = (
randchoice(ascii_lowercase) # it should contain at least one non-digit
- + ''.join(randchoice(LOWER_ALPHA_DIGITS) for _ in range(4))
- )
+ + ''.join(randchoice(LOWER_ALPHA_DIGITS) for _ in range(4)))
if url:
- cit += ' | تاریخ بازبینی=' + date.today().isoformat()
+ cit += f' | تاریخ بازبینی={date.today().isoformat()}'
if not pages and cite_type != 'وب':
sfn += ' | ص='
@@ -266,10 +240,10 @@ def sfn_cit_ref(d: defaultdict) -> tuple:
# Finally create the ref tag.
ref = cit[2:]
if pages and ' | صفحه=' not in ref:
- ref = ref[:-2] + ' | صفحه=' + pages + '}}'
+ ref = f'{ref[:-2]} | صفحه={pages}}}}}'
elif not url:
- ref = ref[:-2] + ' | صفحه=}}'
- ref = '<ref name="' + ref_name + '">' + ref + '\u200F</ref>'
+ ref = f'{ref[:-2]} | صفحه=}}}}'
+ ref = f'<ref name="{ref_name}">{ref}\u200F</ref>'
return sfn, cit, ref
@@ -282,39 +256,35 @@ def names2para(names, fn_parameter, ln_parameter, nofn_parameter=None):
if c == 1:
if first or not nofn_parameter:
s += (
- ' | ' + ln_parameter + '=' + last +
- ' | ' + fn_parameter + '=' + first
- )
+ f' | {ln_parameter}=' + last +
+ f' | {fn_parameter}=' + first)
else:
- s += ' | ' + nofn_parameter + '=' + fullname(first, last)
+ s += f' | {nofn_parameter}=' + fullname(first, last)
else:
if first or not nofn_parameter:
s += (
- ' | ' + ln_parameter + str(c).translate(DIGITS_TO_FA)
- + '=' + last +
- ' | ' + fn_parameter + str(c).translate(DIGITS_TO_FA)
- + '=' + first
- )
+ f' | {ln_parameter}{str(c).translate(DIGITS_TO_FA)}'
+ f'={last} | {fn_parameter}{str(c).translate(DIGITS_TO_FA)}'
+ f'={first}')
else:
s += (
- ' | ' + nofn_parameter + str(c).translate(DIGITS_TO_FA)
- + '=' + fullname(first, last)
- )
+ f' | {nofn_parameter}{str(c).translate(DIGITS_TO_FA)}'
+ f'={fullname(first, last)}')
return s
def names1para(translators, para):
"""Take list of names. Return the string to be appended to citation."""
- s = ' | ' + para + '='
+ s = f' | {para}='
c = 0
for first, last in translators:
c += 1
if c == 1:
s += fullname(first, last)
elif c == len(translators):
- s += ' و ' + fullname(first, last)
+ s += f' و {fullname(first, last)}'
else:
- s += '، ' + fullname(first, last)
+ s += f'، {fullname(first, last)}'
return s
diff --git a/lib/googlebooks.py b/lib/googlebooks.py
index bc09a133..e5729c7e 100644
--- a/lib/googlebooks.py
+++ b/lib/googlebooks.py
@@ -1,59 +1,30 @@
-#! /usr/bin/python
-# -*- coding: utf-8 -*-
-
-"""All things specifically related to the Google Books website."""
-
-
from urllib.parse import parse_qs
-from urllib.parse import urlparse
from langid import classify
-# import bibtex [1]
-from lib.commons import fetch
-from lib.ris import parse as ris_parse
-from lib.commons import dict_to_sfn_cit_ref
+from lib.commons import request
+from lib.ris import ris_parse
-def googlebooks_sfn_cit_ref(url, date_format='%Y-%m-%d') -> tuple:
+def url_to_dict(parsed_url, date_format='%Y-%m-%d') -> dict:
"""Create the response namedtuple."""
- # bibtex_result = get_bibtex(url) [1]
- # dictionary = bibtex.parse(bibtex_result) [1]
- dictionary = ris_parse(get_ris(url))
+ parsed_query = parse_qs(parsed_url.query)
+
+ if (id_ := parsed_query.get('id')) is not None:
+ volume_id = id_[0]
+ else: # the new URL format
+ volume_id = parsed_url.path.rpartition('/')[2]
+
+ dictionary = ris_parse(request(
+ f'https://{parsed_url.netloc}/books/download/?id={volume_id}'
+ f'&output=ris', spoof=True).content.decode('utf8'))
dictionary['date_format'] = date_format
- pu = urlparse(url)
- pq = parse_qs(pu.query)
- # default domain is prefered:
- dictionary['url'] = 'https://' + pu.netloc + '/books?id=' + pq['id'][0]
# manually adding page number to dictionary:
- if 'pg' in pq:
- dictionary['page'] = pq['pg'][0][2:]
- dictionary['url'] += '&pg=' + pq['pg'][0]
+ if (pg := parsed_query.get('pg')) is not None:
+ pg0 = pg[0]
+ dictionary['page'] = pg0[2:]
+ dictionary['url'] += f'&pg={pg0}'
# although google does not provide a language field:
if not dictionary['language']:
dictionary['language'] = classify(dictionary['title'])[0]
- return dict_to_sfn_cit_ref(dictionary)
-
-
-def get_bibtex(googlebook_url) -> bytes:
- """Get bibtex file content from a noormags url."""
- # getting id:
- pu = urlparse(googlebook_url)
- pq = parse_qs(pu.query)
- bookid = pq['id'][0]
- url = 'http://books.google.com/books/download/?id=' +\
- bookid + '&output=bibtex'
- # Agent spoofing is needed, otherwise: HTTP Error 401: Unauthorized
- return fetch(url, spoof=True, timeout=10).content
-
-
-def get_ris(googlebook_url):
- """Get ris file content from a noormags url."""
- # getting id:
- pu = urlparse(googlebook_url)
- pq = parse_qs(pu.query)
- bookid = pq['id'][0]
- url = 'http://books.google.com/books/download/?id=' +\
- bookid + '&output=ris'
- # Agent spoofing is needed, otherwise: HTTP Error 401: Unauthorized
- return fetch(url, spoof=True).text
+ return dictionary
diff --git a/lib/html/en.html b/lib/html/en.html
index 17c1db6a..df30c33c 100644
--- a/lib/html/en.html
+++ b/lib/html/en.html
@@ -1,4 +1,5 @@
+
Citer
@@ -17,11 +18,11 @@
Date format:
- 2017-01-01
- January 1, 2017
- Jan 1, 2017
- 1 January 2017
- 1 Jan 2017
+ 2020-01-01
+ January 1, 2020
+ Jan 1, 2020
+ 1 January 2020
+ 1 Jan 2020
Shortened footnote and citation:
@@ -36,13 +37,13 @@
Google Books URL , DOI , ISBN , PMID , PMCID , OCLC number, or the URL of many major news websites.
- Note that there is always a chance of error in the generated output. Please check the results before using them on Wiki .
+ Note that there is always a chance of error in the generated output. Please check the results before using them on wiki .
- Found a bug or have a suggestion? Contact me on my talk page or open an issue on github .
- Bookmarklet
+ Found a bug or have a suggestion? Contact me on my talk page or open an issue on GitHub .
+ Bookmarklet (drag to favorites bar)
-
\ No newline at end of file
+