Add citer-cli

.gitignore

@@ -1,4 +1,5 @@
 *.py[cod]
 *.log
-.*
+.docs
 __pycache__/
+config.py

README.md

@@ -1,35 +1,53 @@
 # Citer
 
-A citation generator tool for Wikipedia. Currently accessible from:
-http://tools.wmflabs.org/citer/ (the English version)
-http://tools.wmflabs.org/yadfa/ (the Persian version)
+A citation generator tool for Wikipedia. Currently accessible from:\
+[https://citer.toolforge.org/](https://citer.toolforge.org/) (the English version)\
+[https://yadfa.toolforge.org/](https://yadfa.toolforge.org/) (the Persian version)
 
 ## What does it do?
 
-Citer is specially useful for generating citations from Google Books URLs, DOIs (Any Digital object Identifiers) and ISBNs (International Standard Book Numbers).
-Additionally URL of many major news websites are supported, including:
-The New York Times, BBC, Daily Mail, Daily Mirror, The Daily Telegraph, The Huffington Post, The Washington Post, The Boston Globe, Bloomberg Businessweek, Financial Times, and The Times of India. Sepecial support for the URLs of the [Wayback Machine](https://en.wikipedia.org/wiki/Wayback_Machine) is also implemented.
-
-Some other tested and supported Persian web-sites:
-* http://www.noormags.com (نورمگز)
+Citer is especially useful for generating citations from Google Books URLs, DOIs (Any Digital object Identifiers) and ISBNs (International Standard Book Numbers).
+Additionally, URLs of many major news websites are supported, including:
+
+* The New York Times
+* BBC
+* Daily Mail
+* Daily Mirror
+* The Daily Telegraph
+* The Huffington Post
+* The Washington Post
+* The Boston Globe
+* Bloomberg Businessweek
+* Financial Times
+* The Times of India
+
+Special support for the URLs of the [Wayback Machine](https://en.wikipedia.org/wiki/Wayback_Machine) is also implemented.
+
+Some other tested and supported Persian websites:
+* http://www.noormags.ir (نورمگز)
 * http://www.noorlib.ir (کتابخانه دیجیتال نور)
-* http://www.adinebook.com (آدینه‌بوک)
+* http://www.ketab.ir (خانه كتاب)
 * http://socialhistory.ihcs.ac.ir/ (تحقیقات تاریخ اجتماعی)
 
 
 ## Installation
 
 To run Citer on your local computer:
 
-1. Install Python 3.6+.
-2. Clone the project.
-3. Install the dependencies using `pip install -r requirements.txt`.
-3. Make sure that `flup` is __not__ installed in your environment.
-4. Run Citer by calling `main.py`.
+1. Install Python 3.9+
+2. Clone the project
+3. Install the dependencies using `pip install --user -r requirements.txt`
+4. Copy `config.py.example` to `config.py` (You might want to get an NCBI API key and add it to the config file if you're going to use its services)
+5. Run `python3 app.py`
 
-If everything goes fine, the main page will be accessible from:
-    http://127.0.0.1:5000/
+If there are no warnings or error messages (and no HTML is displayed), the main page will be accessible from:\
+    [http://localhost:5000/](http://localhost:5000/)
 
+If you experience any problems or have questions, please open an issue on this repo.
 
 ## Language Setting
-The default language is English and can be change to Persian using the setting in config.py file.
+The default language is English and can be changed to Persian using the setting in the config.py file.
+
+
+## Known issues
+* The bookmarklet does not work on archive.org (issue #26) or any other website that does not allow opening external links. One needs to use Citer directly in such cases.

app.py

@@ -1,58 +1,76 @@
-#! /usr/bin/python
-# -*- coding: utf-8 -*-
-
 from collections import defaultdict
 from html import unescape
 from logging import getLogger, Formatter, WARNING, INFO
 from logging.handlers import RotatingFileHandler
+from os.path import dirname, abspath
 from urllib.parse import parse_qs, urlparse, unquote
 from wsgiref.headers import Headers
 
-from requests import ConnectionError as RequestsConnectionError
+from requests import ConnectionError as RequestsConnectionError, \
+    JSONDecodeError
 
 from config import LANG
-from lib.adinebook import adinehbook_sfn_cit_ref
-from lib.commons import uninum2en, sfn_cit_ref_to_json
-from lib.doi import doi_sfn_cit_ref, DOI_SEARCH
-from lib.googlebooks import googlebooks_sfn_cit_ref
-from lib.isbn_oclc import (
-    ISBN_10OR13_SEARCH, IsbnError, isbn_sfn_cit_ref, oclc_sfn_cit_ref)
-from lib.noorlib import noorlib_sfn_cit_ref
-from lib.noormags import noormags_sfn_cit_ref
-from lib.pubmed import pmcid_sfn_cit_ref, pmid_sfn_cit_ref
-from lib.urls import urls_sfn_cit_ref
-from lib.waybackmachine import waybackmachine_sfn_cit_ref
+from lib.ketabir import url_to_dict as ketabir_url_to_dict
+from lib.commons import uninum2en, scr_to_json, ISBN_10OR13_SEARCH, \
+    dict_to_sfn_cit_ref, ReturnError
+from lib.doi import doi_to_dict, DOI_SEARCH
+from lib.googlebooks import url_to_dict as google_books_dict
+from lib.isbn_oclc import IsbnError, isbn_to_dict, oclc_dict
+from lib.jstor import url_to_dict as jstor_url_to_dict
+from lib.noorlib import url_to_dict as noorlib_url_to_dict
+from lib.noormags import url_to_dict as noormags_url_to_dict
+from lib.pubmed import pmcid_dict, pmid_dict
+from lib.urls import url_to_dict as urls_url_to_dict
+from lib.waybackmachine import url_to_dict as archive_url_to_dict
 if LANG == 'en':
     from lib.html.en import (
-        DEFAULT_SFN_CIT_REF,
-        UNDEFINED_INPUT_SFN_CIT_REF,
-        HTTPERROR_SFN_CIT_REF,
-        OTHER_EXCEPTION_SFN_CIT_REF,
-        sfn_cit_ref_to_html,
+        DEFAULT_SCR,
+        UNDEFINED_INPUT_SCR,
+        HTTPERROR_SCR,
+        OTHER_EXCEPTION_SCR,
+        scr_to_html,
         CSS,
         CSS_HEADERS,
         JS,
         JS_HEADERS)
 else:
     from lib.html.fa import (
-        DEFAULT_SFN_CIT_REF,
-        UNDEFINED_INPUT_SFN_CIT_REF,
-        HTTPERROR_SFN_CIT_REF,
-        OTHER_EXCEPTION_SFN_CIT_REF,
-        sfn_cit_ref_to_html,
+        DEFAULT_SCR,
+        UNDEFINED_INPUT_SCR,
+        HTTPERROR_SCR,
+        OTHER_EXCEPTION_SCR,
+        scr_to_html,
         CSS,
         CSS_HEADERS)
 
 
+def google_encrypted_dict(url, parsed_url, date_format) -> dict:
+    if parsed_url[2][:7] in {'/books', '/books/'}:
+        # sample urls:
+        # https://encrypted.google.com/books?id=6upvonUt0O8C
+        # https://www.google.com/books?id=bwfoCAAAQBAJ&pg=PA32
+        # https://www.google.com/books/edition/_/bwfoCAAAQBAJ?gbpv=1&pg=PA32
+        return google_books_dict(parsed_url, date_format)
+    return urls_url_to_dict(url, date_format)
+
+
 TLDLESS_NETLOC_RESOLVER = {
-    'adinebook': adinehbook_sfn_cit_ref,
-    'adinehbook': adinehbook_sfn_cit_ref,
-    'noorlib': noorlib_sfn_cit_ref,
-    'noormags': noormags_sfn_cit_ref,
-    'web.archive': waybackmachine_sfn_cit_ref,
-    'web-beta.archive': waybackmachine_sfn_cit_ref,
-    'books.google.co': googlebooks_sfn_cit_ref,
-    'books.google': googlebooks_sfn_cit_ref,
+    'ketab': ketabir_url_to_dict,
+
+    'noorlib': noorlib_url_to_dict,
+    'noormags': noormags_url_to_dict,
+
+    'web.archive': archive_url_to_dict,
+    'web-beta.archive': archive_url_to_dict,
+
+    'books.google.co': google_books_dict,
+    'books.google.com': google_books_dict,
+    'books.google': google_books_dict,
+
+    'google': google_encrypted_dict,
+    'encrypted.google': google_encrypted_dict,
+
+    'jstor': jstor_url_to_dict,
 }.get
 
 RESPONSE_HEADERS = Headers([('Content-Type', 'text/html; charset=UTF-8')])
@@ -65,13 +83,14 @@
 def get_root_logger():
     custom_logger = getLogger()
     custom_logger.setLevel(INFO)
+    srcdir = dirname(abspath(__file__))
     handler = RotatingFileHandler(
-        filename='citer.log',
+        filename=f'{srcdir}/citer.log',
         mode='a',
         maxBytes=20000,
         backupCount=0,
         encoding='utf-8',
-        delay=0)
+    )
     handler.setLevel(INFO)
     handler.setFormatter(
         Formatter('\n%(asctime)s\n%(levelname)s\n%(message)s\n'))
@@ -82,118 +101,129 @@ def get_root_logger():
 LOGGER = get_root_logger()
 
 
-def url_doi_isbn_to_sfn_cit_ref(user_input, date_format) -> tuple:
+def input_to_dict(user_input, date_format, /) -> dict:
     en_user_input = unquote(uninum2en(user_input))
     # Checking the user input for dot is important because
     # the use of dotless domains is prohibited.
     # See: https://features.icann.org/dotless-domains
     if '.' in en_user_input:
         # Try predefined URLs
         # Todo: The following code could be done in threads.
-        if not user_input.startswith('http'):
+        if not (url_input := user_input.startswith('http')):
             url = 'http://' + user_input
         else:
             url = user_input
+        parsed_url = urlparse(url)
         # TLD stands for top-level domain
-        tldless_netloc = urlparse(url)[1].rpartition('.')[0]
-        resolver = TLDLESS_NETLOC_RESOLVER(
+        tldless_netloc = parsed_url[1].rpartition('.')[0]
+        # todo: make lazy?
+        if (to_dict := TLDLESS_NETLOC_RESOLVER(
             tldless_netloc[4:] if tldless_netloc.startswith('www.')
-            else tldless_netloc)
-        if resolver:
-            return resolver(url, date_format)
+            else tldless_netloc
+        )) is not None:
+            if to_dict is google_books_dict:
+                return to_dict(parsed_url, date_format)
+            elif to_dict is google_encrypted_dict:
+                return to_dict(url, parsed_url, date_format)
+            return to_dict(url, date_format)
+
         # DOIs contain dots
-        m = DOI_SEARCH(unescape(en_user_input))
-        if m:
-            return doi_sfn_cit_ref(m.group(1), True, date_format)
-        return urls_sfn_cit_ref(url, date_format)
+        if (m := DOI_SEARCH(unescape(en_user_input))) is not None:
+            try:
+                return doi_to_dict(m[0], True, date_format)
+            except JSONDecodeError:
+                if url_input is False:
+                    raise
+                # continue with urls_scr
+
+        return urls_url_to_dict(url, date_format)
     else:
         # We can check user inputs containing dots for ISBNs, but probably is
-        # error prone.
-        m = ISBN_10OR13_SEARCH(en_user_input)
-        if m:
+        # error-prone.
+        if (m := ISBN_10OR13_SEARCH(en_user_input)) is not None:
             try:
-                return isbn_sfn_cit_ref(m.group(), True, date_format)
+                return isbn_to_dict(m[0], True, date_format)
             except IsbnError:
                 pass
-        return UNDEFINED_INPUT_SFN_CIT_REF
-
+        return UNDEFINED_INPUT_SCR
 
-def app(environ, start_response):
-    query_dict_get = parse_qs(environ['QUERY_STRING']).get
 
+def app(environ: dict, start_response: callable) -> tuple:
     path_info = environ['PATH_INFO']
     if '/static/' in path_info:
-        if path_info.endswith('.css'):
+        if path_info[-4:] == '.css':
             start_response('200 OK', CSS_HEADERS)
-            return [CSS]
+            return CSS,
         else:
             # path_info.endswith('.js') and config.lang == 'en'
             start_response('200 OK', JS_HEADERS)
-            return [JS]
+            return JS,
 
+    query_dict_get = parse_qs(environ['QUERY_STRING']).get
     date_format = query_dict_get('dateformat', [''])[0].strip()
-
     input_type = query_dict_get('input_type', [''])[0]
 
     # Warning: input is not escaped!
-    user_input = query_dict_get('user_input', [''])[0].strip()
-    if not user_input:
-        response_body = sfn_cit_ref_to_html(
-            DEFAULT_SFN_CIT_REF, date_format, input_type
+    if not (user_input := query_dict_get('user_input', [''])[0].strip()):
+        response_body = scr_to_html(
+            DEFAULT_SCR, date_format, input_type
         ).encode()
         RESPONSE_HEADERS['Content-Length'] = str(len(response_body))
         start_response('200 OK', RESPONSE_HEADERS.items())
-        return [response_body]
+        return response_body,
 
     output_format = query_dict_get('output_format', [''])[0]  # apiquery
 
-    resolver = input_type_to_resolver[input_type]
+    to_dict = input_type_to_resolver[input_type]
     # noinspection PyBroadException
     try:
-        response = resolver(user_input, date_format)
+        d = to_dict(user_input, date_format)
     except RequestsConnectionError:
         status = '500 ConnectionError'
         LOGGER.exception(user_input)
         if output_format == 'json':
-            response_body = sfn_cit_ref_to_json(HTTPERROR_SFN_CIT_REF)
+            response_body = scr_to_json(HTTPERROR_SCR)
         else:
-            response_body = sfn_cit_ref_to_html(
-                HTTPERROR_SFN_CIT_REF, date_format, input_type)
-    except Exception:
+            response_body = scr_to_html(
+                HTTPERROR_SCR, date_format, input_type)
+    except Exception as e:
         status = '500 Internal Server Error'
-        LOGGER.exception(user_input)
+
+        if isinstance(e, ReturnError):
+            scr = e.args
+        else:
+            LOGGER.exception(user_input)
+            scr = OTHER_EXCEPTION_SCR
+
         if output_format == 'json':
-            response_body = sfn_cit_ref_to_json(OTHER_EXCEPTION_SFN_CIT_REF)
+            response_body = scr_to_json(scr)
         else:
-            response_body = sfn_cit_ref_to_html(
-                OTHER_EXCEPTION_SFN_CIT_REF, date_format, input_type)
+            response_body = scr_to_html(scr, date_format, input_type)
     else:
         status = '200 OK'
+        scr = dict_to_sfn_cit_ref(d)
         if output_format == 'json':
-            response_body = sfn_cit_ref_to_json(response)
+            response_body = scr_to_json(scr)
         else:
-            response_body = sfn_cit_ref_to_html(
-                response, date_format, input_type)
+            response_body = scr_to_html(scr, date_format, input_type)
     response_body = response_body.encode()
     RESPONSE_HEADERS['Content-Length'] = str(len(response_body))
     start_response(status, RESPONSE_HEADERS.items())
-    return [response_body]
+    return response_body,
 
 
 input_type_to_resolver = defaultdict(
-    lambda: url_doi_isbn_to_sfn_cit_ref, {
-        'url-doi-isbn': url_doi_isbn_to_sfn_cit_ref,
-        'pmid': pmid_sfn_cit_ref,
-        'pmcid': pmcid_sfn_cit_ref,
-        'oclc': oclc_sfn_cit_ref})
+    lambda: input_to_dict, {
+        'url-doi-isbn': input_to_dict,  # todo: can be removed?
+        'pmid': pmid_dict,
+        'pmcid': pmcid_dict,
+        'oclc': oclc_dict})
 
 
 if __name__ == '__main__':
     # note that app.py is not run as '__main__' in kubernetes
-    try:
-        from flup.server.fcgi import WSGIServer
-        WSGIServer(app).run()
-    except ImportError:  # on local computer
-        from wsgiref.simple_server import make_server
-        httpd = make_server('localhost', 5000, app)
-        httpd.serve_forever()
+    # only for local computer
+    from wsgiref.simple_server import make_server
+    httpd = make_server('localhost', 5000, app)
+    print('serving on http://localhost:5000')
+    httpd.serve_forever()