From 52b2f869ab9a77339f872e9d0d8954b60a56dac1 Mon Sep 17 00:00:00 2001 From: Blondel MONDESIR Date: Fri, 22 Mar 2024 05:18:11 -0400 Subject: [PATCH 1/7] Add lb search support --- scripts/lb-wrapper | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/scripts/lb-wrapper b/scripts/lb-wrapper index a779492ab0..31d7337d08 100755 --- a/scripts/lb-wrapper +++ b/scripts/lb-wrapper @@ -1,7 +1,7 @@ #!/bin/bash XKLB_INTERNAL_CMD="$1" # e.g. "tubeadd" or "dl" -URL="$2" +URL_OR_SEARCH_TERM="$2" LOG_FILE="/var/log/xklb.log" XKLB_EXECUTABLE="${XKLB_EXECUTABLE:-lb}" @@ -24,7 +24,7 @@ log() { } if [ $# -ne 2 ]; then - log "Error" "Two arguments are required. Please provide a command (tubeadd/dl) and a URL." + log "Error" "Two arguments are required. Please provide a command (tubeadd/dl/search) and a URL or search term." exit 1 fi @@ -47,9 +47,11 @@ fi # fetching metadata. This will prevent hanging for playlist URLs or short URLs. # "...to be able to list videos that are not downloaded yet" if [[ $XKLB_INTERNAL_CMD == "tubeadd" ]]; then - xklb_full_cmd="${XKLB_EXECUTABLE} tubeadd ${XKLB_DB_FILE} ${URL} --force ${VERBOSITY}" + xklb_full_cmd="${XKLB_EXECUTABLE} tubeadd ${XKLB_DB_FILE} ${URL_OR_SEARCH_TERM} --force --extra ${VERBOSITY}" elif [[ $XKLB_INTERNAL_CMD == "dl" ]]; then - xklb_full_cmd="${XKLB_EXECUTABLE} dl ${XKLB_DB_FILE} --prefix ${TMP_DOWNLOADS_DIR} --video --search ${URL} ${FORMAT_OPTIONS} --write-thumbnail ${VERBOSITY}" + xklb_full_cmd="${XKLB_EXECUTABLE} dl ${XKLB_DB_FILE} --prefix ${TMP_DOWNLOADS_DIR} --video --search ${URL_OR_SEARCH_TERM} ${FORMAT_OPTIONS} --write-thumbnail --subs ${VERBOSITY}" +elif [[ $XKLB_INTERNAL_CMD == "search" ]]; then + xklb_full_cmd="${XKLB_EXECUTABLE} search ${XKLB_DB_FILE} ${URL_OR_SEARCH_TERM}" else log "Error" "Invalid xklb command. Please choose 'tubeadd' or 'dl'." exit 1 @@ -60,8 +62,11 @@ log "Info" "Running xklb command: ${xklb_full_cmd}" # >(...) "process substitution" explained at https://unix.stackexchange.com/a/324170 # 1>&2 redirect back-to-STDERR to avoid nested (repeat) logging, explained at https://stackoverflow.com/a/15936384 eval "${xklb_full_cmd}" \ - > >(while read -r line; do if [[ $line == downloading* || $line =~ \[https://.*\]:* ]]; then echo "$line"; else log "Info" "$line"; fi; done) \ - 2> >(while read -r line; do if [[ $line == downloading* || $line =~ \[https://.*\]:* ]]; then echo "$line"; else log "Debug" "$line" 1>&2; fi; done) & + # > >(while read -r line; do if [[ $line == downloading* || $line =~ \[https://.*\]:* ]]; then echo "$line"; else log "Info" "$line"; fi; done) \ + # 2> >(while read -r line; do if [[ $line == downloading* || $line =~ \[https://.*\]:* ]]; then echo "$line"; else log "Debug" "$line" 1>&2; fi; done) & + # adjust above line to also reroute the return code of the underlying command as a line saying "return code: 0" or "return code: 1" or "return code: 2" etc. + > >(while read -r line; do if [[ $line == downloading* || $line =~ \[https://.*\]:* ]] || [[ $line == "yt_dlp_rc: "* ]]; then echo "$line"; else log "Info" "$line"; fi; done) \ + 2> >(while read -r line; do if [[ $line == downloading* || $line =~ \[https://.*\]:* ]]; then echo "$line"; else log "Debug" "$line" 1>&2; fi; done; echo "yt_dlp_rc: $?" 1>&2) & # 2024-01-11: HOW THIS WORKS... # 0) xklb sends a flood of yt-dlp status message lines like "downloading 59.8% 2.29MiB/s" to STDERR. # 1) Then, "2> >(...)" reroutes (only those raw lines!) to STDIN, instead of logging them (as "Debug" lines). @@ -77,9 +82,17 @@ wait $pid rc=$? # Check return code (exit status) -if [ $rc -eq 0 ]; then - log "Info" "lb-wrapper's xklb command (${XKLB_INTERNAL_CMD}) completed successfully." -else - log "Error" "Error $rc occurred while running lb-wrapper's xklb commands." - exit 1 -fi +case $rc in + 0) + log "Info" "lb-wrapper's xklb command (${XKLB_INTERNAL_CMD}) completed successfully." + exit 0 + ;; + 1) + log "Error" "Error $rc occurred while running lb-wrapper's xklb commands." + exit 1 + ;; + *) + log "Error" "Unknown error occurred while running lb-wrapper's xklb commands. Return code: $rc" + exit $rc + ;; +esac From 7cbc16e175621b87d3cbc8deb9f68d9cc185493d Mon Sep 17 00:00:00 2001 From: Blondel MONDESIR Date: Fri, 22 Mar 2024 05:27:22 -0400 Subject: [PATCH 2/7] Support searching through subtitles --- cps/search.py | 80 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 73 insertions(+), 7 deletions(-) diff --git a/cps/search.py b/cps/search.py index f214b3a8b6..a30ce801f0 100644 --- a/cps/search.py +++ b/cps/search.py @@ -15,9 +15,11 @@ # along with this program. If not, see . import json +import os +import re from datetime import datetime -from flask import Blueprint, request, redirect, url_for, flash +from flask import Blueprint, request, redirect, url_for, flash, jsonify from flask import session as flask_session from flask_login import current_user from flask_babel import format_date @@ -26,6 +28,8 @@ from sqlalchemy.sql.functions import coalesce from . import logger, db, calibre_db, config, ub +from .constants import XKLB_DB_FILE +from .subproc_wrapper import process_open from .usermanagement import login_required_if_no_ano from .render_template import render_title_template from .pagination import Pagination @@ -38,16 +42,53 @@ @search.route("/search", methods=["GET"]) @login_required_if_no_ano def simple_search(): - term = request.args.get("query") - if term: - return redirect(url_for('web.books_list', data="search", sort_param='stored', query=term.strip())) - else: + try: + term = request.args.get("query") + + if not term: + return render_title_template('search.html', + searchterm="", + result_count=0, + title=_("Search"), + page="search") + + # Perform CLI search against xklb-metadata.db + video_titles = lb_search(term) + log.debug(f"CLI search results: {video_titles}") + + # Fetch additional results from the Calibre database using the video titles + primary_results = [] + for title in video_titles: + results = calibre_db.get_search_results(title, config, offset=None, order=[db.Books.sort.desc() if hasattr(db.Books, 'sort') else db.Books.id], limit=None)[0] + primary_results.extend(results) + log.debug(f"CLI search results for '{title}': {len(results)}") + + # Fetch results from Calibre database using the search term + join = (db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series) + offset = request.args.get("offset", 0, type=int) + order = [db.Books.sort] if not request.args.get("sort_param") else request.args.get("sort_param") + entries, result_count, pagination = calibre_db.get_search_results(term, config, offset, *join, order=order) + + # Combine the two sets of results + entries.extend(primary_results) + result_count += len(primary_results) + return render_title_template('search.html', - searchterm="", - result_count=0, + searchterm=term, + pagination=pagination, + query=term, + adv_searchterm=term, + entries=entries, + result_count=result_count, title=_("Search"), page="search") + except Exception as e: + log.error(f"Error in simple_search: {e}") + # log the traceback + log.exception(e) + return jsonify({"error": "An error occurred while processing your request."}), 500 + @search.route("/advsearch", methods=['POST']) @login_required_if_no_ano @@ -401,3 +442,28 @@ def render_search_results(term, offset=None, order=None, limit=None): order=order[1]) +def lb_search(term): + # perform CLI search against xklb-metadata.db + video_titles = [] + lb_executable = os.getenv("LB_WRAPPER", "lb-wrapper") + + if term: + subprocess_args = [lb_executable, "search", term] + log.debug("Executing: %s", subprocess_args) + + try: + p = process_open(subprocess_args, newlines=True) + stdout, stderr = p.communicate() + + if p.returncode != 0: + log.error("Error executing lb-wrapper: %s", stderr) + return video_titles + + # regex pattern to extract video titles + pattern = r"^[^\d\n].*?(?= - )" + matches = re.findall(pattern, stdout, re.MULTILINE) + video_titles.extend(matches) + except Exception as ex: + log.error("Error executing lb-wrapper: %s", ex) + + return video_titles From b5c72ca3b1232ce5aedcaf8886bba949d27fcd4b Mon Sep 17 00:00:00 2001 From: Blondel MONDESIR Date: Fri, 22 Mar 2024 05:32:28 -0400 Subject: [PATCH 3/7] revert search.py changes --- cps/search.py | 80 +++++---------------------------------------------- 1 file changed, 7 insertions(+), 73 deletions(-) diff --git a/cps/search.py b/cps/search.py index a30ce801f0..f214b3a8b6 100644 --- a/cps/search.py +++ b/cps/search.py @@ -15,11 +15,9 @@ # along with this program. If not, see . import json -import os -import re from datetime import datetime -from flask import Blueprint, request, redirect, url_for, flash, jsonify +from flask import Blueprint, request, redirect, url_for, flash from flask import session as flask_session from flask_login import current_user from flask_babel import format_date @@ -28,8 +26,6 @@ from sqlalchemy.sql.functions import coalesce from . import logger, db, calibre_db, config, ub -from .constants import XKLB_DB_FILE -from .subproc_wrapper import process_open from .usermanagement import login_required_if_no_ano from .render_template import render_title_template from .pagination import Pagination @@ -42,53 +38,16 @@ @search.route("/search", methods=["GET"]) @login_required_if_no_ano def simple_search(): - try: - term = request.args.get("query") - - if not term: - return render_title_template('search.html', - searchterm="", - result_count=0, - title=_("Search"), - page="search") - - # Perform CLI search against xklb-metadata.db - video_titles = lb_search(term) - log.debug(f"CLI search results: {video_titles}") - - # Fetch additional results from the Calibre database using the video titles - primary_results = [] - for title in video_titles: - results = calibre_db.get_search_results(title, config, offset=None, order=[db.Books.sort.desc() if hasattr(db.Books, 'sort') else db.Books.id], limit=None)[0] - primary_results.extend(results) - log.debug(f"CLI search results for '{title}': {len(results)}") - - # Fetch results from Calibre database using the search term - join = (db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series) - offset = request.args.get("offset", 0, type=int) - order = [db.Books.sort] if not request.args.get("sort_param") else request.args.get("sort_param") - entries, result_count, pagination = calibre_db.get_search_results(term, config, offset, *join, order=order) - - # Combine the two sets of results - entries.extend(primary_results) - result_count += len(primary_results) - + term = request.args.get("query") + if term: + return redirect(url_for('web.books_list', data="search", sort_param='stored', query=term.strip())) + else: return render_title_template('search.html', - searchterm=term, - pagination=pagination, - query=term, - adv_searchterm=term, - entries=entries, - result_count=result_count, + searchterm="", + result_count=0, title=_("Search"), page="search") - except Exception as e: - log.error(f"Error in simple_search: {e}") - # log the traceback - log.exception(e) - return jsonify({"error": "An error occurred while processing your request."}), 500 - @search.route("/advsearch", methods=['POST']) @login_required_if_no_ano @@ -442,28 +401,3 @@ def render_search_results(term, offset=None, order=None, limit=None): order=order[1]) -def lb_search(term): - # perform CLI search against xklb-metadata.db - video_titles = [] - lb_executable = os.getenv("LB_WRAPPER", "lb-wrapper") - - if term: - subprocess_args = [lb_executable, "search", term] - log.debug("Executing: %s", subprocess_args) - - try: - p = process_open(subprocess_args, newlines=True) - stdout, stderr = p.communicate() - - if p.returncode != 0: - log.error("Error executing lb-wrapper: %s", stderr) - return video_titles - - # regex pattern to extract video titles - pattern = r"^[^\d\n].*?(?= - )" - matches = re.findall(pattern, stdout, re.MULTILINE) - video_titles.extend(matches) - except Exception as ex: - log.error("Error executing lb-wrapper: %s", ex) - - return video_titles From 1e257e8112d5501fdebf0f7629b85b002b455265 Mon Sep 17 00:00:00 2001 From: Blondel MONDESIR Date: Thu, 28 Mar 2024 11:20:15 -0400 Subject: [PATCH 4/7] Align with the "Top 100" feature --- scripts/lb-wrapper | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/lb-wrapper b/scripts/lb-wrapper index 31d7337d08..ca220778b8 100755 --- a/scripts/lb-wrapper +++ b/scripts/lb-wrapper @@ -47,7 +47,7 @@ fi # fetching metadata. This will prevent hanging for playlist URLs or short URLs. # "...to be able to list videos that are not downloaded yet" if [[ $XKLB_INTERNAL_CMD == "tubeadd" ]]; then - xklb_full_cmd="${XKLB_EXECUTABLE} tubeadd ${XKLB_DB_FILE} ${URL_OR_SEARCH_TERM} --force --extra ${VERBOSITY}" + xklb_full_cmd="${XKLB_EXECUTABLE} tubeadd ${XKLB_DB_FILE} ${URL_OR_SEARCH_TERM} --force ${VERBOSITY}" elif [[ $XKLB_INTERNAL_CMD == "dl" ]]; then xklb_full_cmd="${XKLB_EXECUTABLE} dl ${XKLB_DB_FILE} --prefix ${TMP_DOWNLOADS_DIR} --video --search ${URL_OR_SEARCH_TERM} ${FORMAT_OPTIONS} --write-thumbnail --subs ${VERBOSITY}" elif [[ $XKLB_INTERNAL_CMD == "search" ]]; then From 950637200e2043f1bae5ac1c1cc5a68a8ea1cc94 Mon Sep 17 00:00:00 2001 From: Blondel MONDESIR Date: Thu, 2 May 2024 16:57:05 -0400 Subject: [PATCH 5/7] Use if statement to handle exit code --- scripts/lb-wrapper | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/scripts/lb-wrapper b/scripts/lb-wrapper index ca220778b8..3d7439478f 100755 --- a/scripts/lb-wrapper +++ b/scripts/lb-wrapper @@ -82,17 +82,9 @@ wait $pid rc=$? # Check return code (exit status) -case $rc in - 0) - log "Info" "lb-wrapper's xklb command (${XKLB_INTERNAL_CMD}) completed successfully." - exit 0 - ;; - 1) - log "Error" "Error $rc occurred while running lb-wrapper's xklb commands." - exit 1 - ;; - *) - log "Error" "Unknown error occurred while running lb-wrapper's xklb commands. Return code: $rc" - exit $rc - ;; -esac +if [ $rc -eq 0 ]; then + log "Info" "lb-wrapper's xklb command (${XKLB_INTERNAL_CMD}) completed successfully." +else + log "Error" "Error $rc occurred while running lb-wrapper's xklb commands." + exit $rc +fi From e28f6001f991679ec1fb13cb394c69fe9ecd0d6f Mon Sep 17 00:00:00 2001 From: Blondel MONDESIR Date: Fri, 3 May 2024 20:38:52 -0400 Subject: [PATCH 6/7] Remove XKLB_EXECUTABLE variable --- scripts/lb-wrapper | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/lb-wrapper b/scripts/lb-wrapper index 580064fdaa..e8dd0447b2 100755 --- a/scripts/lb-wrapper +++ b/scripts/lb-wrapper @@ -53,11 +53,11 @@ log "Info" "Using yt-dlp $(yt-dlp --version)" # fetching metadata. This will prevent hanging for playlist URLs or short URLs. # "...to be able to list videos that are not downloaded yet" if [[ $XKLB_INTERNAL_CMD == "tubeadd" ]]; then - xklb_full_cmd="${XKLB_EXECUTABLE} tubeadd ${XKLB_DB_FILE} ${URL_OR_SEARCH_TERM} --force ${VERBOSITY}" + xklb_full_cmd="lb tubeadd ${XKLB_DB_FILE} ${URL_OR_SEARCH_TERM} --force ${VERBOSITY}" elif [[ $XKLB_INTERNAL_CMD == "dl" ]]; then - xklb_full_cmd="${XKLB_EXECUTABLE} dl ${XKLB_DB_FILE} --prefix ${TMP_DOWNLOADS_DIR} --video --search ${URL_OR_SEARCH_TERM} ${FORMAT_OPTIONS} --write-thumbnail --subs ${VERBOSITY}" + xklb_full_cmd="lb dl ${XKLB_DB_FILE} --prefix ${TMP_DOWNLOADS_DIR} --video --search ${URL_OR_SEARCH_TERM} ${FORMAT_OPTIONS} --write-thumbnail --subs ${VERBOSITY}" elif [[ $XKLB_INTERNAL_CMD == "search" ]]; then - xklb_full_cmd="${XKLB_EXECUTABLE} search ${XKLB_DB_FILE} ${URL_OR_SEARCH_TERM}" + xklb_full_cmd="lb search ${XKLB_DB_FILE} ${URL_OR_SEARCH_TERM}" else log "Error" "Invalid xklb command. Please choose 'tubeadd' or 'dl'." exit 1 From b10d99cc0e2c9506a1e64655403aa5f7995ff58e Mon Sep 17 00:00:00 2001 From: Blondel MONDESIR Date: Fri, 3 May 2024 20:45:43 -0400 Subject: [PATCH 7/7] Remove temporary code --- scripts/lb-wrapper | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scripts/lb-wrapper b/scripts/lb-wrapper index e8dd0447b2..222dedceab 100755 --- a/scripts/lb-wrapper +++ b/scripts/lb-wrapper @@ -68,11 +68,8 @@ log "Info" "Running xklb command: ${xklb_full_cmd}" # >(...) "process substitution" explained at https://unix.stackexchange.com/a/324170 # 1>&2 redirect back-to-STDERR to avoid nested (repeat) logging, explained at https://stackoverflow.com/a/15936384 eval "${xklb_full_cmd}" \ - # > >(while read -r line; do if [[ $line == downloading* || $line =~ \[https://.*\]:* ]]; then echo "$line"; else log "Info" "$line"; fi; done) \ - # 2> >(while read -r line; do if [[ $line == downloading* || $line =~ \[https://.*\]:* ]]; then echo "$line"; else log "Debug" "$line" 1>&2; fi; done) & - # adjust above line to also reroute the return code of the underlying command as a line saying "return code: 0" or "return code: 1" or "return code: 2" etc. - > >(while read -r line; do if [[ $line == downloading* || $line =~ \[https://.*\]:* ]] || [[ $line == "yt_dlp_rc: "* ]]; then echo "$line"; else log "Info" "$line"; fi; done) \ - 2> >(while read -r line; do if [[ $line == downloading* || $line =~ \[https://.*\]:* ]]; then echo "$line"; else log "Debug" "$line" 1>&2; fi; done; echo "yt_dlp_rc: $?" 1>&2) & + > >(while read -r line; do if [[ $line == downloading* || $line =~ \[https://.*\]:* ]]; then echo "$line"; else log "Info" "$line"; fi; done) \ + 2> >(while read -r line; do if [[ $line == downloading* || $line =~ \[https://.*\]:* ]]; then echo "$line"; else log "Debug" "$line" 1>&2; fi; done) & # 2024-01-11: HOW THIS WORKS... # 0) xklb sends a flood of yt-dlp status message lines like "downloading 59.8% 2.29MiB/s" to STDERR. # 1) Then, "2> >(...)" reroutes (only those raw lines!) to STDIN, instead of logging them (as "Debug" lines).