diff --git a/py/src/clippy/backends/fs/execution.py b/py/src/clippy/backends/fs/execution.py index 023935f..b5f0dac 100644 --- a/py/src/clippy/backends/fs/execution.py +++ b/py/src/clippy/backends/fs/execution.py @@ -4,13 +4,13 @@ from __future__ import annotations +import collections import contextlib import json import logging import os import select import subprocess -import sys from ... import cfg from ...clippy_types import AnyDict @@ -44,6 +44,8 @@ def _stream_exec( already be set. """ + full_stderr: bool = cfg.get("full_stderr") == "YES" + logger.debug(f"Submission = {submission_dict}") # PP support passing objects # ~ cmd_stdin = json.dumps(submission_dict) @@ -103,15 +105,16 @@ def _stream_exec( try: d = json.loads(line, object_hook=decode_clippy_json) except json.JSONDecodeError: - warning = f"Warning: invalid JSON on stdout: {line!r}" - stderr_lines.append(warning + "\n") - print(warning, file=sys.stderr, flush=True) + # warning = f"Warning: invalid JSON on stdout: {line!r}" + # logger.debug(warning + "\n") + # if it's not valid JSON, let's print it. + print(line, flush=True) elif fd == stderr_fd: stderr_buffer += text while "\n" in stderr_buffer: line, stderr_buffer = stderr_buffer.split("\n", 1) stderr_lines.append(line + "\n") - print(line, flush=True) + # print(line, flush=True) except BlockingIOError: # No data available right now continue @@ -128,9 +131,23 @@ def _stream_exec( if stderr_buffer.strip(): stderr_lines.append(stderr_buffer) - print(stderr_buffer.rstrip(), flush=True) - - stderr = "".join(stderr_lines) if stderr_lines else None + # print(stderr_buffer.rstrip(), flush=True) + + if not stderr_lines: + stderr = None # type: ignore + else: # we have stderr - process it + if not full_stderr: # deduplicate + logger.debug("stderr deduplication enabled") + # this relies on defaultdict preserving insertion order. + stderr_map: dict[str, int] = collections.defaultdict(int) + for line in stderr_lines: + stderr_map[line] += 1 + stderr = "".join(stderr_map.keys()) + else: # use all the lines + stderr = "".join(stderr_lines) + + if stderr is not None: + print(stderr) if progress is not None: progress.close() # if proc.returncode: diff --git a/py/src/clippy/config.py b/py/src/clippy/config.py index 85d5834..24c092f 100644 --- a/py/src/clippy/config.py +++ b/py/src/clippy/config.py @@ -26,4 +26,5 @@ "%(asctime)s [%(filename)s:%(lineno)d (%(funcName)s) %(levelname)s: %(message)s", ), "logname": ("CLIPPY_LOGNAME", __name__), + "full_stderr": ("CLIPPY_FULL_STDERR", "YES"), } diff --git a/test/run_ipython.sh b/test/run_ipython.sh index 849661c..ee46a2c 100755 --- a/test/run_ipython.sh +++ b/test/run_ipython.sh @@ -3,4 +3,4 @@ PROJ_ROOT_DIR=$(pwd)/.. CPP_BUILD_DIR=$PROJ_ROOT_DIR/cpp/build export PYTHONPATH=$PROJ_ROOT_DIR/py/src:$PYTHONPATH export CLIPPY_BACKEND_PATH=$CPP_BUILD_DIR/examples -CLIPPY_LOGLEVEL=DEBUG ipython +CLIPPY_LOGLEVEL=CRITICAL ipython diff --git a/test/test_clippy.py b/test/test_clippy.py index a274032..27d8592 100644 --- a/test/test_clippy.py +++ b/test/test_clippy.py @@ -211,7 +211,3 @@ def test_graph(examplegraph): assert "c" in c_e_only and "e" in c_e_only and len(c_e_only) == 2 -def test_noise(examplefn, capsys): - assert examplefn.returns_noisy_int() == 42 - captured = capsys.readouterr() - assert "Warning: invalid JSON on stdout" in captured.err