0.9.6: pytest is failing in few units

[kloczek]

I'm packaging your module as an rpm package so I'm using the typical PEP517 based build, install and test cycle used on building packages from non-root account.

• python3 -sBm build -w --no-isolation
• because I'm calling build with --no-isolation I'm using during all processes only locally installed modules
• install .whl file in </install/prefix>
• run pytest with PYTHONPATH pointing to sitearch and sitelib inside </install/prefix>

Here is pytest output:

Details

+ PYTHONPATH=/home/tkloczko/rpmbuild/BUILDROOT/python-nbval-0.9.6-2.fc35.x86_64/usr/lib64/python3.8/site-packages:/home/tkloczko/rpmbuild/BUILDROOT/python-nbval-0.9.6-2.fc35.x86_64/usr/lib/python3.8/site-packages
+ /usr/bin/pytest -ra
=========================================================================== test session starts ============================================================================
platform linux -- Python 3.8.13, pytest-7.1.2, pluggy-1.0.0
rootdir: /home/tkloczko/rpmbuild/BUILD/nbval-0.9.6
plugins: nbval-0.9.6
collected 16 items

issues/67/test_1.py .                                                                                                                                                [  6%]
issues/67/test_2.py FF                                                                                                                                               [ 18%]
issues/7/test_lib.py .                                                                                                                                               [ 25%]
tests/test_collect.py ..                                                                                                                                             [ 37%]
tests/test_coverage.py F                                                                                                                                             [ 43%]
tests/test_ignore.py F                                                                                                                                               [ 50%]
tests/test_plugin.py .                                                                                                                                               [ 56%]
tests/test_timeouts.py .                                                                                                                                             [ 62%]
tests/test_unit_tests_in_notebooks.py ..FF..                                                                                                                         [100%]

================================================================================= FAILURES =================================================================================
__________________________________________________________________________________ test_b __________________________________________________________________________________

    def test_b():
>       raise AssertionError
E       AssertionError

issues/67/test_2.py:2: AssertionError
__________________________________________________________________________________ test_c __________________________________________________________________________________

    def test_c():
>       assert 1 == 2
E       assert 1 == 2

issues/67/test_2.py:5: AssertionError
______________________________________________________________________________ test_coverage _______________________________________________________________________________

testdir = <Testdir local('/tmp/pytest-of-tkloczko/pytest-430/test_coverage0')>

    def test_coverage(testdir):

        testdir.makepyfile(
            # Setup file to cover:
            lib="""
                def mysum(a, b):
                    return a + b
                def myprod(a, b):
                    return a * b
            """,
            # Setup python file to cover mysum function
            test_lib="""
                import lib
                def test_sum():
                    assert lib.mysum(1, 3) == 4
                    assert lib.mysum("cat", "dog") == "catdog"
                    assert lib.mysum(1.5, 2) == 3.5
            """,
        )

        # Setup notebook to cover myprod function
        nb = build_nb([
            "import lib",
            "lib.myprod(1, 3)",
            "lib.myprod(2.5, 2.5)",
            "lib.myprod(2, 'cat')"
        ], mark_run=True)
        add_expected_plaintext_outputs(nb, [
            None, "3", "6.25", "'catcat'"
        ])
        # Write notebook to test dir
        nbformat.write(nb, os.path.join(
            str(testdir.tmpdir), 'test_coverage.ipynb'))

        # Run tests
        result = testdir.runpytest_inprocess('--nbval', '--current-env', '--cov', '.')

        # Check tests went off as they should:
>       assert result.ret == 0
E       assert <ExitCode.USAGE_ERROR: 4> == 0
E        +  where <ExitCode.USAGE_ERROR: 4> = <RunResult ret=ExitCode.USAGE_ERROR len(stdout.lines)=0 len(stderr.lines)=5 duration=0.06s>.ret

/home/tkloczko/rpmbuild/BUILD/nbval-0.9.6/tests/test_coverage.py:52: AssertionError
--------------------------------------------------------------------------- Captured stderr call ---------------------------------------------------------------------------
ERROR: usage: pytest [options] [file_or_dir] [file_or_dir] [...]
pytest: error: unrecognized arguments: --cov
  inifile: None
  rootdir: /tmp/pytest-of-tkloczko/pytest-430/test_coverage0

_________________________________________________________________________ test_conf_ignore_stderr __________________________________________________________________________

testdir = <Testdir local('/tmp/pytest-of-tkloczko/pytest-430/test_conf_ignore_stderr0')>

    def test_conf_ignore_stderr(testdir):

        # Setup test config
        testdir.makeconftest(_ignore_stderr_code)

        # Setup notebook with stream outputs
        nb = build_nb([
            "import sys",
            "sys.stdout.write('test\\n')",
            "sys.stderr.write('error output\\n')",
            "sys.stdout.write('test\\n')\nsys.stderr.write('error output\\n')",
        ], mark_run=True)
        nb.cells[1].outputs.append(nbformat.v4.new_output(
            'stream',
            text=u'test\n',
            ))
        nb.cells[2].outputs.append(nbformat.v4.new_output(
            'stream',
            name='stderr',
            text=u'different error output',
            ))
        nb.cells[3].outputs.append(nbformat.v4.new_output(
            'stream',
            text=u'test\n',
            ))
        nb.cells[3].outputs.append(nbformat.v4.new_output(
            'stream',
            name='stderr',
            text=u'different error output',
            ))

        # Write notebook to test dir
        nbformat.write(nb, os.path.join(
            str(testdir.tmpdir), 'test_ignore.ipynb'))

        # Run tests
        result = testdir.runpytest_subprocess('--nbval', '--current-env', '.')

        # Check tests went off as they should:
>       assert result.ret == 0
E       assert <ExitCode.TESTS_FAILED: 1> == 0
E        +  where <ExitCode.TESTS_FAILED: 1> = <RunResult ret=ExitCode.TESTS_FAILED len(stdout.lines)=52 len(stderr.lines)=0 duration=2.15s>.ret

/home/tkloczko/rpmbuild/BUILD/nbval-0.9.6/tests/test_ignore.py:57: AssertionError
--------------------------------------------------------------------------- Captured stdout call ---------------------------------------------------------------------------
running: /usr/bin/python3 -mpytest --basetemp=/tmp/pytest-of-tkloczko/pytest-430/test_conf_ignore_stderr0/runpytest-0 --nbval --current-env .
     in: /tmp/pytest-of-tkloczko/pytest-430/test_conf_ignore_stderr0
============================= test session starts ==============================
platform linux -- Python 3.8.13, pytest-7.1.2, pluggy-1.0.0
rootdir: /tmp/pytest-of-tkloczko/pytest-430/test_conf_ignore_stderr0
plugins: nbval-0.9.6
collected 4 items

test_ignore.ipynb .FFF                                                   [100%]

=================================== FAILURES ===================================
__________________________ test_ignore.ipynb::Cell 1 ___________________________
Notebook cell execution failed
Cell 1: Cell outputs differ

Input:
sys.stdout.write('test\n')

Traceback:
Unexpected output fields from running code: {'text/plain'}

__________________________ test_ignore.ipynb::Cell 2 ___________________________
Notebook cell execution failed
Cell 2: Cell outputs differ

Input:
sys.stderr.write('error output\n')

Traceback:
Unexpected output fields from running code: {'text/plain'}

__________________________ test_ignore.ipynb::Cell 3 ___________________________
Notebook cell execution failed
Cell 3: Cell outputs differ

Input:
sys.stdout.write('test\n')
sys.stderr.write('error output\n')

Traceback:
Unexpected output fields from running code: {'text/plain'}

=============================== warnings summary ===============================
../../../../usr/lib/python3.8/site-packages/_pytest/nodes.py:633
  /usr/lib/python3.8/site-packages/_pytest/nodes.py:633: PytestRemovedIn8Warning: The (fspath: py.path.local) argument to IPyNbFile is deprecated. Please use the (path: pathlib.Path) argument instead.
  See https://docs.pytest.org/en/latest/deprecations.html#fspath-argument-for-node-constructors-replaced-with-pathlib-path
    return super().from_parent(parent=parent, fspath=fspath, path=path, **kw)

-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
=========================== short test summary info ============================
FAILED test_ignore.ipynb::Cell 1
FAILED test_ignore.ipynb::Cell 2
FAILED test_ignore.ipynb::Cell 3
==================== 3 failed, 1 passed, 1 warning in 1.21s ====================
____________________________ test_print[/home/tkloczko/rpmbuild/BUILD/nbval-0.9.6/tests/ipynb-test-samples/test-latex-pass-correctouput.ipynb] _____________________________

filename = '/home/tkloczko/rpmbuild/BUILD/nbval-0.9.6/tests/ipynb-test-samples/test-latex-pass-correctouput.ipynb', correctoutcome = 'pass'

    @pytest.mark.parametrize("filename, correctoutcome", testdata, ids=testnames)
    def test_print(filename, correctoutcome):

        command = ["py.test", "--nbval", "-v", "--current-env", filename]
        print("Starting parametrized test with filename={}, correctoutcome={}"
              .format(filename, correctoutcome))
        print("Command about to execute is '{}'".format(command))

        if os.name == 'nt':
            exitcode = subprocess.call(command, shell=True)
        else:
            exitcode = subprocess.call(command)

        if correctoutcome == 'pass':
            if exitcode != 0:
>               raise AssertionError("Tests failed on ipynb (expected pass)")
E               AssertionError: Tests failed on ipynb (expected pass)

tests/test_unit_tests_in_notebooks.py:68: AssertionError
--------------------------------------------------------------------------- Captured stdout call ---------------------------------------------------------------------------
Starting parametrized test with filename=/home/tkloczko/rpmbuild/BUILD/nbval-0.9.6/tests/ipynb-test-samples/test-latex-pass-correctouput.ipynb, correctoutcome=pass
Command about to execute is '['py.test', '--nbval', '-v', '--current-env', '/home/tkloczko/rpmbuild/BUILD/nbval-0.9.6/tests/ipynb-test-samples/test-latex-pass-correctouput.ipynb']'
============================= test session starts ==============================
platform linux -- Python 3.8.13, pytest-7.1.2, pluggy-1.0.0 -- /usr/bin/python3
cachedir: .pytest_cache
rootdir: /home/tkloczko/rpmbuild/BUILD/nbval-0.9.6
plugins: nbval-0.9.6
collecting ... collected 2 items

tests/ipynb-test-samples/test-latex-pass-correctouput::ipynb::Cell 0 FAILED [ 50%]
tests/ipynb-test-samples/test-latex-pass-correctouput::ipynb::Cell 1 FAILED [100%]

=================================== FAILURES ===================================
_____ tests/ipynb-test-samples/test-latex-pass-correctouput.ipynb::Cell 0 ______
Notebook cell execution failed
Cell 0: Cell execution caused an exception

Input:
import sympy
sympy.init_printing()
x, y = sympy.symbols(["x", "y"])

Traceback:

---------------------------------------------------------------------------
ModuleNotFoundError                       Traceback (most recent call last)
Input In [1], in <cell line: 1>()
----> 1 import sympy
      2 sympy.init_printing()
      3 x, y = sympy.symbols(["x", "y"])

ModuleNotFoundError: No module named 'sympy'

_____ tests/ipynb-test-samples/test-latex-pass-correctouput.ipynb::Cell 1 ______
Notebook cell execution failed
Cell 1: Cell execution caused an exception

Input:
x**2 + y**3

Traceback:

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
Input In [2], in <cell line: 1>()
----> 1 x**2 + y**3

NameError: name 'x' is not defined

=============================== warnings summary ===============================
../../../../../usr/lib/python3.8/site-packages/_pytest/nodes.py:633
  /usr/lib/python3.8/site-packages/_pytest/nodes.py:633: PytestRemovedIn8Warning: The (fspath: py.path.local) argument to IPyNbFile is deprecated. Please use the (path: pathlib.Path) argument instead.
  See https://docs.pytest.org/en/latest/deprecations.html#fspath-argument-for-node-constructors-replaced-with-pathlib-path
    return super().from_parent(parent=parent, fspath=fspath, path=path, **kw)

-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
=========================== short test summary info ============================
FAILED tests/ipynb-test-samples/test-latex-pass-correctouput.ipynb::Cell 0
FAILED tests/ipynb-test-samples/test-latex-pass-correctouput.ipynb::Cell 1
========================= 2 failed, 1 warning in 1.67s =========================
________________________ test_print[/home/tkloczko/rpmbuild/BUILD/nbval-0.9.6/tests/ipynb-test-samples/test-latex-pass-failsbutignoreoutput.ipynb] _________________________

filename = '/home/tkloczko/rpmbuild/BUILD/nbval-0.9.6/tests/ipynb-test-samples/test-latex-pass-failsbutignoreoutput.ipynb', correctoutcome = 'pass'

    @pytest.mark.parametrize("filename, correctoutcome", testdata, ids=testnames)
    def test_print(filename, correctoutcome):

        command = ["py.test", "--nbval", "-v", "--current-env", filename]
        print("Starting parametrized test with filename={}, correctoutcome={}"
              .format(filename, correctoutcome))
        print("Command about to execute is '{}'".format(command))

        if os.name == 'nt':
            exitcode = subprocess.call(command, shell=True)
        else:
            exitcode = subprocess.call(command)

        if correctoutcome == 'pass':
            if exitcode != 0:
>               raise AssertionError("Tests failed on ipynb (expected pass)")
E               AssertionError: Tests failed on ipynb (expected pass)

tests/test_unit_tests_in_notebooks.py:68: AssertionError
--------------------------------------------------------------------------- Captured stdout call ---------------------------------------------------------------------------
Starting parametrized test with filename=/home/tkloczko/rpmbuild/BUILD/nbval-0.9.6/tests/ipynb-test-samples/test-latex-pass-failsbutignoreoutput.ipynb, correctoutcome=pass
Command about to execute is '['py.test', '--nbval', '-v', '--current-env', '/home/tkloczko/rpmbuild/BUILD/nbval-0.9.6/tests/ipynb-test-samples/test-latex-pass-failsbutignoreoutput.ipynb']'
============================= test session starts ==============================
platform linux -- Python 3.8.13, pytest-7.1.2, pluggy-1.0.0 -- /usr/bin/python3
cachedir: .pytest_cache
rootdir: /home/tkloczko/rpmbuild/BUILD/nbval-0.9.6
plugins: nbval-0.9.6
collecting ... collected 4 items

tests/ipynb-test-samples/test-latex-pass-failsbutignoreoutput::ipynb::Cell 0 FAILED [ 25%]
tests/ipynb-test-samples/test-latex-pass-failsbutignoreoutput::ipynb::Cell 1 PASSED [ 50%]
tests/ipynb-test-samples/test-latex-pass-failsbutignoreoutput::ipynb::Cell 2 FAILED [ 75%]
tests/ipynb-test-samples/test-latex-pass-failsbutignoreoutput::ipynb::Cell 3 PASSED [100%]

=================================== FAILURES ===================================
_ tests/ipynb-test-samples/test-latex-pass-failsbutignoreoutput.ipynb::Cell 0 __
Notebook cell execution failed
Cell 0: Cell execution caused an exception

Input:
import time
import sympy
sympy.init_printing()
x = sympy.Symbol("x")

Traceback:

---------------------------------------------------------------------------
ModuleNotFoundError                       Traceback (most recent call last)
Input In [1], in <cell line: 2>()
      1 import time
----> 2 import sympy
      3 sympy.init_printing()
      4 x = sympy.Symbol("x")

ModuleNotFoundError: No module named 'sympy'

_ tests/ipynb-test-samples/test-latex-pass-failsbutignoreoutput.ipynb::Cell 2 __
Notebook cell execution failed
Cell 2: Cell execution caused an exception

Input:
# NBVAL_IGNORE_OUTPUT
x**n

Traceback:

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
Input In [3], in <cell line: 2>()
      1 # NBVAL_IGNORE_OUTPUT
----> 2 x**n

NameError: name 'x' is not defined

=============================== warnings summary ===============================
../../../../../usr/lib/python3.8/site-packages/_pytest/nodes.py:633
  /usr/lib/python3.8/site-packages/_pytest/nodes.py:633: PytestRemovedIn8Warning: The (fspath: py.path.local) argument to IPyNbFile is deprecated. Please use the (path: pathlib.Path) argument instead.
  See https://docs.pytest.org/en/latest/deprecations.html#fspath-argument-for-node-constructors-replaced-with-pathlib-path
    return super().from_parent(parent=parent, fspath=fspath, path=path, **kw)

-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
=========================== short test summary info ============================
FAILED tests/ipynb-test-samples/test-latex-pass-failsbutignoreoutput.ipynb::Cell 0
FAILED tests/ipynb-test-samples/test-latex-pass-failsbutignoreoutput.ipynb::Cell 2
==================== 2 failed, 2 passed, 1 warning in 1.72s ====================
========================================================================= short test summary info ==========================================================================
FAILED issues/67/test_2.py::test_b - AssertionError
FAILED issues/67/test_2.py::test_c - assert 1 == 2
FAILED tests/test_coverage.py::test_coverage - assert <ExitCode.USAGE_ERROR: 4> == 0
FAILED tests/test_ignore.py::test_conf_ignore_stderr - assert <ExitCode.TESTS_FAILED: 1> == 0
FAILED tests/test_unit_tests_in_notebooks.py::test_print[/home/tkloczko/rpmbuild/BUILD/nbval-0.9.6/tests/ipynb-test-samples/test-latex-pass-correctouput.ipynb] - Asserti...
FAILED tests/test_unit_tests_in_notebooks.py::test_print[/home/tkloczko/rpmbuild/BUILD/nbval-0.9.6/tests/ipynb-test-samples/test-latex-pass-failsbutignoreoutput.ipynb]
====================================================================== 6 failed, 10 passed in 34.36s =======================================================================

Here is list of modules isntalled in build env:

Details

Package                       Version
----------------------------- -----------------
alabaster                     0.7.12
appdirs                       1.4.4
asttokens                     2.0.5
attrs                         22.1.0.dev0
Babel                         2.10.2
backcall                      0.2.0
beautifulsoup4                4.10.0
bleach                        5.0.0
Brlapi                        0.8.3
build                         0.8.0
cffi                          1.15.1
charset-normalizer            2.1.0
codespell                     2.1.0
coverage                      6.4.2
cssselect                     1.1.0
cycler                        0.11.0
debugpy                       1.6.2
decorator                     5.1.1
defusedxml                    0.7.1
distro                        1.7.0
docutils                      0.18.1
entrypoints                   0.4
executing                     0.8.3
extras                        1.0.0
fastjsonschema                2.16.1
fixtures                      4.0.0
fonttools                     4.34.4
gpg                           1.17.1-unknown
html2text                     2020.1.16
html5lib                      1.1
idna                          3.3
imagesize                     1.4.1
importlib-metadata            4.12.0
importlib-resources           5.9.0
iniconfig                     1.1.1
ipykernel                     6.15.1
ipython                       8.4.0
jedi                          0.18.1
Jinja2                        3.1.1
jsonschema                    4.7.2
jupyter-client                7.3.4
jupyter-core                  4.11.1
jupyterlab-pygments           0.1.2
kiwisolver                    1.4.4
libcomps                      0.1.18
louis                         3.22.0
lxml                          4.9.1
MarkupSafe                    2.1.1
matplotlib                    3.5.2
matplotlib-inline             0.1.3
metaextract                   1.0.8
mistune                       2.0.3
nbclient                      0.6.6
nbconvert                     6.5.0
nbformat                      5.4.0
nbsphinx                      0.8.8
nest-asyncio                  1.5.5
numpy                         1.23.1
olefile                       0.46
packaging                     21.3
pandocfilters                 1.5.0
parso                         0.8.3
pbr                           5.8.1
pep517                        0.12.0
pexpect                       4.8.0
pickleshare                   0.7.5
Pillow                        9.2.0
pip                           22.0.4
pluggy                        1.0.0
ply                           3.11
prompt-toolkit                3.0.29
psutil                        5.9.0
ptyprocess                    0.7.0
pure-eval                     0.2.2
py                            1.11.0
py2pack                       0.8.7
pycparser                     2.21
Pygments                      2.12.0
PyGObject                     3.42.2
pyparsing                     3.0.9
pypi-search                   1.2.1
pyrsistent                    0.18.1
pytest                        7.1.2
python-dateutil               2.8.2
pytz                          2022.1
pyzmq                         23.2.0
requests                      2.28.1
rpm                           4.17.0
scour                         0.38.2
setuptools                    63.2.0
six                           1.16.0
sniffio                       1.2.0
snowballstemmer               2.2.0
soupsieve                     2.3.2.post1
Sphinx                        5.1.1
sphinxcontrib-applehelp       1.0.2.dev20220714
sphinxcontrib-devhelp         1.0.2.dev20220714
sphinxcontrib-htmlhelp        2.0.0
sphinxcontrib-jsmath          1.0.1.dev20220712
sphinxcontrib-qthelp          1.0.3.dev20220712
sphinxcontrib-serializinghtml 1.1.5
stack-data                    0.3.0
testtools                     2.5.0
tinycss2                      1.1.1
tomli                         2.0.1
tornado                       6.1
traitlets                     5.3.0
urllib3                       1.26.9
wcwidth                       0.2.5
webencodings                  0.5.1
wheel                         0.37.1
zipp                          3.8.1

[takluyver]

Some extra packages are needed for testing, and some of the tests are not meant to be run normally (we're testing a testing tool, so naturally it has some sample failing tests around). The testing procedure used in this repo is defined in the dodo.py, used with the doit command (kind of like make but easier for Windows users).

nbval/dodo.py

Lines 22 to 33 in 6dd2eaa

	def task_test():
	return {
	'actions': [
	_make_cmd(["py.test", "-v", "tests/", "--nbval", "--nbval-current-env", "--nbval-sanitize-with", "tests/sanitize_defaults.cfg", "--ignore", "tests/ipynb-test-samples"]),
	],
	}
	def task_install_test_deps():
	test_deps = ['matplotlib', 'sympy', 'pytest-cov', 'pytest-mock', 'nbdime']
	return {
	'actions': [_make_cmd(['pip', 'install'] + test_deps)],
	}

[kloczek]

nbval/dodo.py

Line 25 in 6dd2eaa

_make_cmd(["py.test", "-v", "tests/", "--nbval", "--nbval-current-env", "--nbval-sanitize-with", "tests/sanitize_defaults.cfg", "--ignore", "tests/ipynb-test-samples"]),

Instead passing `tests/' it should be part of the pytest.ini testpaths variable.
PAckagers should not be forced to decipher with what kind of additional pytest params test suite needs to executed.

As well if you want to use coveryty test please move that from pytest.ini to CI.
Forcing coveryty tests on for example packaging (like I'm doing) is not convinient.

[kloczek]

Updated pytest output of 0.10.0

Details

+ PYTHONPATH=/home/tkloczko/rpmbuild/BUILDROOT/python-nbval-0.10.0-2.fc35.x86_64/usr/lib64/python3.8/site-packages:/home/tkloczko/rpmbuild/BUILDROOT/python-nbval-0.10.0-2.fc35.x86_64/usr/lib/python3.8/site-packages
+ /usr/bin/pytest -ra -m 'not network'
============================= test session starts ==============================
platform linux -- Python 3.8.17, pytest-7.4.0, pluggy-1.2.0
rootdir: /home/tkloczko/rpmbuild/BUILD/nbval-0.10.0
plugins: nbval-0.10.0, anyio-3.7.0, mock-3.11.1
collected 17 items

issues/67/test_1.py .                                                    [  5%]
issues/67/test_2.py FF                                                   [ 17%]
issues/7/test_lib.py .                                                   [ 23%]
tests/test_collect.py ..                                                 [ 35%]
tests/test_coverage.py F                                                 [ 41%]
tests/test_ignore.py .                                                   [ 47%]
tests/test_nbdime_reporter.py .                                          [ 52%]
tests/test_plugin.py .                                                   [ 58%]
tests/test_timeouts.py .                                                 [ 64%]
tests/test_unit_tests_in_notebooks.py ......                             [100%]

=================================== FAILURES ===================================
____________________________________ test_b ____________________________________

    def test_b():
>       raise AssertionError
E       AssertionError

issues/67/test_2.py:2: AssertionError
____________________________________ test_c ____________________________________

    def test_c():
>       assert 1 == 2
E       assert 1 == 2

issues/67/test_2.py:5: AssertionError
________________________________ test_coverage _________________________________

testdir = <Testdir local('/tmp/pytest-of-tkloczko/pytest-163/test_coverage0')>

    def test_coverage(testdir):

        testdir.makepyfile(
            # Setup file to cover:
            lib="""
                def mysum(a, b):
                    return a + b
                def myprod(a, b):
                    return a * b
            """,
            # Setup python file to cover mysum function
            test_lib="""
                import lib
                def test_sum():
                    assert lib.mysum(1, 3) == 4
                    assert lib.mysum("cat", "dog") == "catdog"
                    assert lib.mysum(1.5, 2) == 3.5
            """,
        )

        # Setup notebook to cover myprod function
        nb = build_nb([
            "import lib",
            "lib.myprod(1, 3)",
            "lib.myprod(2.5, 2.5)",
            "lib.myprod(2, 'cat')"
        ], mark_run=True)
        add_expected_plaintext_outputs(nb, [
            None, "3", "6.25", "'catcat'"
        ])
        # Write notebook to test dir
        nbformat.write(nb, os.path.join(
            str(testdir.tmpdir), 'test_coverage.ipynb'))

        # Run tests
        result = testdir.runpytest_inprocess('--nbval', '--nbval-current-env', '--cov', '.')

        # Check tests went off as they should:
>       assert result.ret == 0
E       assert <ExitCode.USAGE_ERROR: 4> == 0
E        +  where <ExitCode.USAGE_ERROR: 4> = <RunResult ret=ExitCode.USAGE_ERROR len(stdout.lines)=0 len(stderr.lines)=5 duration=0.04s>.ret

/home/tkloczko/rpmbuild/BUILD/nbval-0.10.0/tests/test_coverage.py:52: AssertionError
----------------------------- Captured stderr call -----------------------------
ERROR: usage: pytest [options] [file_or_dir] [file_or_dir] [...]
pytest: error: unrecognized arguments: --cov
  inifile: None
  rootdir: /tmp/pytest-of-tkloczko/pytest-163/test_coverage0

=========================== short test summary info ============================
FAILED issues/67/test_2.py::test_b - AssertionError
FAILED issues/67/test_2.py::test_c - assert 1 == 2
FAILED tests/test_coverage.py::test_coverage - assert <ExitCode.USAGE_ERROR: ...
======================== 3 failed, 14 passed in 37.11s =========================