Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,41 @@ Clarification:
- test title in testomat.io == test name in pytest
- test suit title in testomat.io == test file name in pytest

##### Tagging tests on sync
Since a Python test function name can't contain special characters like `@tag`, tags are
assigned via the built-in `testomatio_tags` marker, and are added to the test title on sync:

```python
import pytest


@pytest.mark.testomatio_tags("smoke", "regression")
def test_example():
assert 2 + 2 == 4
```

This produces a test titled `Example @smoke @regression` in testomat.io.

You can also reuse plain pytest markers as tags by allowlisting their names via the
`TESTOMATIO_TAG_MARKERS` env var (comma-separated). We don't pick up every `pytest.mark.*`
automatically, since that would also catch pytest/plugin service markers like `parametrize`,
`skip`, `usefixtures`, etc. - only markers explicitly listed are treated as tags:

```bash
export TESTOMATIO_TAG_MARKERS=smoke,regression
```

```python
import pytest


@pytest.mark.smoke
def test_example():
assert 2 + 2 == 4
```

This produces a test titled `Example @smoke` in testomat.io.


#### Clean up
Remove all test ids from source code. Tests will not be executed
Expand Down Expand Up @@ -317,6 +352,7 @@ You can use environment variable to control certain features of testomat.io. Env
| TESTOMATIO | Provides token for pytestomatio to access and push data to testomat.io. Required for **sync** and **report** commands | TESTOMATIO=tstmt_***** pytest --testomatio sync |
| TESTOMATIO_SYNC_LABELS | Assign labels to a test case when you synchronise test from code with testomat.io. Labels must exist in project and their scope must be enabled for tests | TESTOMATIO_SYNC_LABELS="number:1,list:one,standalone" pytest --testomatio report |
| TESTOMATIO_CODE_STYLE | Code parsing style for test synchronization. If you are not sure, don't set this variable. Default value is 'default' | TESTOMATIO_CODE_STYLE=pep8 pytest --testomatio sync |
| TESTOMATIO_TAG_MARKERS | Allowlist of bare pytest marker names to treat as tags and append to the test title on **sync** (comma-separated). See [Tagging tests on sync](#tagging-tests-on-sync) | TESTOMATIO_TAG_MARKERS=smoke,regression pytest --testomatio sync |
| TESTOMATIO_CI_DOWNSTREAM | If set, pytestomatio will not set or update build url for a test run. This is useful in scenarios where build url is already set in the test run by Testomat.io for test runs that a created directly on Testomat.io. | TESTOMATIO_CI_DOWNSTREAM=true pytest --testomatio report |
| TESTOMATIO_URL | Customize testomat.io url | TESTOMATIO_URL=https://custom.com/ pytest --testomatio report |
| BUILD_URL | Overrides build url run tests | BUILD_URL=http://custom.com/ pytest --testomatio report |
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ version_provider = "pep621"
update_changelog_on_bump = false
[project]
name = "pytestomatio"
version = "2.12.0"
version = "2.13.0b0"

dependencies = [
"requests>=2.32.4",
Expand Down
3 changes: 3 additions & 0 deletions pytestomatio/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,9 @@ def pytest_configure(config: Config):
config.addinivalue_line(
"markers", "testomatio(arg): built in marker to connect test case with testomat.io by unique id"
)
config.addinivalue_line(
"markers", "testomatio_tags(*tags): assign one or more tags to a test for testomat.io sync"
)

option = validations.validate_option(config)
if option == 'debug':
Expand Down
30 changes: 28 additions & 2 deletions pytestomatio/testing/testItem.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from re import sub
from typing import Iterable
from os import getenv
import uuid
import json
from pytest import Item
Expand Down Expand Up @@ -88,20 +89,43 @@ def _get_suite_title(self, test):
return self.file_name

# Testomatio resolves test id on BE by parsing test name to find test id
def _get_sync_test_title(self, item: Item) -> str:
def _get_sync_test_title(self, item: Item, include_tags: bool = True) -> str:
test_name = self.pytest_title_to_testomatio_title(item.name)
test_name = self._resolve_parameter_key_in_test_name(item, test_name)
tags = self._get_test_tags(item) if include_tags else []
if tags:
test_name = test_name + ' ' + ' '.join(f'@{tag}' for tag in tags)
# Test id is present on already synced tests
# New test don't have testomatio test id.
test_id = self.id
if (test_id):
test_name = f'{test_name} {test_id}'
# ex. "User adds item to cart"
# ex. "User adds item to cart @T1234"
# ex. "User adds item to cart @smoke @T1234"
# ex. "User adds ${item} to cart @T1234"
# ex. "User adds ${variation} ${item} to cart @T1234"
return test_name

def _get_test_tags(self, item: Item) -> list:
"""Returns list of tags assigned via @pytest.mark.testomatio_tags("tag1", "tag2")
and via bare markers allowlisted through the TESTOMATIO_TAG_MARKERS env var"""
tags = []
for marker in item.iter_markers('testomatio_tags'):
tags.extend([tag for tag in marker.args if tag not in tags])

for marker_name in self._get_allowed_tag_markers():
if marker_name not in tags and any(item.iter_markers(marker_name)):
tags.append(marker_name)
return tags

def _get_allowed_tag_markers(self) -> list:
"""Allowlist of bare marker names treated as tags"""
raw = getenv('TESTOMATIO_TAG_MARKERS')
if not raw:
return []
return [name.strip() for name in raw.split(',') if name.strip()]

# Fix such example @pytest.mark.parametrize("version", "1.0.0"), ref. https://github.com/testomatio/check-tests/issues/147
# that doesn't parse value correctly
def _get_execution_test_title(self, item: Item) -> str:
Expand All @@ -117,7 +141,9 @@ def pytest_title_to_testomatio_title(self, test_name: str) -> str:
return test_name.lower().replace('_', ' ').replace("test", "", 1).strip().capitalize()

def _get_resync_test_title(self, name: str) -> str:
name = self._get_sync_test_title(name)
# tags from testomatio_tags marker are not present in titles of tests
# returned by testomatio BE, so they must be excluded here for matching to work
name = self._get_sync_test_title(name, include_tags=False)
tag_at = name.find("@T")
if tag_at > 0:
return name[0:tag_at].strip()
Expand Down
8 changes: 6 additions & 2 deletions tests/test_main.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,10 +93,14 @@ def test_configure_adds_testomatio_marker(self, mock_validate, mock_config):
mock_validate.return_value = 'debug'
main.pytest_configure(mock_config)

mock_config.addinivalue_line.assert_called_with(
mock_config.addinivalue_line.assert_any_call(
"markers",
"testomatio(arg): built in marker to connect test case with testomat.io by unique id"
)
mock_config.addinivalue_line.assert_any_call(
"markers",
"testomatio_tags(*tags): assign one or more tags to a test for testomat.io sync"
)

@patch('pytestomatio.main.validations.validate_option')
def test_configure_debug_mode_early_return(self, mock_validate, mock_config):
Expand All @@ -105,7 +109,7 @@ def test_configure_debug_mode_early_return(self, mock_validate, mock_config):

main.pytest_configure(mock_config)

assert mock_config.addinivalue_line.call_count == 1
assert mock_config.addinivalue_line.call_count == 2
assert not hasattr(pytest, 'testomatio') or pytest.testomatio is None

@patch('pytestomatio.main.validations.validate_option')
Expand Down
145 changes: 145 additions & 0 deletions tests/test_testing/test_TestItem.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,15 @@ def mock_item_with_marker(self, mock_item):
mock_item.iter_markers.return_value = iter([marker])
return mock_item

@pytest.fixture
def mock_item_with_tags(self, mock_item):
"""Mock Item with testomatio_tags marker"""
tag_marker = Mock()
tag_marker.args = ("smoke", "regression")
mock_item.iter_markers.side_effect = \
lambda name=None: iter([tag_marker]) if name == 'testomatio_tags' else iter([])
return mock_item

@patch('inspect.getdoc')
@patch('inspect.getsource')
def test_init_basic(self, mock_getsource, mock_getdoc, mock_item):
Expand Down Expand Up @@ -463,3 +472,139 @@ def test_suite_title_return_filename_for_bdd_test(self, mock_getsource, mock_ite
mock_item.function.__scenario__ = scenario_mock
test_item = TestItem(mock_item)
assert test_item.suite_title == 'test_file.py'

def test_get_test_tags_with_marker(self, mock_item_with_tags):
"""Test _get_test_tags returns tags from testomatio_tags marker"""
test_item = TestItem.__new__(TestItem)

result = test_item._get_test_tags(mock_item_with_tags)

assert result == ["smoke", "regression"]

def test_get_test_tags_without_marker(self, mock_item):
"""Test _get_test_tags returns empty list without marker"""
test_item = TestItem.__new__(TestItem)

result = test_item._get_test_tags(mock_item)

assert result == []

def test_get_test_tags_deduplicates_across_multiple_markers(self, mock_item):
"""Test _get_test_tags merges tags from stacked markers without duplicates"""
marker1 = Mock()
marker1.args = ("smoke", "regression")
marker2 = Mock()
marker2.args = ("regression", "api")
mock_item.iter_markers.side_effect = \
lambda name=None: iter([marker1, marker2]) if name == 'testomatio_tags' else iter([])
test_item = TestItem.__new__(TestItem)

result = test_item._get_test_tags(mock_item)

assert result == ["smoke", "regression", "api"]

@patch('inspect.getsource')
def test_get_sync_test_title_includes_tags_before_test_id(self, mock_source, mock_item_with_tags):
"""Test _get_sync_test_title appends tags before the testomatio test id"""
test_item = TestItem(mock_item_with_tags)
test_item.id = "@T12345678"

result = test_item._get_sync_test_title(mock_item_with_tags)

assert result == "Example @smoke @regression @T12345678"

@patch('inspect.getsource')
def test_get_sync_test_title_without_tags_marker(self, mock_source, mock_item):
"""Test _get_sync_test_title is unchanged when no tags marker is present"""
test_item = TestItem(mock_item)
test_item.id = "@T12345678"

result = test_item._get_sync_test_title(mock_item)

assert result == "Example @T12345678"

@patch('inspect.getsource')
def test_get_sync_test_title_include_tags_false_excludes_tags(self, mock_source, mock_item_with_tags):
"""Test _get_sync_test_title with include_tags=False ignores the tags marker"""
test_item = TestItem(mock_item_with_tags)
test_item.id = "@T12345678"

result = test_item._get_sync_test_title(mock_item_with_tags, include_tags=False)

assert result == "Example @T12345678"

@patch('inspect.getsource')
def test_get_resync_test_title_excludes_tags(self, mock_source, mock_item_with_tags):
"""Test _get_resync_test_title strips both tags and test id since BE titles have neither"""
test_item = TestItem(mock_item_with_tags)
test_item.id = "@T12345678"

result = test_item._get_resync_test_title(mock_item_with_tags)

assert result == "Example"

def test_get_allowed_tag_markers_empty_when_env_not_set(self, monkeypatch):
"""Test _get_allowed_tag_markers returns empty list when env var is not set"""
monkeypatch.delenv('TESTOMATIO_TAG_MARKERS', raising=False)
test_item = TestItem.__new__(TestItem)

result = test_item._get_allowed_tag_markers()

assert result == []

def test_get_allowed_tag_markers_parses_comma_separated_list(self, monkeypatch):
"""Test _get_allowed_tag_markers trims whitespace and drops empty entries"""
monkeypatch.setenv('TESTOMATIO_TAG_MARKERS', 'smoke, regression,, api ')
test_item = TestItem.__new__(TestItem)

result = test_item._get_allowed_tag_markers()

assert result == ["smoke", "regression", "api"]

def test_get_test_tags_with_allowlisted_bare_marker(self, mock_item, monkeypatch):
"""Test _get_test_tags picks up a bare marker whose name is allowlisted"""
monkeypatch.setenv('TESTOMATIO_TAG_MARKERS', 'smoke')
smoke_marker = Mock()
mock_item.iter_markers.side_effect = \
lambda name=None: iter([smoke_marker]) if name == 'smoke' else iter([])
test_item = TestItem.__new__(TestItem)

result = test_item._get_test_tags(mock_item)

assert result == ["smoke"]

def test_get_test_tags_ignores_marker_not_in_allowlist(self, mock_item, monkeypatch):
"""Test _get_test_tags ignores markers whose name isn't allowlisted"""
monkeypatch.setenv('TESTOMATIO_TAG_MARKERS', 'smoke')
other_marker = Mock()
mock_item.iter_markers.side_effect = \
lambda name=None: iter([other_marker]) if name == 'asyncio' else iter([])
test_item = TestItem.__new__(TestItem)

result = test_item._get_test_tags(mock_item)

assert result == []

def test_get_test_tags_combines_testomatio_tags_marker_and_allowlisted_bare_markers(self, mock_item, monkeypatch):
"""Test _get_test_tags merges testomatio_tags marker args with allowlisted bare markers, deduped"""
monkeypatch.setenv('TESTOMATIO_TAG_MARKERS', 'regression,api')
tags_marker = Mock()
tags_marker.args = ("smoke", "regression")
regression_marker = Mock()
api_marker = Mock()

def iter_markers(name=None):
if name == 'testomatio_tags':
return iter([tags_marker])
if name == 'regression':
return iter([regression_marker])
if name == 'api':
return iter([api_marker])
return iter([])

mock_item.iter_markers.side_effect = iter_markers
test_item = TestItem.__new__(TestItem)

result = test_item._get_test_tags(mock_item)

assert result == ["smoke", "regression", "api"]
Loading