Feat/create test from report

README.md

@@ -297,6 +297,8 @@ You can use environment variable to control certain features of testomat.io
 | TESTOMATIO_STACK_PASSED       | Enables logs for passed tests. Disabled by default.                                                                                                                              | TESTOMATIO_STACK_PASSED=true pytest --testomatio report                                                          |
 | TESTOMATIO_SHARED_RUN         | Report parallel execution to the same run matching it by title. If the run was created more than 20 minutes ago, a new run will be created instead.                              | TESTOMATIO_TITLE="Run1" TESTOMATIO_SHARED_RUN=1 pytest --testomatio report                                  |
 | TESTOMATIO_SHARED_RUN_TIMEOUT | Changes timeout of shared run. After timeout, shared run won`t accept other runs with same name, and new runs will be created. Timeout is set in minutes, default is 20 minutes. | TESTOMATIO_TITLE="Run1" TESTOMATIO_SHARED_RUN=1 TESTOMATIO_SHARED_RUN_TIMEOUT=10 pytest --testomatio report |
+| TESTOMATIO_CREATE             | Create test which are not yet exist in a project                                                                                                                                 | TESTOMATIO_CREATE=1 pytest --testomatio report                                                              |
+| TESTOMATIO_WORKDIR            | Specify a custom working directory for relative file paths in test reports. When tests are created with **TESTOMATIO_CREATE=1**, file paths will be relative to this directory.  | TESTOMATIO_WORKDIR=new_dir pytest --testomatio report                                                       |
 | TESTOMATIO_DISABLE_BATCH_UPLOAD | Disables batch uploading and uploads each test result one by one.                                                                                                                | TESTOMATIO_DISABLE_BATCH_UPLOAD=True pytest --testomatio report                                             |
 | TESTOMATIO_BATCH_SIZE | Changes size of batch for batch uploading. Default is 50. Maximum is 100.                                                                                                        | TESTOMATIO_BATCH_SIZE=15 pytest --testomatio report                                                         |
 

pytestomatio/connect/connector.py

@@ -251,6 +251,7 @@ def update_test_status(self, run_id: str,
                            rid: str,
                            status: str,
                            title: str,
+                           create: bool | None,
                            suite_title: str,
                            suite_id: str,
                            test_id: str,
@@ -262,13 +263,16 @@ def update_test_status(self, run_id: str,
                            steps: str,
                            code: str,
                            example: dict,
+                           file: str | None,
                            overwrite: bool | None,
                            meta: dict) -> None:
 
         log.info(f'Reporting test. Id: {test_id}. Title: {title}')
         request = {
             "status": status,  # Enum: "passed" "failed" "skipped"
             "title": title,
+            "create": create,
+            "file": file,
             "suite_title": suite_title,
             "suite_id": suite_id,
             "test_id": test_id,

pytestomatio/main.py

@@ -1,5 +1,6 @@
 import os, pytest, logging, json, time
 import warnings
+from os.path import join, normpath
 
 from pytest import Parser, Session, Config, Item, CallInfo
 from pytestomatio.connect.connector import Connector
@@ -198,7 +199,9 @@ def pytest_runtest_makereport(item: Item, call: CallInfo):
     request = {
         'status': None,
         'title': test_item.exec_title,
+        'create': None,
         'run_time': call.duration,
+        'file': None,
         'suite_title': test_item.suite_title,
         'suite_id': None,
         'test_id': test_id,
@@ -218,6 +221,12 @@ def pytest_runtest_makereport(item: Item, call: CallInfo):
         request['code'] = test_item.source_code
         request['overwrite'] = True
 
+    if pytest.testomatio.test_run_config.create_tests:
+        request['create'] = True
+        workdir = pytest.testomatio.test_run_config.workdir
+        if workdir:
+            request['file'] = normpath(join(workdir, test_item.file_name))
+
     # TODO: refactor it and use TestItem setter to upate those attributes
     if call.when in ['setup', 'call']:
         logs = get_test_logs(item.nodeid, True)

pytestomatio/testomatio/testRunConfig.py

@@ -12,6 +12,7 @@ class TestRunConfig:
     def __init__(self, kind: str = 'automated'):
         run_id = os.environ.get('TESTOMATIO_RUN_ID') or os.environ.get('TESTOMATIO_RUN')
         title = os.environ.get('TESTOMATIO_TITLE') if os.environ.get('TESTOMATIO_TITLE') else 'test run at ' + dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        create_tests = os.environ.get('TESTOMATIO_CREATE', False) in ['True', 'true', '1']
         disable_batch_upload = os.environ.get('TESTOMATIO_DISABLE_BATCH_UPLOAD') in ['True', 'true', '1']
         batch_size = os.environ.get('TESTOMATIO_BATCH_SIZE', '')
         shared_run = os.environ.get('TESTOMATIO_SHARED_RUN') in ['True', 'true', '1']
@@ -33,6 +34,7 @@ def __init__(self, kind: str = 'automated'):
         self.environment = safe_string_list(os.environ.get('TESTOMATIO_ENV'))
         self.disable_timestamp = disable_timestamp
         self.exclude_skipped = exclude_skipped
+        self.create_tests = create_tests if create_tests else None
         self.label = safe_string_list(os.environ.get('TESTOMATIO_LABEL'))
         self.group_title = os.environ.get('TESTOMATIO_RUNGROUP_TITLE')
         self.jira_id = os.environ.get('TESTOMATIO_JIRA_ID')
@@ -45,6 +47,7 @@ def __init__(self, kind: str = 'automated'):
         self.proceed = os.getenv('TESTOMATIO_PROCEED', False)
         self.status_request = {}
         self.update_code = update_code
+        self.workdir = os.getenv('TESTOMATIO_WORKDIR', None)
         self.build_url = self.resolve_build_url()
         self.meta = self.update_meta()
 

tests/test_connect/test_connector.py

@@ -431,6 +431,7 @@ def test_update_test_status_success(self, mock_post, connector):
             rid="rid123",
             status="passed",
             title="Test Login",
+            create=True,
             suite_title="Auth Suite",
             suite_id="suite_456",
             test_id="test_789",
@@ -443,7 +444,8 @@ def test_update_test_status_success(self, mock_post, connector):
             code="def test_login(): pass",
             example={"param": "value"},
             overwrite=True,
-            meta={}
+            meta={},
+            file='file',
         )
 
         assert mock_post.call_count == 1
@@ -452,6 +454,8 @@ def test_update_test_status_success(self, mock_post, connector):
         assert f'{connector.base_url}/api/reporter/run_123/testrun' in call_args[0][0]
 
         payload = call_args[1]['json']
+        assert payload['create']
+        assert payload['file']
         assert payload['status'] == 'passed'
         assert payload['title'] == 'Test Login'
         assert payload['run_time'] == 1.5
@@ -477,6 +481,8 @@ def test_update_test_status_should_raise_report_failed_on_403_status_code(self,
                 suite_id="suite_456",
                 test_id="test_789",
                 timestamp=213.5,
+                create=False,
+                file="file",
                 message=None,
                 stack=None,
                 run_time=1.5,
@@ -531,6 +537,7 @@ def test_update_test_status_filters_none_values(self, mock_post, connector):
             run_id="run_123",
             status="passed",
             title="Test Login",
+            create=None,
             suite_title="Auth Suite",
             suite_id="suite_456",
             test_id="test_789",
@@ -544,7 +551,8 @@ def test_update_test_status_filters_none_values(self, mock_post, connector):
             example={"param": "value"},
             overwrite=None,
             rid=None,
-            meta=None
+            meta=None,
+            file=None,
         )
 
         assert mock_post.call_count == 1
@@ -559,6 +567,8 @@ def test_update_test_status_filters_none_values(self, mock_post, connector):
         assert payload['artifacts'] == ["screenshot.png"]
         assert payload['timestamp'] == 123222.21
         assert 'message' not in payload
+        assert 'create' not in payload
+        assert 'file' not in payload
         assert 'code' not in payload
         assert 'overwrite' not in payload
 

tests/test_main.py

@@ -437,6 +437,7 @@ def test_creates_test_request_structure(self, mock_call, single_test_item):
         pytest.testomatio = Mock()
         pytest.testomatio.test_run_config = Mock()
         pytest.testomatio.test_run_config.test_run_id = 'run_123'
+        pytest.testomatio.test_run_config.create_tests = None
         pytest.testomatio.test_run_config.status_request = {}
 
         main.pytest_runtest_makereport(item, mock_call)
@@ -472,6 +473,7 @@ def test_processing_skipped_test(self, mock_call, single_test_item):
         pytest.testomatio.test_run_config = Mock()
         pytest.testomatio.test_run_config.test_run_id = 'run_123'
         pytest.testomatio.test_run_config.exclude_skipped = False
+        pytest.testomatio.test_run_config.create_tests = None
         pytest.testomatio.test_run_config.status_request = {}
 
         main.pytest_runtest_makereport(item, mock_call)
@@ -526,6 +528,7 @@ def test_exclude_skipped_test_if_option_enabled(self, mock_call, single_test_ite
         pytest.testomatio.test_run_config = Mock()
         pytest.testomatio.test_run_config.test_run_id = 'run_123'
         pytest.testomatio.test_run_config.exclude_skipped = True
+        pytest.testomatio.test_run_config.create_tests = None
         pytest.testomatio.test_run_config.status_request = {}
 
         main.pytest_runtest_makereport(item, mock_call)
@@ -566,6 +569,7 @@ def test_rid_without_env(self, mock_call, single_test_item):
         pytest.testomatio.test_run_config = Mock()
         pytest.testomatio.test_run_config.test_run_id = 'run_123'
         pytest.testomatio.test_run_config.environment = testrun_env
+        pytest.testomatio.test_run_config.create_tests = None
         pytest.testomatio.test_run_config.status_request = {}
 
         main.pytest_runtest_makereport(item, mock_call)
@@ -592,6 +596,7 @@ def test_rid_with_env(self, mock_call, single_test_item):
         pytest.testomatio.test_run_config = Mock()
         pytest.testomatio.test_run_config.test_run_id = 'run_123'
         pytest.testomatio.test_run_config.environment = testrun_env
+        pytest.testomatio.test_run_config.create_tests = None
         pytest.testomatio.test_run_config.status_request = {}
 
         main.pytest_runtest_makereport(item, mock_call)
@@ -604,6 +609,83 @@ def test_rid_with_env(self, mock_call, single_test_item):
         assert request['test_id'] == '12345678'
         assert request['rid'] == f'{testrun_env}-{item.name}-12345678'
 
+    def test_create_test_when_option_enabled(self, mock_call, single_test_item):
+        item = single_test_item.copy()[0]
+        item.config.option.testomatio = 'report'
+
+        mock_call.duration = 1.5
+        mock_call.when = 'call'
+        mock_call.excinfo = None
+
+        pytest.testomatio = Mock()
+        pytest.testomatio.test_run_config = Mock()
+        pytest.testomatio.test_run_config.test_run_id = 'run_123'
+        pytest.testomatio.test_run_config.create_tests = True
+        pytest.testomatio.test_run_config.workdir = None
+        pytest.testomatio.test_run_config.status_request = {}
+
+        main.pytest_runtest_makereport(item, mock_call)
+
+        assert item.nodeid in pytest.testomatio.test_run_config.status_request
+        request = pytest.testomatio.test_run_config.status_request[item.nodeid]
+
+        assert 'create' in request.keys()
+
+        assert request['test_id'] == '12345678'
+        assert request['create'] is True
+        assert request['file'] is None
+
+    def test_create_test_when_option_disabled(self, mock_call, single_test_item):
+        item = single_test_item.copy()[0]
+        item.config.option.testomatio = 'report'
+
+        mock_call.duration = 1.5
+        mock_call.when = 'call'
+        mock_call.excinfo = None
+
+        pytest.testomatio = Mock()
+        pytest.testomatio.test_run_config = Mock()
+        pytest.testomatio.test_run_config.test_run_id = 'run_123'
+        pytest.testomatio.test_run_config.create_tests = None
+        pytest.testomatio.test_run_config.status_request = {}
+
+        main.pytest_runtest_makereport(item, mock_call)
+
+        assert item.nodeid in pytest.testomatio.test_run_config.status_request
+        request = pytest.testomatio.test_run_config.status_request[item.nodeid]
+
+        assert 'create' in request.keys()
+
+        assert request['test_id'] == '12345678'
+        assert request['create'] is None
+
+    def test_add_workdir_for_test_when_option_enabled(self, mock_call, single_test_item):
+        item = single_test_item.copy()[0]
+        item.config.option.testomatio = 'report'
+
+        mock_call.duration = 1.5
+        mock_call.when = 'call'
+        mock_call.excinfo = None
+
+        pytest.testomatio = Mock()
+        pytest.testomatio.test_run_config = Mock()
+        pytest.testomatio.test_run_config.test_run_id = 'run_123'
+        pytest.testomatio.test_run_config.create_tests = True
+        pytest.testomatio.test_run_config.workdir = 'new_dir'
+        pytest.testomatio.test_run_config.status_request = {}
+
+        main.pytest_runtest_makereport(item, mock_call)
+
+        assert item.nodeid in pytest.testomatio.test_run_config.status_request
+        request = pytest.testomatio.test_run_config.status_request[item.nodeid]
+
+        assert 'create' in request.keys()
+
+        assert request['test_id'] == '12345678'
+        assert request['create'] is True
+        assert request['file']
+        assert request['file'] == pytest.testomatio.test_run_config.workdir + '/' + item.path.name
+
     def test_code_field_when_update_code_option_disabled(self, mock_call, single_test_item):
         """Test code and overwrite fields in request not updated if update_code option disabled"""
         item = single_test_item.copy()[0]
@@ -616,6 +698,7 @@ def test_code_field_when_update_code_option_disabled(self, mock_call, single_tes
         pytest.testomatio = Mock()
         pytest.testomatio.test_run_config = Mock()
         pytest.testomatio.test_run_config.update_code = False
+        pytest.testomatio.test_run_config.create_tests = None
         pytest.testomatio.test_run_config.test_run_id = 'run_123'
         pytest.testomatio.test_run_config.status_request = {}
 
@@ -650,6 +733,7 @@ def test_code_field_when_update_code_option_enabled(self, mock_call, single_test
         pytest.testomatio = Mock()
         pytest.testomatio.test_run_config = Mock()
         pytest.testomatio.test_run_config.update_code = True
+        pytest.testomatio.test_run_config.create_tests = None
         pytest.testomatio.test_run_config.test_run_id = 'run_123'
         pytest.testomatio.test_run_config.status_request = {}
 
@@ -685,6 +769,7 @@ def test_code_field_for_bdd_when_update_code_option_disabled(self, mock_call, si
         pytest.testomatio = Mock()
         pytest.testomatio.test_run_config = Mock()
         pytest.testomatio.test_run_config.update_code = False
+        pytest.testomatio.test_run_config.create_tests = None
         pytest.testomatio.test_run_config.test_run_id = 'run_123'
         pytest.testomatio.test_run_config.status_request = {}
 
@@ -720,6 +805,7 @@ def test_code_field_for_bdd_when_update_code_option_enabled(self, mock_call, sin
         pytest.testomatio = Mock()
         pytest.testomatio.test_run_config = Mock()
         pytest.testomatio.test_run_config.update_code = True
+        pytest.testomatio.test_run_config.create_tests = None
         pytest.testomatio.test_run_config.test_run_id = 'run_123'
         pytest.testomatio.test_run_config.status_request = {}
 
@@ -755,6 +841,7 @@ def test_adds_timestamp_to_request(self, mock_call, single_test_item):
         pytest.testomatio.test_run_config = Mock()
         pytest.testomatio.test_run_config.disable_timestamp = False
         pytest.testomatio.test_run_config.test_run_id = 'run_123'
+        pytest.testomatio.test_run_config.create_tests = None
         pytest.testomatio.test_run_config.status_request = {}
 
         main.pytest_runtest_makereport(item, mock_call)
@@ -788,6 +875,7 @@ def test_timestamp_not_updated_when_option_enabled(self, mock_call, single_test_
         pytest.testomatio.test_run_config = Mock()
         pytest.testomatio.test_run_config.disable_timestamp = True
         pytest.testomatio.test_run_config.test_run_id = 'run_123'
+        pytest.testomatio.test_run_config.create_tests = None
         pytest.testomatio.test_run_config.status_request = {}
 
         main.pytest_runtest_makereport(item, mock_call)

tests/test_testomatio/test_testRunConfig.py

@@ -20,6 +20,7 @@ def test_init_default_values(self):
                 assert config.access_event is None
                 assert config.test_run_id is None
                 assert config.title == "test run at 2024-01-15 10:30:45"
+                assert config.create_tests is None
                 assert config.environment is None
                 assert config.enable_steps_for_passed_test is False
                 assert config.disable_steps is False
@@ -54,6 +55,7 @@ def test_init_with_env_variables(self):
             'TESTOMATIO_UPDATE_CODE': '1',
             'TESTOMATIO_PUBLISH': '1',
             'TESTOMATIO_EXCLUDE_SKIPPED': '1',
+            'TESTOMATIO_CREATE': 'True',
             'TESTOMATIO_STACK_PASSED': '1',
             'TESTOMATIO_DISABLE_BATCH_UPLOAD': 'True',
             'TESTOMATIO_BATCH_SIZE': '12'
@@ -65,6 +67,7 @@ def test_init_with_env_variables(self):
             assert config.access_event == 'publish'
             assert config.test_run_id == 'run_12345'
             assert config.title == 'Custom Test Run'
+            assert config.create_tests is True
             assert config.enable_steps_for_passed_test is True
             assert config.disable_steps is True
             assert config.disable_timestamp is True
@@ -187,6 +190,22 @@ def test_init_update_code_false_variations(self, value):
 
             assert config.update_code is False
 
+    @pytest.mark.parametrize('value', ['True', 'true', '1'])
+    def test_init_create_test_true_variations(self, value):
+        """Test different true values for TESTOMATIO_CREATE"""
+        with patch.dict(os.environ, {'TESTOMATIO_CREATE': value}, clear=True):
+            config = TestRunConfig()
+
+            assert config.create_tests is True
+
+    @pytest.mark.parametrize('value', ['False', 'false', '0', 'anything'])
+    def test_init_create_test_false_variations(self, value):
+        """Test different false values TESTOMATIO_CREATE"""
+        with patch.dict(os.environ, {'TESTOMATIO_CREATE': value}, clear=True):
+            config = TestRunConfig()
+
+            assert config.create_tests is None
+
     @pytest.mark.parametrize('value', ['True', 'true', '1'])
     def test_init_exclude_skipped_true_variations(self, value):
         """Test different true values for TESTOMATIO_EXCLUDE_SKIPPED"""