Support IBKR second-factor device selection and manual 2FA fallback

.gitignore

@@ -1,9 +1,10 @@
 *.iml
+# Local runtime secrets for docker compose
 env.list
 /copy_cache/chrome_driver
 ibeam.egg-info
 /outputs/
 /copy_cache/clientportal.gw/root/webapps/demo/gateway.demo.js
 /dev
 __pycache__
-.idea
+.idea

README.md

@@ -104,6 +104,8 @@ IBEAM_ACCOUNT=your_account123
 IBEAM_PASSWORD=your_password123
 ```
 
+For a reusable starting point, copy [`env.list.example`](./env.list.example) to `env.list` and adjust the values for your account.
+
 Run the following command:
 
 ```posh
@@ -125,6 +127,40 @@ curl -X GET "https://localhost:5000/v1/api/iserver/auth/status" -k
 
 Read more in [Installation and Startup][installation-and-startup] and [Advanced Secrets][advanced-secrets].
 
+### Two-factor device selection
+
+Some IBKR accounts show an extra `Select Second Factor Device` dropdown after submitting the username and password. If your account does, add the following settings to `env.list`:
+
+```posh
+IBEAM_TWO_FA_SELECT=true
+IBEAM_TWO_FA_SELECT_TARGET=IB Key
+```
+
+If your IBKR page uses a different device label, set `IBEAM_TWO_FA_SELECT_TARGET` to the text shown in that dropdown, for example `Mobile Authenticator App`.
+
+`IBEAM_TWO_FA_SELECT_EL_ID` defaults to `TAG@@select` and usually does not need to be changed.
+
+IBeam also includes a built-in fallback for code-entry screens that expose an input with a placeholder containing `Code`, which covers variants such as `Mobile Authenticator App Code` without requiring an extra env override.
+
+### Manual 2FA fallback
+
+If you do not have a usable automated 2FA handler, IBeam can pause at the second-factor step and wait for you to finish the login manually in your own browser.
+
+Add the following optional settings to `env.list`:
+
+```posh
+IBEAM_MANUAL_TWO_FA=true
+IBEAM_MANUAL_TWO_FA_TIMEOUT=300
+```
+
+When enabled and no `IBEAM_TWO_FA_HANDLER` is configured, IBeam will:
+
+1. Drive the login flow up to the second-factor step.
+1. Log instructions to open `https://localhost:5000` and complete the IBKR login manually.
+1. Wait for the gateway session to become authenticated before continuing.
+
+This is intended for setups where the second factor is device-bound and cannot be retrieved programmatically, such as a mobile authenticator app.
+
 ## <a name="how-ibeam-works"></a>How does IBeam work?
 
 In a standard startup IBeam performs the following:
@@ -258,4 +294,4 @@ Thanks and have an awesome day 👋
 
 [secret-manager-docs]: https://cloud.google.com/secret-manager/docs
 
-[ibind]: https://github.com/Voyz/ibind
+[ibind]: https://github.com/Voyz/ibind

env.list.example

@@ -0,0 +1,14 @@
+IBEAM_ACCOUNT=your_account123
+IBEAM_PASSWORD=your_password123
+
+# Enable this only if IBKR shows a "Select Second Factor Device" dropdown
+IBEAM_TWO_FA_SELECT=false
+
+# Defaults shown for reference. Override only if IBKR changes the page structure.
+IBEAM_TWO_FA_SELECT_EL_ID=TAG@@select
+IBEAM_TWO_FA_SELECT_TARGET=IB Key
+
+# Optional manual fallback when no 2FA handler is configured.
+# IBeam will wait for you to complete login manually in a normal browser.
+IBEAM_MANUAL_TWO_FA=false
+IBEAM_MANUAL_TWO_FA_TIMEOUT=300

ibeam/ibeam_starter.py

@@ -28,6 +28,25 @@ def add_to_path():
 
 _LOGGER = logging.getLogger('ibeam')
 
+_SENSITIVE_CONFIG_KEYS = {
+    'ACCOUNT',
+    'PASSWORD',
+    'KEY',
+    'SECRET',
+    'TOKEN',
+}
+
+
+def redact_config(config: dict) -> dict:
+    redacted = {}
+    for key, value in config.items():
+        if any(sensitive_key in key for sensitive_key in _SENSITIVE_CONFIG_KEYS):
+            redacted[key] = '***REDACTED***' if value is not None else None
+        else:
+            redacted[key] = value
+
+    return redacted
+
 def parse_args():
     parser = argparse.ArgumentParser(description='Start, authenticate and verify the IB Gateway.')
     parser.add_argument('-a', '--authenticate', action='store_true', help='Authenticates the existing gateway.')
@@ -100,10 +119,14 @@ def parse_args():
         targets=targets,
         base_url=cnf.GATEWAY_BASE_URL,
         route_auth=cnf.ROUTE_AUTH,
+        route_tickle=cnf.ROUTE_TICKLE,
         two_fa_select_target=cnf.TWO_FA_SELECT_TARGET,
         strict_two_fa_code=cnf.STRICT_TWO_FA_CODE,
+        manual_two_fa=cnf.MANUAL_TWO_FA,
+        manual_two_fa_timeout=cnf.MANUAL_TWO_FA_TIMEOUT,
         max_immediate_attempts=cnf.MAX_IMMEDIATE_ATTEMPTS,
         oauth_timeout=cnf.OAUTH_TIMEOUT,
+        request_timeout=cnf.REQUEST_TIMEOUT,
         max_presubmit_buffer=cnf.MAX_PRESUBMIT_BUFFER,
         min_presubmit_buffer=cnf.MIN_PRESUBMIT_BUFFER,
         max_failed_auth=cnf.MAX_FAILED_AUTH,
@@ -149,7 +172,7 @@ def stop(_, _1):
     signal.signal(signal.SIGINT, stop)
     signal.signal(signal.SIGTERM, stop)
 
-    _LOGGER.info(f'Configuration:\n{cnf.all_variables}')
+    _LOGGER.info(f'Configuration:\n{redact_config(cnf.all_variables)}')
 
     if args.start:
         pids = process_handler.start_gateway()
@@ -190,4 +213,4 @@ def stop(_, _1):
                 client.maintain()
         else:
             _LOGGER.info(f'IBeam initialised in an inactive state. Starting maintenance loop.')
-            client.maintain()
+            client.maintain()

ibeam/src/handlers/login_handler.py

@@ -1,8 +1,12 @@
+import json
 import logging
+import ssl
 import time
+import urllib.request
 from functools import partial
 from pathlib import Path
 from typing import Optional, cast
+from urllib.error import HTTPError, URLError
 
 from cryptography.fernet import Fernet
 from selenium import webdriver
@@ -113,10 +117,14 @@ def __init__(self,
                  targets: Targets,
                  base_url: str,
                  route_auth: str,
+                 route_tickle: str,
                  two_fa_select_target: str,
                  strict_two_fa_code: bool,
+                 manual_two_fa: bool,
+                 manual_two_fa_timeout: int,
                  max_immediate_attempts: int,
                  oauth_timeout: int,
+                 request_timeout: int,
                  max_presubmit_buffer: int,
                  min_presubmit_buffer: int,
                  max_failed_auth: int,
@@ -132,10 +140,14 @@ def __init__(self,
 
         self.base_url = base_url
         self.route_auth = route_auth
+        self.route_tickle = route_tickle
         self.two_fa_select_target = two_fa_select_target
         self.strict_two_fa_code = strict_two_fa_code
+        self.manual_two_fa = manual_two_fa
+        self.manual_two_fa_timeout = manual_two_fa_timeout
         self.max_immediate_attempts = max_immediate_attempts
         self.oauth_timeout = oauth_timeout
+        self.request_timeout = request_timeout
         self.max_presubmit_buffer = max_presubmit_buffer
         self.min_presubmit_buffer = min_presubmit_buffer
         self.max_failed_auth = max_failed_auth
@@ -146,6 +158,36 @@ def __init__(self,
         self.failed_attempts = 0
         self.presubmit_buffer = self.min_presubmit_buffer
 
+    def _wait_for_manual_gateway_authentication(self) -> bool:
+        deadline = time.time() + self.manual_two_fa_timeout
+        ssl_context = ssl._create_unverified_context()
+
+        while time.time() < deadline:
+            try:
+                response = urllib.request.urlopen(
+                    urllib.request.Request(self.base_url + self.route_tickle, method='POST'),
+                    context=ssl_context,
+                    timeout=self.request_timeout,
+                )
+                payload = json.loads(response.read().decode('utf8'))
+                auth_status = payload.get('iserver', {}).get('authStatus', {})
+                if auth_status.get('authenticated'):
+                    return True
+            except (HTTPError, URLError, TimeoutError, json.JSONDecodeError):
+                pass
+
+            time.sleep(5)
+
+        return False
+
+    def _find_two_fa_input_target(self, targets: Targets, driver: webdriver.Chrome) -> Target:
+        for target_name in ('TWO_FA_INPUT', 'TWO_FA_INPUT_GENERIC'):
+            target = targets[target_name]
+            if driver.find_elements(target.by, target.locator_identifier):
+                return target
+
+        return targets['TWO_FA_INPUT']
+
     def step_login(self,
                    targets: Targets,
                    wait_and_identify_trigger: callable,
@@ -198,10 +240,12 @@ def step_login(self,
         trigger, target = wait_and_identify_trigger(
             has_text(targets['SUCCESS']),
             is_visible(targets['TWO_FA']),
-            is_visible(targets['TWO_FA_SELECT']),
+            is_clickable(targets['TWO_FA_INPUT']),
+            is_clickable(targets['TWO_FA_INPUT_GENERIC']),
             is_visible(targets['TWO_FA_NOTIFICATION']),
             is_visible(targets['ERROR']),
             is_clickable(targets['IBKEY_PROMO']),
+            *( [is_visible(targets['TWO_FA_SELECT'])] if 'TWO_FA_SELECT' in targets else [] ),
         )
 
         return trigger, target
@@ -215,17 +259,43 @@ def step_select_two_fa(self,
         _LOGGER.info(f'Required to select a 2FA method.')
         select_el = find_element(targets['TWO_FA_SELECT'], driver)
         select = Select(select_el)
-        select.select_by_visible_text(two_fa_select_target)
+        option_texts = [option.text.strip() for option in select.options if option.text.strip()]
+        selectable_option_texts = [text for text in option_texts if text.lower() != 'select type']
+
+        _LOGGER.info(f'Available 2FA methods: {selectable_option_texts}')
+
+        selected_text = None
+
+        try:
+            select.select_by_visible_text(two_fa_select_target)
+            selected_text = two_fa_select_target
+        except Exception:
+            for option_text in selectable_option_texts:
+                if two_fa_select_target.lower() in option_text.lower():
+                    select.select_by_visible_text(option_text)
+                    selected_text = option_text
+                    break
+
+        if selected_text is None and len(selectable_option_texts) == 1:
+            selected_text = selectable_option_texts[0]
+            select.select_by_visible_text(selected_text)
+
+        if selected_text is None:
+            raise RuntimeError(
+                f'Unable to select 2FA method "{two_fa_select_target}". Available methods: {selectable_option_texts}'
+            )
 
         trigger, target = wait_and_identify_trigger(
             has_text(targets['SUCCESS']),
             is_visible(targets['TWO_FA']),
+            is_clickable(targets['TWO_FA_INPUT']),
+            is_clickable(targets['TWO_FA_INPUT_GENERIC']),
             is_visible(targets['TWO_FA_NOTIFICATION']),
             is_visible(targets['ERROR']),
             is_clickable(targets['IBKEY_PROMO'])
         )
 
-        _LOGGER.info(f'2FA method "{two_fa_select_target}" selected successfully.')
+        _LOGGER.info(f'2FA method "{selected_text}" selected successfully.')
         return trigger, target
 
 
@@ -261,6 +331,22 @@ def step_two_fa(self,
                     ):
         _LOGGER.info(f'Credentials correct, but Gateway requires two-factor authentication.')
         if two_fa_handler is None:
+            if self.manual_two_fa:
+                _LOGGER.warning(
+                    f'######## ATTENTION! ######## No 2FA handler found. Waiting up to {self.manual_two_fa_timeout} seconds for manual 2FA completion.'
+                )
+                _LOGGER.warning(
+                    f'Open {self.base_url} in your browser, complete the IBKR login manually, and keep IBeam running while it waits for the gateway session to become authenticated.'
+                )
+                save_screenshot(driver, self.outputs_dir, '__manual-two-fa')
+
+                if self._wait_for_manual_gateway_authentication():
+                    _LOGGER.info('Gateway authenticated after manual 2FA completion.')
+                    self.step_success()
+
+                _LOGGER.error('Manual 2FA timeout reached before the gateway session became authenticated.')
+                raise AttemptException(cause='break')
+
             _LOGGER.critical(
                 f'######## ATTENTION! ######## No 2FA handler found. You may define your own 2FA handler or use built-in handlers. See documentation for more: https://github.com/Voyz/ibeam/wiki/Two-Factor-Authentication')
             raise AttemptException(cause='shutdown')
@@ -271,7 +357,8 @@ def step_two_fa(self,
             _LOGGER.warning(f'No 2FA code returned. Aborting authentication.')
             raise AttemptException(cause='break')
         else:
-            two_fa_el, _ = wait_and_identify_trigger(is_clickable(targets['TWO_FA_INPUT']), skip_identify=True)
+            two_fa_input_target = self._find_two_fa_input_target(targets, driver)
+            two_fa_el, _ = wait_and_identify_trigger(is_clickable(two_fa_input_target), skip_identify=True)
 
             two_fa_el.clear()
             two_fa_el.send_keys(two_fa_code)
@@ -324,10 +411,12 @@ def step_paper_toggle(self,
         trigger, target = wait_and_identify_trigger(
             has_text(targets['SUCCESS']),
             is_visible(targets['TWO_FA']),
-            is_visible(targets['TWO_FA_SELECT']),
+            is_clickable(targets['TWO_FA_INPUT']),
+            is_clickable(targets['TWO_FA_INPUT_GENERIC']),
             is_visible(targets['TWO_FA_NOTIFICATION']),
             is_visible(targets['ERROR']),
             is_clickable(targets['IBKEY_PROMO']),
+            *( [is_visible(targets['TWO_FA_SELECT'])] if 'TWO_FA_SELECT' in targets else [] ),
         )
 
         return trigger, target
@@ -418,13 +507,13 @@ def attempt(
         if target == targets['ERROR'] and trigger.text == 'You have selected the Live Account Mode, but the specified user is a Paper Trading user. Please select the correct Login mode.':
             trigger, target = self.step_paper_toggle(driver, targets, wait_and_identify_trigger)
 
-        if target == targets['TWO_FA_SELECT']:
+        if 'TWO_FA_SELECT' in targets and target == targets['TWO_FA_SELECT']:
             trigger, target = self.step_select_two_fa(targets, wait_and_identify_trigger, driver, self.two_fa_select_target)
 
         if target == targets['TWO_FA_NOTIFICATION']:
             trigger, target = self.step_two_fa_notification(targets, wait_and_identify_trigger, driver, self.two_fa_handler)
 
-        if target == targets['TWO_FA']:
+        if target == targets['TWO_FA'] or target == targets['TWO_FA_INPUT'] or target == targets['TWO_FA_INPUT_GENERIC']:
             trigger, target = self.step_two_fa(targets, wait_and_identify_trigger, driver, self.two_fa_handler, self.strict_two_fa_code)
 
         if target == targets['IBKEY_PROMO']:
@@ -433,7 +522,7 @@ def attempt(
         if target == targets['ERROR']:
             self.step_error(driver, trigger, self.max_presubmit_buffer, self.max_failed_auth, self.outputs_dir)
 
-        elif target == targets['TWO_FA']:
+        elif target == targets['TWO_FA'] or target == targets['TWO_FA_INPUT'] or target == targets['TWO_FA_INPUT_GENERIC']:
             self.step_failed_two_fa(driver)
 
         elif target == targets['SUCCESS']:
@@ -525,4 +614,4 @@ def login(self) -> (bool, bool):
         finally:
             shut_down_browser(driver, display)
 
-        return success, False
+        return success, False