[TDL-15656] Added Support for dev mode

tap_eloqua/__init__.py

@@ -51,6 +51,8 @@ def parse_args(required_config_keys):
     -d,--discover   Run in discover mode
     -p,--properties Properties file: DEPRECATED, please use --catalog instead
     --catalog       Catalog file
+    -dev, --dev     Runs the tap in dev mode
+
 
     Returns the parsed args object from argparse. For each argument that
     point to JSON files (config, state, properties), we will automatically
@@ -80,6 +82,11 @@ def parse_args(required_config_keys):
         action='store_true',
         help='Do schema discovery')
 
+    parser.add_argument(
+        '-dev', '--dev',
+        action='store_true',
+        help='Runs tap in dev mode')
+
     args = parser.parse_args()
     if args.config:
         setattr(args, 'config_path', args.config)
@@ -106,13 +113,16 @@ def parse_args(required_config_keys):
 def main():
     #parsed_args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS)
     parsed_args = parse_args(REQUIRED_CONFIG_KEYS)
-
+    if parsed_args.dev:
+        LOGGER.warning("Executing Tap in Dev mode",)    
     with EloquaClient(parsed_args.config_path,
                       parsed_args.config['client_id'],
                       parsed_args.config['client_secret'],
                       parsed_args.config['refresh_token'],
                       parsed_args.config['redirect_uri'],
-                      parsed_args.config.get('user_agent')) as client:
+                      parsed_args.config.get('user_agent'),
+                      parsed_args.dev
+                      ) as client:
 
         if parsed_args.discover:
             do_discover(client)

tap_eloqua/client.py

@@ -1,28 +1,36 @@
 import json
 from datetime import datetime, timedelta
+import sys
 
 import backoff
 import requests
 from requests.exceptions import ConnectionError
-from singer import metrics
+from singer import metrics,get_logger
+from singer.utils import strptime_to_utc,now,strftime
+from .utils import read_config,write_config
+LOGGER = get_logger()
+
 
 class Server5xxError(Exception):
     pass
 
+
 class EloquaClient(object):
     def __init__(self,
                  config_path,
                  client_id,
                  client_secret,
                  refresh_token,
                  redirect_uri,
-                 user_agent):
+                 user_agent,
+                 dev_mode=False):
         self.__config_path = config_path
         self.__client_id = client_id
         self.__client_secret = client_secret
         self.__refresh_token = refresh_token
         self.__redirect_uri = redirect_uri
         self.__user_agent = user_agent
+        self.dev_mode = dev_mode
         self.__access_token = None
         self.__expires = None
         self.__session = requests.Session()
@@ -40,7 +48,21 @@ def __exit__(self, type, value, traceback):
                           max_tries=5,
                           factor=2)
     def get_access_token(self):
-        if self.__access_token is not None and self.__expires > datetime.utcnow():
+        if self.dev_mode:
+            config = read_config(self.__config_path)
+            try:
+                self.__access_token = config['access_token']
+                self.__refresh_token = config['refresh_token']
+                self.__expires=strptime_to_utc(config['expires_in'])
+            except KeyError as _:
+                LOGGER.fatal("Unable to locate key %s in config",_)
+                raise Exception("Unable to locate key in config")
+            if self.__access_token is not None and self.__expires > now():
+                return
+            LOGGER.fatal("Access Token in config is expired, unable to authenticate in dev mode")
+            raise Exception("Access Token in config is expired, unable to authenticate in dev mode")
+
+        if self.__access_token is not None and self.__expires > now():
             return
 
         headers = {}
@@ -70,19 +92,14 @@ def get_access_token(self):
                     eloqua_response))
 
         data = response.json()
-
         self.__access_token = data['access_token']
         self.__refresh_token = data['refresh_token']
-
-        ## refresh_token rotates on every reauth
-        with open(self.__config_path) as file:
-            config = json.load(file)
-        config['refresh_token'] = data['refresh_token']
-        with open(self.__config_path, 'w') as file:
-            json.dump(config, file, indent=2)
-
         expires_seconds = data['expires_in'] - 10 # pad by 10 seconds
-        self.__expires = datetime.utcnow() + timedelta(seconds=expires_seconds)
+        self.__expires = now() + timedelta(seconds=expires_seconds)
+
+        if not self.dev_mode:
+            update_config_keys = {"refresh_token":self.__refresh_token,"access_token":self.__access_token,"expires_in": strftime(self.__expires)}
+            config = write_config(self.__config_path,update_config_keys)
 
     def get_base_urls(self):
         data = self.request('GET',
@@ -135,3 +152,4 @@ def get(self, path, **kwargs):
 
     def post(self, path, **kwargs):
         return self.request('POST', path=path, **kwargs)
+

tap_eloqua/utils.py

@@ -0,0 +1,21 @@
+from typing import Dict
+import json
+from singer import get_logger
+
+LOGGER = get_logger()
+
+def read_config(config_path) -> Dict:
+    try:
+        with open(config_path,'r') as tap_config:
+            return json.load(tap_config)
+    except FileNotFoundError as _:
+        LOGGER.fatal("Failed to load config in dev mode")
+        return {}
+
+
+def write_config(config_path,data :Dict) -> Dict:
+    config =  read_config(config_path)
+    config.update(data)
+    with open(config_path,'w') as tap_config:
+        json.dump(config, tap_config, indent=2)
+    return config

tests/unittests/test_devmode.py

@@ -0,0 +1,102 @@
+import json
+import unittest
+from tap_eloqua.client import EloquaClient, strftime
+from unittest.mock import patch
+import datetime
+import pytz
+
+
+class MockResponse:
+    def __init__(self, json_data, status_code):
+        self.json_data = json_data
+        self.status_code = status_code
+
+    def json(self):
+        return self.json_data
+
+
+def mocked_auth_post(*args, **kwargs):
+    return MockResponse({"access_token": "hellothere", "refresh_token": "hello", "expires_in": 5000}, status_code=200)
+
+
+class Test_ClientDevMode(unittest.TestCase):
+    """Test the dev mode functionality"""
+
+    base_config = {"client_id": "",
+                   "client_secret": "",
+                   "refresh_token": "",
+                   "redirect_uri": "",
+                   "user_agent": "", }
+    tmp_config_path = "/tmp/eloqua_config.json"
+
+
+    @patch("requests.Session.post", side_effect=mocked_auth_post)
+    def test_client_without_dev_mode(self, mocked_auth_post):
+        """checks if the client can write refresh token and expiry to config"""
+
+        config_sample = self.base_config
+
+        with open(self.tmp_config_path, "w") as ff:
+            json.dump(config_sample, ff)
+
+        params = {"config_path": self.tmp_config_path, "dev_mode": False}
+        params.update(self.base_config)
+
+        client = EloquaClient(**params)
+        client.get_access_token()
+        with open(self.tmp_config_path, "r") as config_file:
+            config = json.load(config_file)
+            self.assertEqual(True, "expires_in" in config)
+            self.assertEqual(True, "access_token" in config)
+            self.assertEqual(True, "refresh_token" in config)
+
+    def test_devmode_accesstoken_absent(self, *args, **kwargs):
+        """
+        checks exception if access token is not present in config
+        """
+        config_sample = self.base_config
+
+        with open(self.tmp_config_path, "w") as config_file:
+            json.dump(config_sample, config_file)
+
+        params = {"config_path": self.tmp_config_path, "dev_mode": True}
+        params.update(self.base_config)
+        try:
+            client = EloquaClient(**params)
+            client.get_access_token()
+        except Exception as _:
+            self.assertEqual(str(_), "Unable to locate key in config")
+
+    @patch("requests.Session.post", side_effect=mocked_auth_post)
+    def test_client_valid_token(self, mocked_auth_post):
+        """checks if the client can fetch and validate refresh token and expiry from config"""
+
+        self.tmp_config_path = "/tmp/eloqua_config.json"
+        config_sample = {
+            "access_token": "hellothere",
+            "expires_in": strftime(datetime.datetime.utcnow().replace(tzinfo=pytz.UTC) + datetime.timedelta(minutes=5))}
+        config_sample.update(self.base_config)
+        with open(self.tmp_config_path, "w") as config_file:
+            json.dump(config_sample, config_file)
+        params = {"config_path": self.tmp_config_path, "dev_mode": True}
+        params.update(self.base_config)
+        EloquaClient(**params).get_access_token()
+
+
+    @patch("requests.Session.post", side_effect=mocked_auth_post)
+    def test_client_invalid_token(self, mocked_auth_post):
+        """checks if the client can fetch and validate refresh token and expiry from config"""
+
+        config_sample = {
+            "access_token": "hellothere",
+            "expires_in": strftime(datetime.datetime.utcnow().replace(tzinfo=pytz.UTC) - datetime.timedelta(minutes=5))}
+        config_sample.update(self.base_config)
+        with open(self.tmp_config_path, "w") as config_file:
+            json.dump(config_sample, config_file)
+        params = {"config_path": self.tmp_config_path, "dev_mode": True}
+        params.update(self.base_config)
+        try:
+            EloquaClient(**params).get_access_token()
+        except Exception as _:
+            self.assertEqual(
+                str(_), "Access Token in config is expired, unable to authenticate in dev mode")