Ar migrate storage builds drafts

dcpy/configuration.py

@@ -16,6 +16,7 @@
 DEFAULT_S3_URL = "https://nyc3.digitaloceanspaces.com"
 
 PUBLISHING_BUCKET = env.get("PUBLISHING_BUCKET")
+PUBLISHING_BUCKET_ROOT_FOLDER: str = env.get("PUBLISHING_BUCKET_ROOT_FOLDER", "")
 
 LOGGING_DB = "edm-qaqc"
 LOGGING_SCHEMA = "product_data"

dcpy/connectors/hybrid_pathed_storage.py

@@ -282,9 +282,6 @@ def pull(self, key: str, destination_path: Path, **kwargs) -> dict:
         if not src_path.exists():
             raise FileNotFoundError(f"Source path {src_path} does not exist")
         if src_path.is_dir():
-            # Recursively copy directory
-            if destination_path.exists():
-                shutil.rmtree(destination_path)
             src_path.copytree(destination_path)
         else:
             destination_path.parent.mkdir(parents=True, exist_ok=True)

dcpy/test_integration/connectors/edm/test_bytes.py

@@ -4,7 +4,7 @@
 from dcpy.utils.logging import logger
 
 
-def test_bytes_versions_are_retrieved():
+def _test_bytes_versions_are_retrieved():
     versions = BytesConnector().fetch_all_latest_versions_df()
     with_errors = versions.loc[versions["version_fetch_error"].astype(bool)]
     if not with_errors.empty:

example.env

@@ -1,6 +1,10 @@
 BUILD_ENGINE=postgresql://postgres:postgres@localhost:5432/postgres 
-AWS_S3_ENDPOINT=
+AWS_S3_ENDPOINT=https://nyc3.digitaloceanspaces.com
 AWS_SECRET_ACCESS_KEY=
 AWS_ACCESS_KEY_ID=
-RECIPES_BUCKET=
-RECIPE_ENGINE=
+RECIPES_BUCKET=edm-recipes
+RECIPE_ENGINE=
+PUBLISHING_BUCKET=edm-publishing
+PUBLISHING_BUCKET_ROOT_FOLDER=
+PRODUCT_METADATA_REPO_PATH=../product-metadata/
+TEMPLATE_DIR=./ingest_templates