Merge pull request #47 from data-integrations/feature/CDAP-14477-action-path

CDAP-14477 Remove bucket from GCS Create/Delete action

Author: rohitsinha54

docs/GCSBucketCreate-action.md

@@ -30,9 +30,7 @@ Properties
 **Project ID**: Google Cloud Project ID, which uniquely identifies a project.
 It can be found on the Dashboard in the Google Cloud Platform Console.
 
-**Bucket Name**: Bucket to create objects in.
-
-**Objects to Create**: Objects to create in the bucket.
+**Objects to Create**: Comma separated list of objects to create.
 
 **Fail if Object Exists**: Whether to fail the pipeline if an object already exists.
 

docs/GCSBucketDelete-action.md

@@ -30,9 +30,7 @@ Properties
 **Project ID**: The Google Cloud Project ID, which uniquely identifies a project.
 It can be found on the Dashboard in the Google Cloud Platform Console.
 
-**Bucket Name**: The bucket to delete objects from.
-
-**Objects to Delete**: Objects to delete from the bucket.
+**Objects to Delete**: Comma separated list of objects to delete.
 
 **Service Account File Path**: Path on the local file system of the service account key used for
 authorization. Can be set to 'auto-detect' when running on a Dataproc cluster.

src/main/java/co/cask/gcp/gcs/actions/GCSBucketCreate.java

@@ -23,6 +23,7 @@
 import co.cask.cdap.etl.api.action.Action;
 import co.cask.cdap.etl.api.action.ActionContext;
 import co.cask.gcp.common.GCPConfig;
+import co.cask.gcp.gcs.GCSConfigHelper;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -31,7 +32,9 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
+import java.util.stream.Collectors;
 
 
 /**
@@ -58,18 +61,12 @@ public void run(ActionContext context) throws Exception {
     String projectId = config.getProject();
     configuration.set("fs.gs.project.id", projectId);
 
-    configuration.set("fs.gs.system.bucket", config.bucket);
     configuration.setBoolean("fs.gs.impl.disable.cache", true);
     configuration.setBoolean("fs.gs.metadata.cache.enable", false);
 
-    String[] paths = config.paths.split(",");
     List<Path> gcsPaths = new ArrayList<>();
-    for (String path : paths) {
-      String gcsPath = String.format("gs://%s/%s", config.bucket, path);
-      if (path.startsWith("/")) {
-        gcsPath = String.format("gs://%s%s", config.bucket, path);
-      }
-      gcsPaths.add(new Path(gcsPath));
+    for (String path : config.getPaths()) {
+      gcsPaths.add(new Path(GCSConfigHelper.getPath(path)));
     }
 
     FileSystem fs;
@@ -97,10 +94,10 @@ public void run(ActionContext context) throws Exception {
             throw e;
           }
         } else {
-          if (config.failIfExists.equalsIgnoreCase("yes")) {
+          if (config.getFailIfExists()) {
             rollback = true;
             throw new Exception(
-              String.format("Object %s already exists")
+              String.format("Object %s already exists", gcsPath)
             );
           }
         }
@@ -126,20 +123,22 @@ public void run(ActionContext context) throws Exception {
    * Config for the plugin.
    */
   public final class Config extends GCPConfig {
-
-    @Name("bucket")
-    @Description("Name of the bucket.")
-    @Macro
-    public String bucket;
-
     @Name("paths")
-    @Description("List of objects to be created within a bucket")
+    @Description("Comma separated list of objects to be created.")
     @Macro
-    public String paths;
+    private String paths;
 
     @Name("failIfExists")
     @Description("Fail if path exists.")
     @Macro
-    public String failIfExists;
+    private boolean failIfExists;
+
+    public List<String> getPaths() {
+      return Arrays.stream(paths.split(",")).map(String::trim).collect(Collectors.toList());
+    }
+
+    public boolean getFailIfExists() {
+      return failIfExists;
+    }
   }
 }

src/main/java/co/cask/gcp/gcs/actions/GCSBucketDelete.java

@@ -23,6 +23,7 @@
 import co.cask.cdap.etl.api.action.Action;
 import co.cask.cdap.etl.api.action.ActionContext;
 import co.cask.gcp.common.GCPConfig;
+import co.cask.gcp.gcs.GCSConfigHelper;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -31,12 +32,13 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
+import java.util.stream.Collectors;
 
 
 /**
- * This action plugin <code>GCSBucketDelete</code> provides a way to delete
- * directories within a given GCS bucket.
+ * This action plugin <code>GCSBucketDelete</code> provides a way to delete directories within a given GCS bucket.
  */
 @Plugin(type = Action.PLUGIN_TYPE)
 @Name(GCSBucketDelete.NAME)
@@ -59,17 +61,11 @@ public void run(ActionContext context) throws Exception {
     String projectId = config.getProject();
     configuration.set("fs.gs.project.id", projectId);
 
-    configuration.set("fs.gs.system.bucket", config.bucket);
     configuration.setBoolean("fs.gs.impl.disable.cache", true);
 
-    String[] paths = config.paths.split(",");
     List<Path> gcsPaths = new ArrayList<>();
-    for (String path : paths) {
-      String gcsPath = String.format("gs://%s/%s", config.bucket, path);
-      if (path.startsWith("/")) {
-        gcsPath = String.format("gs://%s%s", config.bucket, path);
-      }
-      gcsPaths.add(new Path(gcsPath));
+    for (String path : config.getPaths()) {
+      gcsPaths.add(new Path(GCSConfigHelper.getPath(path)));
     }
 
     FileSystem fs;
@@ -98,15 +94,13 @@ public void run(ActionContext context) throws Exception {
    * Config for the plugin.
    */
   public final class Config extends GCPConfig {
-
-    @Name("bucket")
-    @Description("Name of the bucket.")
-    @Macro
-    public String bucket;
-
     @Name("paths")
-    @Description("List of objects to be deleted within a bucket")
+    @Description("Comma separated list of objects to be deleted.")
     @Macro
-    public String paths;
+    private String paths;
+
+    public List<String> getPaths() {
+      return Arrays.stream(paths.split(",")).map(String::trim).collect(Collectors.toList());
+    }
   }
 }

widgets/GCSBucketCreate-action.json

@@ -15,14 +15,6 @@
             "default": "auto-detect"
           }
         },
-        {
-          "widget-type": "textbox",
-          "label": "Bucket Name",
-          "name": "bucket",
-          "widget-attributes" : {
-            "placeholder": "Bucket to create objects in"
-          }
-        },
         {
           "name": "paths",
           "widget-type": "csv",

widgets/GCSBucketDelete-action.json

@@ -15,14 +15,6 @@
             "default": "auto-detect"
           }
         },
-        {
-          "widget-type": "textbox",
-          "label": "Bucket Name",
-          "name": "bucket",
-          "widget-attributes" : {
-            "placeholder": "Bucket to delete objects from"
-          }
-        },
         {
           "name": "paths",
           "widget-type": "csv",