Issues #101, #102, #103, #104

README.md

@@ -229,20 +229,25 @@ supports the `$now` operator for `inputFormat`, which will set the current
 timestamp at the specified path, formatted according to the `outputFormat`.
 `$unix` is supported for both input and output formats as a Unix time, the
 number of seconds elapsed since January 1, 1970 UTC as an integer string.
+Usage of `$unixext` will reference to unix number of milliseconds since 
+January 1, 1970 UTC .
+
 ```javascript
 {
   "operation": "timestamp",
-  "timestamp[0]": {
-    "inputFormat": "Mon Jan _2 15:04:05 -0700 2006",
-    "outputFormat": "2006-01-02T15:04:05-0700"
-  },
-  "nowTimestamp": {
-    "inputFormat": "$now",
-    "outputFormat": "2006-01-02T15:04:05-0700"
-  },
-  "epochTimestamp": {
-    "inputFormat": "2006-01-02T15:04:05-0700",
-    "outputFormat": "$unix"
+  "spec": {
+    "timestamp[0]": {
+      "inputFormat": "Mon Jan _2 15:04:05 -0700 2006",
+      "outputFormat": "2006-01-02T15:04:05-0700"
+    },
+    "nowTimestamp": {
+      "inputFormat": "$now",
+      "outputFormat": "2006-01-02T15:04:05-0700"
+    },
+    "epochTimestamp": {
+      "inputFormat": "2006-01-02T15:04:05-0000",
+      "outputFormat": "$unix"
+    }
   }
 }
 
@@ -266,8 +271,9 @@ would result in
     "2017-07-22T08:15:27+0000",
     "Sun Jul 23 08:15:27 +0000 2017",
     "Mon Jul 24 08:15:27 +0000 2017"
-  ]
-  "nowTimestamp": "2017-09-08T19:15:27+0000"
+  ],
+  "nowTimestamp": "2017-09-08T19:15:27+0000",
+  "epochTimestamp": 1136210645
 }
 ```
 

go.mod

@@ -0,0 +1,8 @@
+module github.com/qntfy/kazaam
+
+go 1.14
+
+require (
+	github.com/gofrs/uuid v3.2.0+incompatible
+	github.com/qntfy/jsonparser v1.0.2
+)

go.sum

@@ -0,0 +1,5 @@
+github.com/gofrs/uuid v1.2.0 h1:coDhrjgyJaglxSjxuJdqQSSdUpG3w6p1OwN2od6frBU=
+github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
+github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/qntfy/jsonparser v1.0.2 h1:hko+J4L7HSaYoB2yuzinWc9MkO93zWKUmzPHJwB53OM=
+github.com/qntfy/jsonparser v1.0.2/go.mod h1:F+LCdwPnFBsubQ+ugnBczIP9RWv5wSCqnUmLHPUx4ZU=

kazaam_int_test.go

@@ -238,6 +238,47 @@ func TestKazaamShiftTransformWithTimestamp(t *testing.T) {
 	}
 }
 
+func TestKazaamShiftTransformWithUnixTimestamp(t *testing.T) {
+	spec := `[{
+		"operation": "shift",
+		"spec": {"newTimestamp":"oldTimestamp","oldTimestamp":"oldTimestamp","unixOutput":"oldTimestamp","unixExtOutput":"oldTimestamp", "unixInput":"unixInput", "unixExtInput":"unixExtInput"}
+	}, {
+		"operation": "timestamp",
+		"spec": {
+			"newTimestamp":{"inputFormat":"Mon Jan _2 15:04:05 -0700 2006","outputFormat":"2006-01-02T15:04:05-0700"},
+			"unixOutput":{"inputFormat":"Mon Jan _2 15:04:05 -0700 2006","outputFormat":"$unix"},
+			"unixExtOutput":{"inputFormat":"Mon Jan _2 15:04:05 -0700 2006","outputFormat":"$unixext"},
+			"unixInput":{"inputFormat": "$unix", "outputFormat": "Mon Jan _2 15:04:05 -0700 2006"},
+			"unixExtInput":{"inputFormat": "$unixext", "outputFormat": "Mon Jan _2 15:04:05 -0700 2006"}
+		}
+	}]`
+	jsonIn := `{"oldTimestamp":"Fri Jul 21 08:15:27 +0000 2017",  "unixInput": 1587999255, , "unixExtInput": 1587999255123}`
+	jsonOut := `{"oldTimestamp":"Fri Jul 21 08:15:27 +0000 2017","newTimestamp":"2017-07-21T08:15:27+0000", "unixOutput": 1500624927, "unixExtOutput":1500624927000, "unixInput": "Mon Apr 27 16:54:15 +0200 2020", "unixExtInput": "Mon Apr 27 16:54:15 +0200 2020"}`
+
+	kazaamTransform, err := kazaam.NewKazaam(spec)
+	if err != nil {
+		t.Error("Init produced error.")
+		t.Log("Error: ", err.Error())
+		t.FailNow()
+	}
+	kazaamOut, err := kazaamTransform.TransformJSONStringToString(jsonIn)
+	if err != nil {
+		t.Error("Transform produced error.")
+		t.Log("Error: ", err.Error())
+		t.FailNow()
+	}
+	areEqual, _ := checkJSONStringsEqual(kazaamOut, jsonOut)
+	t.Log("Expected: ", jsonOut)
+	t.Log("Actual:   ", kazaamOut)
+
+	if !areEqual {
+		t.Error("Transformed data does not match expectation.")
+		t.Log("Expected: ", jsonOut)
+		t.Log("Actual:   ", kazaamOut)
+		t.FailNow()
+	}
+}
+
 func TestShiftWithOverAndWildcard(t *testing.T) {
 	spec := `[{"operation": "shift","spec": {"docs": "documents[*]"}}, {"operation": "shift",  "spec": {"data": "norm.text"}, "over":"docs"}]`
 	jsonIn := `{"documents":[{"norm":{"text":"String 1"}},{"norm":{"text":"String 2"}}]}`
@@ -489,17 +530,17 @@ func TestKazaamOverArrayStrings(t *testing.T) {
                 "over": "doc.guidObjects",
                 "spec": {"raw": "$"}
         }]`
-        jsonIn := `{"doc":{"guidObjects":["foo",5,false]}}`
-        jsonOut := `{"doc":{"guidObjects":[{"raw":"foo"},{"raw":5},{"raw":false}]}}`
-
-        kazaamTransform, _ := kazaam.NewKazaam(spec)
-        kazaamOut, _ := kazaamTransform.TransformJSONStringToString(jsonIn)
-        areEqual, _ := checkJSONStringsEqual(kazaamOut, jsonOut)
-
-        if !areEqual {
-                t.Error("Transformed data does not match expectation.")
-                t.Log("Expected: ", jsonOut)
-                t.Log("Actual:   ", kazaamOut)
-                t.FailNow()
-        }
+	jsonIn := `{"doc":{"guidObjects":["foo",5,false]}}`
+	jsonOut := `{"doc":{"guidObjects":[{"raw":"foo"},{"raw":5},{"raw":false}]}}`
+
+	kazaamTransform, _ := kazaam.NewKazaam(spec)
+	kazaamOut, _ := kazaamTransform.TransformJSONStringToString(jsonIn)
+	areEqual, _ := checkJSONStringsEqual(kazaamOut, jsonOut)
+
+	if !areEqual {
+		t.Error("Transformed data does not match expectation.")
+		t.Log("Expected: ", jsonOut)
+		t.Log("Actual:   ", kazaamOut)
+		t.FailNow()
+	}
 }

transform/timestamp.go

@@ -14,6 +14,7 @@ import (
 var now = time.Now
 
 const unixFormat = "$unix"
+const unixExtendedFormat = "$unixext"
 
 // Timestamp parses and formats timestamp strings using the golang syntax
 func Timestamp(spec *Config, data []byte) ([]byte, error) {
@@ -59,8 +60,8 @@ func Timestamp(spec *Config, data []byte) ([]byte, error) {
 		}
 		// can only parse and format strings and arrays of strings, check the
 		// value type and handle accordingly
-		switch dataForV[0] {
-		case '"':
+		switch {
+		case dataForV[0] == '"':
 			formattedItem, err := parseAndFormatValue(inputFormat, outputFormat, string(dataForV[1:len(dataForV)-1]))
 			if err != nil {
 				return nil, err
@@ -69,7 +70,16 @@ func Timestamp(spec *Config, data []byte) ([]byte, error) {
 			if err != nil {
 				return nil, err
 			}
-		case '[':
+		case (dataForV[0] >= '0') && (dataForV[0] <= '9'):
+			formattedItem, err := parseAndFormatValue(inputFormat, outputFormat, string(dataForV))
+			if err != nil {
+				return nil, err
+			}
+			data, err = setJSONRaw(data, []byte(formattedItem), k, spec.KeySeparator)
+			if err != nil {
+				return nil, err
+			}
+		case dataForV[0] == '[':
 			var unformattedItems []string
 			_, err = jsonparser.ArrayEach(dataForV, func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
 				unformattedItems = append(unformattedItems, string(value))
@@ -110,6 +120,12 @@ func parseAndFormatValue(inputFormat, outputFormat, unformattedItem string) (str
 			return "", err
 		}
 		parsedItem = time.Unix(i, 0)
+	} else if inputFormat == unixExtendedFormat {
+		i, err := strconv.ParseInt(unformattedItem, 10, 64)
+		if err != nil {
+			return "", err
+		}
+		parsedItem = time.Unix(0, int64(i)*int64(time.Millisecond))
 	} else {
 		parsedItem, err = time.Parse(inputFormat, unformattedItem)
 		if err != nil {
@@ -119,8 +135,11 @@ func parseAndFormatValue(inputFormat, outputFormat, unformattedItem string) (str
 
 	if outputFormat == unixFormat {
 		formattedItem = strconv.FormatInt(parsedItem.Unix(), 10)
+	} else if outputFormat == unixExtendedFormat {
+		formattedItem = strconv.FormatInt(parsedItem.UnixNano()/1000000, 10)
 	} else {
 		formattedItem = parsedItem.Format(outputFormat)
+		formattedItem = strings.Join([]string{"\"", formattedItem, "\""}, "")
 	}
-	return strings.Join([]string{"\"", formattedItem, "\""}, ""), nil
+	return formattedItem, nil
 }

transform/timestamp_test.go

@@ -170,6 +170,7 @@ func TestParseAndFormatValue(t *testing.T) {
 		{time.RFC3339, "\"2017-07-21T08:15:27+01:00\""},
 		{time.StampNano, "\"Jul 21 08:15:27.000000000\""},
 		{"$unix", "\"1500621327\""},
+		{"$unixext", "\"1500621327000\""},
 	}
 	for _, testItem := range parseAndFormatTests {
 		actual, _ := parseAndFormatValue(inputFormat, testItem.outputFormat, inputTimestamp)
@@ -194,6 +195,7 @@ func TestParseAndFormatValueOutputUnix(t *testing.T) {
 		{time.UnixDate, "Fri Jul 21 08:15:27 GMT 2017", "\"1500624927\""},
 		{time.RFC3339, "2017-07-21T08:15:27+01:00", "\"1500621327\""},
 		{"$unix", "1500621327", "\"1500621327\""},
+		{"$unixext", "1500621327567", "\"1500621327\""},
 	}
 	for _, testItem := range parseAndFormatTests {
 		actual, _ := parseAndFormatValue(testItem.inputFormat, "$unix", testItem.inputTimestamp)
@@ -204,3 +206,28 @@ func TestParseAndFormatValueOutputUnix(t *testing.T) {
 		}
 	}
 }
+
+func TestParseAndFormatValueOutputUnixExt(t *testing.T) {
+	parseAndFormatTests := []struct {
+		inputFormat    string
+		inputTimestamp string
+		expectedOutput string
+	}{
+		// test against a sampling of common formats
+		{"2006-01-02T15:04:05-0700", "2017-07-21T08:15:27+0100", "\"1500621327000\""},
+		{"January _2, 2006", "July 21, 2017", "\"1500595200000\""},
+		{time.ANSIC, "Fri Jul 21 08:15:27 2017", "\"1500624927000\""},
+		{time.UnixDate, "Fri Jul 21 08:15:27 GMT 2017", "\"1500624927000\""},
+		{time.RFC3339, "2017-07-21T08:15:27+01:00", "\"1500621327000\""},
+		{"$unix", "1500621327", "\"1500621327000\""},
+		{"$unixext", "1500621327234", "\"1500621327234\""},
+	}
+	for _, testItem := range parseAndFormatTests {
+		actual, _ := parseAndFormatValue(testItem.inputFormat, "$unixext", testItem.inputTimestamp)
+		if actual != testItem.expectedOutput {
+			t.Error("Error data does not match expectation.", testItem.inputFormat)
+			t.Log("Expected:   ", testItem.expectedOutput)
+			t.Log("Actual:     ", actual)
+		}
+	}
+}