diff --git a/README.md b/README.md index 607e18f..4847739 100644 --- a/README.md +++ b/README.md @@ -229,20 +229,25 @@ supports the `$now` operator for `inputFormat`, which will set the current timestamp at the specified path, formatted according to the `outputFormat`. `$unix` is supported for both input and output formats as a Unix time, the number of seconds elapsed since January 1, 1970 UTC as an integer string. +Usage of `$unixext` will reference to unix number of milliseconds since +January 1, 1970 UTC . + ```javascript { "operation": "timestamp", - "timestamp[0]": { - "inputFormat": "Mon Jan _2 15:04:05 -0700 2006", - "outputFormat": "2006-01-02T15:04:05-0700" - }, - "nowTimestamp": { - "inputFormat": "$now", - "outputFormat": "2006-01-02T15:04:05-0700" - }, - "epochTimestamp": { - "inputFormat": "2006-01-02T15:04:05-0700", - "outputFormat": "$unix" + "spec": { + "timestamp[0]": { + "inputFormat": "Mon Jan _2 15:04:05 -0700 2006", + "outputFormat": "2006-01-02T15:04:05-0700" + }, + "nowTimestamp": { + "inputFormat": "$now", + "outputFormat": "2006-01-02T15:04:05-0700" + }, + "epochTimestamp": { + "inputFormat": "2006-01-02T15:04:05-0000", + "outputFormat": "$unix" + } } } @@ -266,8 +271,9 @@ would result in "2017-07-22T08:15:27+0000", "Sun Jul 23 08:15:27 +0000 2017", "Mon Jul 24 08:15:27 +0000 2017" - ] - "nowTimestamp": "2017-09-08T19:15:27+0000" + ], + "nowTimestamp": "2017-09-08T19:15:27+0000", + "epochTimestamp": 1136210645 } ``` diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..44613e4 --- /dev/null +++ b/go.mod @@ -0,0 +1,8 @@ +module github.com/qntfy/kazaam + +go 1.14 + +require ( + github.com/gofrs/uuid v3.2.0+incompatible + github.com/qntfy/jsonparser v1.0.2 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..05e18ad --- /dev/null +++ b/go.sum @@ -0,0 +1,5 @@ +github.com/gofrs/uuid v1.2.0 h1:coDhrjgyJaglxSjxuJdqQSSdUpG3w6p1OwN2od6frBU= +github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/qntfy/jsonparser v1.0.2 h1:hko+J4L7HSaYoB2yuzinWc9MkO93zWKUmzPHJwB53OM= +github.com/qntfy/jsonparser v1.0.2/go.mod h1:F+LCdwPnFBsubQ+ugnBczIP9RWv5wSCqnUmLHPUx4ZU= diff --git a/kazaam_int_test.go b/kazaam_int_test.go index 6650ad6..6cab039 100644 --- a/kazaam_int_test.go +++ b/kazaam_int_test.go @@ -238,6 +238,47 @@ func TestKazaamShiftTransformWithTimestamp(t *testing.T) { } } +func TestKazaamShiftTransformWithUnixTimestamp(t *testing.T) { + spec := `[{ + "operation": "shift", + "spec": {"newTimestamp":"oldTimestamp","oldTimestamp":"oldTimestamp","unixOutput":"oldTimestamp","unixExtOutput":"oldTimestamp", "unixInput":"unixInput", "unixExtInput":"unixExtInput"} + }, { + "operation": "timestamp", + "spec": { + "newTimestamp":{"inputFormat":"Mon Jan _2 15:04:05 -0700 2006","outputFormat":"2006-01-02T15:04:05-0700"}, + "unixOutput":{"inputFormat":"Mon Jan _2 15:04:05 -0700 2006","outputFormat":"$unix"}, + "unixExtOutput":{"inputFormat":"Mon Jan _2 15:04:05 -0700 2006","outputFormat":"$unixext"}, + "unixInput":{"inputFormat": "$unix", "outputFormat": "Mon Jan _2 15:04:05 -0700 2006"}, + "unixExtInput":{"inputFormat": "$unixext", "outputFormat": "Mon Jan _2 15:04:05 -0700 2006"} + } + }]` + jsonIn := `{"oldTimestamp":"Fri Jul 21 08:15:27 +0000 2017", "unixInput": 1587999255, , "unixExtInput": 1587999255123}` + jsonOut := `{"oldTimestamp":"Fri Jul 21 08:15:27 +0000 2017","newTimestamp":"2017-07-21T08:15:27+0000", "unixOutput": 1500624927, "unixExtOutput":1500624927000, "unixInput": "Mon Apr 27 16:54:15 +0200 2020", "unixExtInput": "Mon Apr 27 16:54:15 +0200 2020"}` + + kazaamTransform, err := kazaam.NewKazaam(spec) + if err != nil { + t.Error("Init produced error.") + t.Log("Error: ", err.Error()) + t.FailNow() + } + kazaamOut, err := kazaamTransform.TransformJSONStringToString(jsonIn) + if err != nil { + t.Error("Transform produced error.") + t.Log("Error: ", err.Error()) + t.FailNow() + } + areEqual, _ := checkJSONStringsEqual(kazaamOut, jsonOut) + t.Log("Expected: ", jsonOut) + t.Log("Actual: ", kazaamOut) + + if !areEqual { + t.Error("Transformed data does not match expectation.") + t.Log("Expected: ", jsonOut) + t.Log("Actual: ", kazaamOut) + t.FailNow() + } +} + func TestShiftWithOverAndWildcard(t *testing.T) { spec := `[{"operation": "shift","spec": {"docs": "documents[*]"}}, {"operation": "shift", "spec": {"data": "norm.text"}, "over":"docs"}]` jsonIn := `{"documents":[{"norm":{"text":"String 1"}},{"norm":{"text":"String 2"}}]}` @@ -489,17 +530,17 @@ func TestKazaamOverArrayStrings(t *testing.T) { "over": "doc.guidObjects", "spec": {"raw": "$"} }]` - jsonIn := `{"doc":{"guidObjects":["foo",5,false]}}` - jsonOut := `{"doc":{"guidObjects":[{"raw":"foo"},{"raw":5},{"raw":false}]}}` - - kazaamTransform, _ := kazaam.NewKazaam(spec) - kazaamOut, _ := kazaamTransform.TransformJSONStringToString(jsonIn) - areEqual, _ := checkJSONStringsEqual(kazaamOut, jsonOut) - - if !areEqual { - t.Error("Transformed data does not match expectation.") - t.Log("Expected: ", jsonOut) - t.Log("Actual: ", kazaamOut) - t.FailNow() - } + jsonIn := `{"doc":{"guidObjects":["foo",5,false]}}` + jsonOut := `{"doc":{"guidObjects":[{"raw":"foo"},{"raw":5},{"raw":false}]}}` + + kazaamTransform, _ := kazaam.NewKazaam(spec) + kazaamOut, _ := kazaamTransform.TransformJSONStringToString(jsonIn) + areEqual, _ := checkJSONStringsEqual(kazaamOut, jsonOut) + + if !areEqual { + t.Error("Transformed data does not match expectation.") + t.Log("Expected: ", jsonOut) + t.Log("Actual: ", kazaamOut) + t.FailNow() + } } diff --git a/transform/timestamp.go b/transform/timestamp.go index eed3bfd..c536330 100644 --- a/transform/timestamp.go +++ b/transform/timestamp.go @@ -14,6 +14,7 @@ import ( var now = time.Now const unixFormat = "$unix" +const unixExtendedFormat = "$unixext" // Timestamp parses and formats timestamp strings using the golang syntax func Timestamp(spec *Config, data []byte) ([]byte, error) { @@ -59,8 +60,8 @@ func Timestamp(spec *Config, data []byte) ([]byte, error) { } // can only parse and format strings and arrays of strings, check the // value type and handle accordingly - switch dataForV[0] { - case '"': + switch { + case dataForV[0] == '"': formattedItem, err := parseAndFormatValue(inputFormat, outputFormat, string(dataForV[1:len(dataForV)-1])) if err != nil { return nil, err @@ -69,7 +70,16 @@ func Timestamp(spec *Config, data []byte) ([]byte, error) { if err != nil { return nil, err } - case '[': + case (dataForV[0] >= '0') && (dataForV[0] <= '9'): + formattedItem, err := parseAndFormatValue(inputFormat, outputFormat, string(dataForV)) + if err != nil { + return nil, err + } + data, err = setJSONRaw(data, []byte(formattedItem), k, spec.KeySeparator) + if err != nil { + return nil, err + } + case dataForV[0] == '[': var unformattedItems []string _, err = jsonparser.ArrayEach(dataForV, func(value []byte, dataType jsonparser.ValueType, offset int, err error) { unformattedItems = append(unformattedItems, string(value)) @@ -110,6 +120,12 @@ func parseAndFormatValue(inputFormat, outputFormat, unformattedItem string) (str return "", err } parsedItem = time.Unix(i, 0) + } else if inputFormat == unixExtendedFormat { + i, err := strconv.ParseInt(unformattedItem, 10, 64) + if err != nil { + return "", err + } + parsedItem = time.Unix(0, int64(i)*int64(time.Millisecond)) } else { parsedItem, err = time.Parse(inputFormat, unformattedItem) if err != nil { @@ -119,8 +135,11 @@ func parseAndFormatValue(inputFormat, outputFormat, unformattedItem string) (str if outputFormat == unixFormat { formattedItem = strconv.FormatInt(parsedItem.Unix(), 10) + } else if outputFormat == unixExtendedFormat { + formattedItem = strconv.FormatInt(parsedItem.UnixNano()/1000000, 10) } else { formattedItem = parsedItem.Format(outputFormat) + formattedItem = strings.Join([]string{"\"", formattedItem, "\""}, "") } - return strings.Join([]string{"\"", formattedItem, "\""}, ""), nil + return formattedItem, nil } diff --git a/transform/timestamp_test.go b/transform/timestamp_test.go index 5367f52..ec0ae06 100644 --- a/transform/timestamp_test.go +++ b/transform/timestamp_test.go @@ -170,6 +170,7 @@ func TestParseAndFormatValue(t *testing.T) { {time.RFC3339, "\"2017-07-21T08:15:27+01:00\""}, {time.StampNano, "\"Jul 21 08:15:27.000000000\""}, {"$unix", "\"1500621327\""}, + {"$unixext", "\"1500621327000\""}, } for _, testItem := range parseAndFormatTests { actual, _ := parseAndFormatValue(inputFormat, testItem.outputFormat, inputTimestamp) @@ -194,6 +195,7 @@ func TestParseAndFormatValueOutputUnix(t *testing.T) { {time.UnixDate, "Fri Jul 21 08:15:27 GMT 2017", "\"1500624927\""}, {time.RFC3339, "2017-07-21T08:15:27+01:00", "\"1500621327\""}, {"$unix", "1500621327", "\"1500621327\""}, + {"$unixext", "1500621327567", "\"1500621327\""}, } for _, testItem := range parseAndFormatTests { actual, _ := parseAndFormatValue(testItem.inputFormat, "$unix", testItem.inputTimestamp) @@ -204,3 +206,28 @@ func TestParseAndFormatValueOutputUnix(t *testing.T) { } } } + +func TestParseAndFormatValueOutputUnixExt(t *testing.T) { + parseAndFormatTests := []struct { + inputFormat string + inputTimestamp string + expectedOutput string + }{ + // test against a sampling of common formats + {"2006-01-02T15:04:05-0700", "2017-07-21T08:15:27+0100", "\"1500621327000\""}, + {"January _2, 2006", "July 21, 2017", "\"1500595200000\""}, + {time.ANSIC, "Fri Jul 21 08:15:27 2017", "\"1500624927000\""}, + {time.UnixDate, "Fri Jul 21 08:15:27 GMT 2017", "\"1500624927000\""}, + {time.RFC3339, "2017-07-21T08:15:27+01:00", "\"1500621327000\""}, + {"$unix", "1500621327", "\"1500621327000\""}, + {"$unixext", "1500621327234", "\"1500621327234\""}, + } + for _, testItem := range parseAndFormatTests { + actual, _ := parseAndFormatValue(testItem.inputFormat, "$unixext", testItem.inputTimestamp) + if actual != testItem.expectedOutput { + t.Error("Error data does not match expectation.", testItem.inputFormat) + t.Log("Expected: ", testItem.expectedOutput) + t.Log("Actual: ", actual) + } + } +}