diff --git a/Makefile b/Makefile index f6a73c0c3..0a7825dab 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,8 @@ loaders: tsbs_load \ tsbs_load_siridb \ tsbs_load_timescaledb \ tsbs_load_victoriametrics \ - tsbs_load_questdb + tsbs_load_questdb \ + tsbs_load_tdengine runners: tsbs_run_queries_akumuli \ tsbs_run_queries_cassandra \ @@ -38,7 +39,8 @@ runners: tsbs_run_queries_akumuli \ tsbs_run_queries_timescaledb \ tsbs_run_queries_timestream \ tsbs_run_queries_victoriametrics \ - tsbs_run_queries_questdb + tsbs_run_queries_questdb \ + tsbs_run_queries_tdengine test: $(GOTEST) -v ./... diff --git a/README.md b/README.md index bf78a77df..385dfcd53 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ Current databases supported: + TimescaleDB [(supplemental docs)](docs/timescaledb.md) + Timestream [(supplemental docs)](docs/timestream.md) + VictoriaMetrics [(supplemental docs)](docs/victoriametrics.md) ++ TDengine [(supplemental docs)](docs/tdengine.md) ## Overview @@ -81,6 +82,7 @@ cases are implemented for each database: |TimescaleDB|X|X| |Timestream|X|| |VictoriaMetrics|X²|| +|TDengine|X|X| ¹ Does not support the `groupby-orderby-limit` query ² Does not support the `groupby-orderby-limit`, `lastpoint`, `high-cpu-1`, `high-cpu-all` queries @@ -112,6 +114,10 @@ $ cd $GOPATH/src/github.com/timescale/tsbs $ make ``` +**`Notice`** + +The make process will failed if the TDengine client isn't installed. Please refer to the [supplementary documentation for TDengine](docs/tdengine.md). + ## How to use TSBS Using TSBS for benchmarking involves 3 phases: data and query @@ -135,7 +141,7 @@ Variables needed: 1. how much time should be between each reading per device, in seconds. E.g., `10s` 1. and which database(s) you want to generate for. E.g., `timescaledb` (choose from `cassandra`, `clickhouse`, `cratedb`, `influx`, `mongo`, `questdb`, `siridb`, - `timescaledb` or `victoriametrics`) + `timescaledb` `victoriametrics` or `tdengine`) Given the above steps you can now generate a dataset (or multiple datasets, if you chose to generate for multiple databases) that can diff --git a/cmd/tsbs_generate_data/main.go b/cmd/tsbs_generate_data/main.go index de9deb540..d604d80dd 100644 --- a/cmd/tsbs_generate_data/main.go +++ b/cmd/tsbs_generate_data/main.go @@ -7,6 +7,7 @@ // MongoDB BSON format // TimescaleDB pseudo-CSV format (the same as for ClickHouse) // VictoriaMetrics bulk load format (the same as for InfluxDB) +// TDengine pseudo-CSV format // Supported use cases: // devops: scale is the number of hosts to simulate, with log messages diff --git a/cmd/tsbs_generate_queries/databases/tdengine/common.go b/cmd/tsbs_generate_queries/databases/tdengine/common.go new file mode 100644 index 000000000..bc1c8e0ff --- /dev/null +++ b/cmd/tsbs_generate_queries/databases/tdengine/common.go @@ -0,0 +1,59 @@ +package tdengine + +import ( + "time" + + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/uses/devops" + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/uses/iot" + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/utils" + "github.com/timescale/tsbs/pkg/query" +) + +// BaseGenerator contains settings specific for Influx database. +type BaseGenerator struct { +} + +func (g *BaseGenerator) GenerateEmptyQuery() query.Query { + return query.NewTDengine() +} + +// fillInQuery fills the query struct with data. +func (g *BaseGenerator) fillInQuery(qi query.Query, humanLabel, humanDesc, table, sql string) { + q := qi.(*query.TDengine) + q.HumanLabel = []byte(humanLabel) + q.HumanDescription = []byte(humanDesc) + q.Hypertable = []byte(table) + q.SqlQuery = []byte(sql) +} + +// NewDevops creates a new devops use case query generator. +func (g *BaseGenerator) NewDevops(start, end time.Time, scale int) (utils.QueryGenerator, error) { + core, err := devops.NewCore(start, end, scale) + + if err != nil { + return nil, err + } + + devops := &Devops{ + BaseGenerator: g, + Core: core, + } + + return devops, nil +} + +// NewIoT creates a new iot use case query generator. +func (g *BaseGenerator) NewIoT(start, end time.Time, scale int) (utils.QueryGenerator, error) { + core, err := iot.NewCore(start, end, scale) + + if err != nil { + return nil, err + } + + iot := &IoT{ + BaseGenerator: g, + Core: core, + } + + return iot, nil +} diff --git a/cmd/tsbs_generate_queries/databases/tdengine/devops.go b/cmd/tsbs_generate_queries/databases/tdengine/devops.go new file mode 100644 index 000000000..814d8880c --- /dev/null +++ b/cmd/tsbs_generate_queries/databases/tdengine/devops.go @@ -0,0 +1,174 @@ +package tdengine + +import ( + "fmt" + "strings" + "time" + + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/uses/devops" + "github.com/timescale/tsbs/pkg/query" +) + +// TODO: Remove the need for this by continuing to bubble up errors +func panicIfErr(err error) { + if err != nil { + panic(err.Error()) + } +} + +// Devops produces TimescaleDB-specific queries for all the devops query types. +type Devops struct { + *BaseGenerator + *devops.Core +} + +// getHostWhereWithHostnames creates WHERE SQL statement for multiple hostnames. +// NOTE 'WHERE' itself is not included, just hostname filter clauses, ready to concatenate to 'WHERE' string +func (d *Devops) getHostWhereWithHostnames(hostnames []string) string { + var hostnameClauses []string + for _, s := range hostnames { + hostnameClauses = append(hostnameClauses, fmt.Sprintf("'%s'", s)) + } + return fmt.Sprintf("tbname IN (%s)", strings.Join(hostnameClauses, ",")) +} + +// getHostWhereString gets multiple random hostnames and creates a WHERE SQL statement for these hostnames. +func (d *Devops) getHostWhereString(nHosts int) string { + hostnames, err := d.GetRandomHosts(nHosts) + panicIfErr(err) + return d.getHostWhereWithHostnames(hostnames) +} + +func (d *Devops) getSelectClausesAggMetrics(agg string, metrics []string) []string { + selectClauses := make([]string, len(metrics)) + for i, m := range metrics { + selectClauses[i] = fmt.Sprintf("%s(%s)", agg, m) + } + + return selectClauses +} + +func (d *Devops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRange time.Duration) { + interval := d.Interval.MustRandWindow(timeRange) + metrics, err := devops.GetCPUMetricsSlice(numMetrics) + panicIfErr(err) + selectClauses := d.getSelectClausesAggMetrics("max", metrics) + if len(selectClauses) < 1 { + panic(fmt.Sprintf("invalid number of select clauses: got %d", len(selectClauses))) + } + + //SELECT _wstart as ts,max(usage_user) FROM cpu WHERE tbname IN ('host_249') AND ts >= 1451618560000 AND ts < 1451622160000 INTERVAL(1m) ; + //SELECT _wstart as ts,max(usage_user) FROM host_249 WHERE ts >= 1451618560000 AND ts < 1451622160000 INTERVAL(1m) ; + sql := "" + if nHosts == 1 { + hostnames, err := d.GetRandomHosts(nHosts) + panicIfErr(err) + sql = fmt.Sprintf(`SELECT _wstart as ts,%s FROM %s WHERE ts >= %d AND ts < %d INTERVAL(1m)`, + strings.Join(selectClauses, ", "), + hostnames[0], + interval.StartUnixMillis(), + interval.EndUnixMillis()) + + } else { + sql = fmt.Sprintf(`SELECT _wstart as ts,%s FROM cpu WHERE %s AND ts >= %d AND ts < %d INTERVAL(1m)`, + strings.Join(selectClauses, ", "), + d.getHostWhereString(nHosts), + interval.StartUnixMillis(), + interval.EndUnixMillis()) + } + + humanLabel := fmt.Sprintf("TDengine %d cpu metric(s), random %4d hosts, random %s by 1m", numMetrics, nHosts, timeRange) + humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) + d.fillInQuery(qi, humanLabel, humanDesc, devops.TableName, sql) +} + +func (d *Devops) GroupByOrderByLimit(qi query.Query) { + interval := d.Interval.MustRandWindow(time.Hour) + //SELECT _wstart as ts,max(usage_user) FROM cpu WHERE ts < 1451618228646 INTERVAL(1m) LIMIT 5; + sql := fmt.Sprintf(`SELECT _wstart as ts,max(usage_user) FROM cpu WHERE ts < %d INTERVAL(1m) LIMIT 5`, + interval.EndUnixMillis()) + + humanLabel := "TDengine max cpu over last 5 min-intervals (random end)" + humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.EndString()) + d.fillInQuery(qi, humanLabel, humanDesc, devops.TableName, sql) +} + +// GroupByTimeAndPrimaryTag selects the AVG of numMetrics metrics under 'cpu' per device per hour for a day, +func (d *Devops) GroupByTimeAndPrimaryTag(qi query.Query, numMetrics int) { + metrics, err := devops.GetCPUMetricsSlice(numMetrics) + panicIfErr(err) + interval := d.Interval.MustRandWindow(devops.DoubleGroupByDuration) + + selectClauses := d.getSelectClausesAggMetrics("avg", metrics) + //SELECT _wstart as ts,tbname, avg(usage_user) from cpu where ts >= 1451733760646 and ts < 1451776960646 partition by tbname interval(1h) order by tbname,ts asc; + //SELECT _wstart as ts,tbname, avg(usage_user), avg(usage_system), avg(usage_idle), avg(usage_nice), avg(usage_iowait), avg(usage_irq), avg(usage_softirq), avg(usage_steal), avg(usage_guest), avg(usage_guest_nice) from cpu where ts >= 1451733760646 and ts < 1451776960646 partition by tbname interval(1h) order by tbname,ts asc; + sql := fmt.Sprintf("SELECT _wstart as ts,tbname,%s from cpu where ts >= %d and ts < %d partition by tbname interval(1h) order by tbname,ts asc", strings.Join(selectClauses, ", "), interval.StartUnixMillis(), interval.EndUnixMillis()) + + humanLabel := devops.GetDoubleGroupByLabel("TDengine", numMetrics) + humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) + d.fillInQuery(qi, humanLabel, humanDesc, devops.TableName, sql) +} + +func (d *Devops) MaxAllCPU(qi query.Query, nHosts int, duration time.Duration) { + interval := d.Interval.MustRandWindow(duration) + + metrics := devops.GetAllCPUMetrics() + selectClauses := d.getSelectClausesAggMetrics("max", metrics) + //SELECT _wstart as ts,max(usage_user), max(usage_system), max(usage_idle), max(usage_nice), max(usage_iowait), max(usage_irq), max(usage_softirq), max(usage_steal), max(usage_guest), max(usage_guest_nice) FROM host_249 WHERE ts >= 1451648911646 AND ts < 1451677711646 interval(1h); + //SELECT_wstart as ts, max(usage_user), max(usage_system), max(usage_idle), max(usage_nice), max(usage_iowait), max(usage_irq), max(usage_softirq), max(usage_steal), max(usage_guest), max(usage_guest_nice) FROM cpu WHERE tbname IN ('host_249','host_403','host_435','host_39','host_139','host_75','host_315','host_121') AND ts >= 1451648911646 AND ts < 1451677711646 interval(1h) + + sql := "" + if nHosts == 1 { + hostnames, err := d.GetRandomHosts(nHosts) + panicIfErr(err) + sql = fmt.Sprintf(`SELECT _wstart as ts,%s FROM %s WHERE ts >= %d AND ts < %d interval(1h)`, + strings.Join(selectClauses, ", "), + hostnames[0], + interval.StartUnixMillis(), + interval.EndUnixMillis()) + } else { + sql = fmt.Sprintf(`SELECT _wstart as ts,%s FROM cpu WHERE %s AND ts >= %d AND ts < %d interval(1h)`, + strings.Join(selectClauses, ", "), + d.getHostWhereString(nHosts), + interval.StartUnixMillis(), + interval.EndUnixMillis()) + } + humanLabel := devops.GetMaxAllLabel("TDengine", nHosts) + humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) + d.fillInQuery(qi, humanLabel, humanDesc, devops.TableName, sql) +} + +func (d *Devops) LastPointPerHost(qi query.Query) { + //SELECT last_row(*),tbname from cpu group by tbname; + sql := "SELECT last_row(*),tbname from cpu group by tbname" + humanLabel := "TDengine last row per host" + humanDesc := humanLabel + d.fillInQuery(qi, humanLabel, humanDesc, devops.TableName, sql) +} + +func (d *Devops) HighCPUForHosts(qi query.Query, nHosts int) { + interval := d.Interval.MustRandWindow(devops.HighCPUDuration) + var hostWhereClause string + if nHosts == 0 { + hostWhereClause = "" + } else { + hostWhereClause = fmt.Sprintf("AND %s", d.getHostWhereString(nHosts)) + } + //SELECT ts,usage_user,usage_system,usage_idle,usage_nice,usage_iowait,usage_irq,usage_softirq,usage_steal,usage_guest,usage_guest_nice FROM cpu WHERE usage_user > 90.0 and ts >= 1451777731138 AND ts < 1451820931138 AND tbname IN ('host_35') + //modify:SELECT * FROM host_35 WHERE usage_user > 90.0 and ts >= 1451777731138 AND ts < 1451820931138 + + sql := "" + if nHosts == 1 { + hostnames, err := d.GetRandomHosts(nHosts) + panicIfErr(err) + sql = fmt.Sprintf(`SELECT * FROM %s WHERE usage_user > 90.0 and ts >= %d AND ts < %d `, + hostnames[0], interval.StartUnixMillis(), interval.EndUnixMillis()) + } else { + sql = fmt.Sprintf(`SELECT ts,usage_user,usage_system,usage_idle,usage_nice,usage_iowait,usage_irq,usage_softirq,usage_steal,usage_guest,usage_guest_nice FROM cpu WHERE usage_user > 90.0 and ts >= %d AND ts < %d %s`, + interval.StartUnixMillis(), interval.EndUnixMillis(), hostWhereClause) + } + humanLabel, err := devops.GetHighCPULabel("TDengine", nHosts) + panicIfErr(err) + humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString()) + d.fillInQuery(qi, humanLabel, humanDesc, devops.TableName, sql) +} diff --git a/cmd/tsbs_generate_queries/databases/tdengine/iot.go b/cmd/tsbs_generate_queries/databases/tdengine/iot.go new file mode 100644 index 000000000..bf8267b77 --- /dev/null +++ b/cmd/tsbs_generate_queries/databases/tdengine/iot.go @@ -0,0 +1,193 @@ +package tdengine + +import ( + "fmt" + "strings" + "time" + + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/uses/iot" + "github.com/timescale/tsbs/pkg/query" +) + +// IoT produces TDengine-specific queries for all the iot query types. +type IoT struct { + *iot.Core + *BaseGenerator +} + +//last-loc +//single-last-loc +//low-fuel +//avg-vs-projected-fuel-consumption +//avg-daily-driving-duration +//daily-activity + +func (i *IoT) getTrucksWhereWithNames(names []string) string { + var nameClauses []string + + for _, s := range names { + nameClauses = append(nameClauses, fmt.Sprintf("'%s'", s)) + } + return fmt.Sprintf("name IN (%s)", strings.Join(nameClauses, ",")) +} + +// getHostWhereString gets multiple random hostnames and creates a WHERE SQL statement for these hostnames. +func (i *IoT) getTruckWhereString(nTrucks int) string { + names, err := i.GetRandomTrucks(nTrucks) + panicIfErr(err) + return i.getTrucksWhereWithNames(names) +} + +// LastLocByTruck finds the truck location for nTrucks. +func (i *IoT) LastLocByTruck(qi query.Query, nTrucks int) { + sql := fmt.Sprintf(`SELECT last_row(ts),last_row(latitude),last_row(longitude) FROM readings WHERE %s GROUP BY name`, + i.getTruckWhereString(nTrucks)) + + humanLabel := "TDengine last location by specific truck" + humanDesc := fmt.Sprintf("%s: random %4d trucks", humanLabel, nTrucks) + + i.fillInQuery(qi, humanLabel, humanDesc, iot.ReadingsTableName, sql) +} + +// LastLocPerTruck finds all the truck locations along with truck and driver names. +func (i *IoT) LastLocPerTruck(qi query.Query) { + //SELECT last_row(ts),name,driver,latitude,longitude FROM readings WHERE fleet='South' and name IS NOT NULL partition BY name,driver; + sql := fmt.Sprintf(`SELECT last_row(ts),name,driver,latitude,longitude FROM readings WHERE fleet='%s' and name IS NOT NULL partition BY name,driver`, + i.GetRandomFleet()) + + humanLabel := "TDengine last location per truck" + humanDesc := humanLabel + + i.fillInQuery(qi, humanLabel, humanDesc, iot.ReadingsTableName, sql) +} + +// TrucksWithLowFuel finds all trucks with low fuel (less than 10%). +func (i *IoT) TrucksWithLowFuel(qi query.Query) { + //SELECT last_row(ts),name,driver,fuel_state FROM diagnostics WHERE fuel_state <= 0.1 AND fleet = 'South' and name IS NOT NULL GROUP BY name,driver ; + sql := fmt.Sprintf(`SELECT last_row(ts),name,driver,fuel_state FROM diagnostics WHERE fuel_state <= 0.1 AND fleet = '%s' and name IS NOT NULL GROUP BY name,driver`, + i.GetRandomFleet()) + + humanLabel := "TDengine trucks with low fuel" + humanDesc := fmt.Sprintf("%s: under 10 percent", humanLabel) + + i.fillInQuery(qi, humanLabel, humanDesc, iot.DiagnosticsTableName, sql) +} + +// TrucksWithHighLoad finds all trucks that have load over 90%. +func (i *IoT) TrucksWithHighLoad(qi query.Query) { + //SELECT ts,name,driver,current_load,load_capacity FROM (SELECT last_row(ts) as ts,name,driver, current_load,load_capacity FROM diagnostics WHERE fleet = 'South' partition by name,driver) WHERE current_load>= (0.9 * load_capacity); + //pre sql := fmt.Sprintf("SELECT ts,name,driver,current_load,load_capacity FROM (SELECT last_row(ts) as ts,name,driver, current_load,load_capacity FROM diagnostics WHERE fleet = '%s' partition by name,driver) WHERE current_load>= (0.9 * load_capacity)", i.GetRandomFleet()) + sql := fmt.Sprintf("SELECT ts,name,driver,current_load,load_capacity FROM (SELECT last_row(ts) as ts,name,driver, current_load,load_capacity FROM diagnostics WHERE fleet = '%s' partition by name,driver) WHERE current_load>= (0.9 * load_capacity)", i.GetRandomFleet()) + + humanLabel := "TDengine trucks with high load" + humanDesc := fmt.Sprintf("%s: over 90 percent", humanLabel) + + i.fillInQuery(qi, humanLabel, humanDesc, iot.DiagnosticsTableName, sql) +} + +// StationaryTrucks finds all trucks that have low average velocity in a time window. +func (i *IoT) StationaryTrucks(qi query.Query) { + interval := i.Interval.MustRandWindow(iot.StationaryDuration) + //select name,driver from (SELECT name,driver,fleet ,avg(velocity) as mean_velocity FROM readings WHERE ts > '2016-01-01T15:07:21Z' AND ts <= '2016-01-01T16:17:21Z' partition BY name,driver,fleet interval(10m) LIMIT 1) WHERE fleet = 'West' AND mean_velocity < 1 ; + sql := fmt.Sprintf("select name,driver from (SELECT name,driver,avg(velocity) as mean_velocity FROM readings WHERE ts > %d AND ts <= %d and fleet = '%s' partition BY name,driver interval(10m) LIMIT 1) WHERE mean_velocity < 1;", interval.StartUnixMillis(), interval.EndUnixMillis(), i.GetRandomFleet()) + //sql := fmt.Sprintf("SELECT name,driver FROM readings WHERE ts > %d AND ts <= %d and fleet = '%s' partition BY name,driver,fleet interval(10m) having (avg(velocity) < 1) LIMIT 1;", interval.StartUnixMillis(), interval.EndUnixMillis(), i.GetRandomFleet()) + humanLabel := "TDengine stationary trucks" + humanDesc := fmt.Sprintf("%s: with low avg velocity in last 10 minutes", humanLabel) + + i.fillInQuery(qi, humanLabel, humanDesc, iot.ReadingsTableName, sql) +} + +// TrucksWithLongDrivingSessions finds all trucks that have not stopped at least 20 mins in the last 4 hours. +func (i *IoT) TrucksWithLongDrivingSessions(qi query.Query) { + interval := i.Interval.MustRandWindow(iot.LongDrivingSessionDuration) + //SELECT name,driver FROM(SELECT _wstart as ts,name,driver,avg(velocity) as mean_velocity FROM readings WHERE fleet ="West" AND ts > '2016-01-03T13:46:34Z' AND ts <= '2016-01-03T17:46:34Z' partition BY name,driver interval(10m)) WHERE mean_velocity > 1 GROUP BY name,driver having count(*) > 22 + //pre sql := fmt.Sprintf("SELECT name,driver FROM(SELECT _wstart as ts,name,driver,avg(velocity) as mean_velocity FROM readings WHERE fleet =\"%s\" AND ts > %d AND ts <= %d partition BY name,driver interval(10m)) WHERE mean_velocity > 1 GROUP BY name,driver having count(*) > %d", i.GetRandomFleet(), interval.StartUnixMillis(), interval.EndUnixMillis(), tenMinutePeriods(5, iot.LongDrivingSessionDuration)) + sql := fmt.Sprintf("SELECT name,driver FROM(SELECT name,driver,avg(velocity) as mean_velocity FROM readings WHERE fleet =\"%s\" AND ts > %d AND ts <= %d partition BY name,driver interval(10m)) WHERE mean_velocity > 1 GROUP BY name,driver having count(*) > %d", i.GetRandomFleet(), interval.StartUnixMillis(), interval.EndUnixMillis(), tenMinutePeriods(5, iot.LongDrivingSessionDuration)) + humanLabel := "TDengine trucks with longer driving sessions" + humanDesc := fmt.Sprintf("%s: stopped less than 20 mins in 4 hour period", humanLabel) + + i.fillInQuery(qi, humanLabel, humanDesc, iot.ReadingsTableName, sql) +} + +// TrucksWithLongDailySessions finds all trucks that have driven more than 10 hours in the last 24 hours. +func (i *IoT) TrucksWithLongDailySessions(qi query.Query) { + //SELECT name,driver FROM(SELECT name,driver,avg(velocity) as mean_velocity FROM readings WHERE fleet ='West' AND ts > '2016-01-01T12:31:37Z' AND ts <= '2016-01-05T12:31:37Z' partition BY name,driver interval(10m) ) WHERE mean_velocity > 1 GROUP BY name,driver having count(*) > 60 + + interval := i.Interval.MustRandWindow(iot.DailyDrivingDuration) + sql := fmt.Sprintf("SELECT name,driver FROM(SELECT name,driver,avg(velocity) as mean_velocity FROM readings WHERE fleet ='%s' AND ts > %d AND ts <= %d partition BY name,driver interval(10m) ) WHERE mean_velocity > 1 GROUP BY name,driver having count(*) > %d", i.GetRandomFleet(), interval.StartUnixMillis(), interval.EndUnixMillis(), tenMinutePeriods(35, iot.DailyDrivingDuration)) + + humanLabel := "TDengine trucks with longer daily sessions" + humanDesc := fmt.Sprintf("%s: drove more than 10 hours in the last 24 hours", humanLabel) + + i.fillInQuery(qi, humanLabel, humanDesc, iot.ReadingsTableName, sql) +} + +// AvgVsProjectedFuelConsumption calculates average and projected fuel consumption per fleet. +func (i *IoT) AvgVsProjectedFuelConsumption(qi query.Query) { + //select avg(fuel_consumption) as avg_fuel_consumption,avg(nominal_fuel_consumption) as nominal_fuel_consumption from readings where velocity > 1 group by fleet + sql := fmt.Sprintf("select avg(fuel_consumption) as avg_fuel_consumption,avg(nominal_fuel_consumption) as nominal_fuel_consumption from readings where velocity > 1 group by fleet") + humanLabel := "TDengine average vs projected fuel consumption per fleet" + humanDesc := humanLabel + + i.fillInQuery(qi, humanLabel, humanDesc, iot.ReadingsTableName, sql) +} + +// AvgDailyDrivingDuration finds the average driving duration per driver. +func (i *IoT) AvgDailyDrivingDuration(qi query.Query) { + //select fleet,name,driver,avg(hours_driven) as avg_daily_hours from( select _wstart as ts,fleet,name,driver,count(mv)/6 as hours_driven from ( select _wstart as ts,fleet,tbname,name,driver,avg(velocity) as mv from readings where ts > '2016-01-01T00:00:00Z' and ts < '2016-01-05T00:00:01Z' partition by fleet,tbname,name,driver interval(10m) ) where mv >1 partition by fleet,name,driver interval(1d) )partition by fleet,name,driver ; + sql := fmt.Sprintf("select fleet,name,driver,avg(hours_driven) as avg_daily_hours from( select _wstart as ts,fleet,name,driver,count(mv)/6 as hours_driven from ( select _wstart as ts,fleet,tbname,name,driver,avg(velocity) as mv from readings where ts > %d and ts < %d partition by fleet,tbname,name,driver interval(10m) ) where mv >1 partition by fleet,name,driver interval(1d) )partition by fleet,name,driver ;", i.Interval.StartUnixMillis(), i.Interval.EndUnixMillis()) + + humanLabel := "TDengine average driver driving duration per day" + humanDesc := humanLabel + + i.fillInQuery(qi, humanLabel, humanDesc, iot.ReadingsTableName, sql) +} + +// AvgDailyDrivingSession finds the average driving session without stopping per driver per day. +func (i *IoT) AvgDailyDrivingSession(qi query.Query) { + // select _wstart as ts,name,avg(ela) from (select ts,name,ela from (SELECT ts,name, diff(difka) as dif, diff(cast(ts as bigint)) as ela FROM (SELECT ts,name,difka FROM (SELECT ts,name,diff(mv) AS difka FROM (SELECT _wstart as ts,name,cast(cast(floor(avg(velocity)/5) as bool) as int) AS mv FROM readings WHERE name is not null AND ts > 1451637149138 AND ts < 1451637749138 partition by name interval(10m))partition BY name ) WHERE difka!=0 order by ts) partition BY name order by ts ) WHERE dif = -2 partition BY name order by ts ) partition BY name interval(1d); + interval := i.Interval + sql := fmt.Sprintf(" select _wstart as ts,name,avg(ela) from (select ts,name,ela from (SELECT ts,name, diff(difka) as dif, diff(cast(ts as bigint)) as ela FROM (SELECT ts,name,difka FROM (SELECT ts,name,diff(mv) AS difka FROM (SELECT _wstart as ts,name,cast(cast(floor(avg(velocity)/5) as bool) as int) AS mv FROM readings WHERE name is not null AND ts > %d AND ts < %d partition by name interval(10m))partition BY name ) WHERE difka!=0 order by ts) partition BY name order by ts ) WHERE dif = -2 partition BY name order by ts ) partition BY name interval(1d);", interval.StartUnixMillis(), interval.EndUnixMillis()) + humanLabel := "TDengine average driver driving session without stopping per day" + humanDesc := humanLabel + + i.fillInQuery(qi, humanLabel, humanDesc, iot.ReadingsTableName, sql) +} + +// AvgLoad finds the average load per truck model per fleet. +func (i *IoT) AvgLoad(qi query.Query) { + //select fleet,model,load_capacity,avg(ml/load_capacity) from(SELECT fleet, model,tbname,load_capacity ,avg(current_load) AS ml FROM diagnostics where name is not null partition BY fleet, model,tbname,load_capacity) partition BY fleet, model,load_capacity; + sql := fmt.Sprintf("select fleet,model,load_capacity,avg(ml/load_capacity) from(SELECT fleet, model,tbname,load_capacity ,avg(current_load) AS ml FROM diagnostics where name is not null partition BY fleet, model,tbname,load_capacity) partition BY fleet, model,load_capacity") + + humanLabel := "TDengine average load per truck model per fleet" + humanDesc := humanLabel + + i.fillInQuery(qi, humanLabel, humanDesc, iot.ReadingsTableName, sql) +} + +// DailyTruckActivity returns the number of hours trucks has been active (not out-of-commission) per day per fleet per model. +func (i *IoT) DailyTruckActivity(qi query.Query) { + //SELECT _wstart as ts,model,fleet,count(ms1)/144 FROM (SELECT _wstart as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= 1451606400000 AND ts < 1451952001000 partition by model, fleet, tbname interval(10m)) WHERE ms1<1 partition by model, fleet interval(1d) + sql := fmt.Sprintf("SELECT _wstart as ts,model,fleet,count(ms1)/144 FROM (SELECT _wstart as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= %d AND ts < %d partition by model, fleet, tbname interval(10m)) WHERE ms1<1 partition by model, fleet interval(1d)", i.Interval.StartUnixMillis(), i.Interval.EndUnixMillis()) + humanLabel := "TDengine daily truck activity per fleet per model" + humanDesc := humanLabel + + i.fillInQuery(qi, humanLabel, humanDesc, iot.ReadingsTableName, sql) +} + +// TruckBreakdownFrequency calculates the amount of times a truck model broke down in the last period. +func (i *IoT) TruckBreakdownFrequency(qi query.Query) { + // SELECT model,count(state_changed) FROM (SELECT _rowts,model,diff(broken_down) AS state_changed FROM (SELECT model,tb,cast(cast(floor(2*(nzs)) as bool) as int) AS broken_down FROM (SELECT _wstart as ts,model,tbname as tb, sum(cast(cast(status as bool) as int))/count(cast(cast(status as bool) as int)) AS nzs FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2023-01-05T00:00:01Z' partition BY tbname,model interval(10m)) order by ts ) partition BY tb,model ) WHERE state_changed = 1 partition BY model ; + sql := fmt.Sprintf("SELECT model,count(state_changed) FROM (SELECT _rowts,model,diff(broken_down) AS state_changed FROM (SELECT ts,model,tb,cast(cast(floor(2*(nzs)) as bool) as int) AS broken_down FROM (SELECT _wstart as ts,model,tbname as tb, sum(cast(cast(status as bool) as int))/count(cast(cast(status as bool) as int)) AS nzs FROM diagnostics WHERE ts >= %d AND ts < %d partition BY tbname,model interval(10m)) order by ts) partition BY tb,model) WHERE state_changed = 1 partition BY model", i.Interval.StartUnixMillis(), i.Interval.EndUnixMillis()) + + humanLabel := "TDengine truck breakdown frequency per model" + humanDesc := humanLabel + + i.fillInQuery(qi, humanLabel, humanDesc, iot.ReadingsTableName, sql) +} + +func tenMinutePeriods(minutesPerHour float64, duration time.Duration) int { + durationMinutes := duration.Minutes() + leftover := minutesPerHour * duration.Hours() + return int((durationMinutes - leftover) / 10) +} diff --git a/cmd/tsbs_load/parse_config.go b/cmd/tsbs_load/parse_config.go index c45449684..f085986d3 100644 --- a/cmd/tsbs_load/parse_config.go +++ b/cmd/tsbs_load/parse_config.go @@ -3,6 +3,7 @@ package main import ( "errors" "fmt" + "github.com/blagojts/viper" "github.com/timescale/tsbs/load" "github.com/timescale/tsbs/pkg/data/source" @@ -47,7 +48,10 @@ func parseConfig(target targets.ImplementedTarget, v *viper.Viper) (targets.Benc if err != nil { return nil, nil, err } - + configurableBenchmark, ok := benchmark.(targets.ConfigurableBenchmark) + if ok { + configurableBenchmark.SetConfig(loaderConfigInternal.BatchSize, loaderConfig.Workers) + } return benchmark, load.GetBenchmarkRunner(*loaderConfigInternal), nil } diff --git a/cmd/tsbs_load_cratedb/creator.go b/cmd/tsbs_load_cratedb/creator.go index 914d86b92..f9fd65f46 100644 --- a/cmd/tsbs_load_cratedb/creator.go +++ b/cmd/tsbs_load_cratedb/creator.go @@ -58,18 +58,21 @@ func (d *dbCreator) Init() { // First N lines are the header, describing the data structure. // The first line contains the tags table name followed by a comma-separated // list of tags: -// tags,hostname,region,datacenter,rack,os,arch,team,service,service_version +// +// tags,hostname,region,datacenter,rack,os,arch,team,service,service_version // // The second through N-1 line containing table name (ex.: 'disk') followed by // list of column names, comma-separated: -// disk,total,free,used,used_percent,inodes_total,inodes_free,inodes_used +// +// disk,total,free,used,used_percent,inodes_total,inodes_free,inodes_used // // The last line being blank to separate the header from the data. // // Header example: -// tags,hostname,region,datacenter,rack,os,arch,team,service,service_version,service_environment -// disk,total,free,used,used_percent,inodes_total,inodes_free,inodes_used -// nginx,accepts,active,handled,reading,requests,waiting,writing +// +// tags,hostname,region,datacenter,rack,os,arch,team,service,service_version,service_environment +// disk,total,free,used,used_percent,inodes_total,inodes_free,inodes_used +// nginx,accepts,active,handled,reading,requests,waiting,writing func (d *dbCreator) readDataHeader(header *common.GeneratedDataHeaders) ([]*tableDef, error) { var tableDefs []*tableDef for tableName, fieldCols := range header.FieldKeys { diff --git a/cmd/tsbs_load_tdengine/main.go b/cmd/tsbs_load_tdengine/main.go new file mode 100644 index 000000000..2a9e79d37 --- /dev/null +++ b/cmd/tsbs_load_tdengine/main.go @@ -0,0 +1,89 @@ +package main + +import ( + "fmt" + "log" + "os" + "runtime/pprof" + + "github.com/blagojts/viper" + "github.com/spf13/pflag" + "github.com/timescale/tsbs/internal/utils" + "github.com/timescale/tsbs/load" + "github.com/timescale/tsbs/pkg/data/source" + "github.com/timescale/tsbs/pkg/targets/tdengine" +) + +func initProgramOptions() (*tdengine.LoadingOptions, load.BenchmarkRunner, *load.BenchmarkRunnerConfig) { + target := tdengine.NewTarget() + loaderConf := load.BenchmarkRunnerConfig{} + loaderConf.AddToFlagSet(pflag.CommandLine) + target.TargetSpecificFlags("", pflag.CommandLine) + pflag.Parse() + err := utils.SetupConfigFile() + + if err != nil { + panic(fmt.Errorf("fatal error config file: %s", err)) + } + + if err := viper.Unmarshal(&loaderConf); err != nil { + panic(fmt.Errorf("unable to decode config: %s", err)) + } + opts := tdengine.LoadingOptions{} + viper.SetTypeByDefaultValue(true) + opts.User = viper.GetString("user") + opts.Pass = viper.GetString("pass") + opts.Host = viper.GetString("host") + opts.Port = viper.GetInt("port") + vgroups := viper.GetInt("vgroups") + if vgroups > 0 { + opts.VGroups = vgroups + } + buffer := viper.GetInt("buffer") + if buffer > 0 { + opts.Buffer = buffer + } + pages := viper.GetInt("pages") + if pages > 0 { + opts.Pages = pages + } + sttTrigger := viper.GetInt("stt_trigger") + if sttTrigger > 0 { + opts.SttTrigger = sttTrigger + } + if viper.IsSet("wal_fsync_period") { + walFsyncPeriod := viper.GetInt("wal_fsync_period") + opts.WalFsyncPeriod = &walFsyncPeriod + } + if viper.IsSet("wal_level") { + walLevel := viper.GetInt("wal_level") + opts.WalLevel = &walLevel + } + if viper.IsSet("db_parameters") { + opts.DBParameters = viper.GetString("db_parameters") + } + loaderConf.HashWorkers = true + loaderConf.NoFlowControl = true + loaderConf.ChannelCapacity = 50 + loader := load.GetBenchmarkRunner(loaderConf) + return &opts, loader, &loaderConf +} +func main() { + f, err := os.Create("./cpu.prof") + if err != nil { + log.Fatal("could not create CPU profile: ", err) + } + if err := pprof.StartCPUProfile(f); err != nil { + log.Fatal("could not start CPU profile: ", err) + } + defer pprof.StopCPUProfile() + opts, loader, loaderConf := initProgramOptions() + benchmark, err := tdengine.NewBenchmark(loaderConf.DBName, opts, &source.DataSourceConfig{ + Type: source.FileDataSourceType, + File: &source.FileDataSourceConfig{Location: loaderConf.FileName}, + }) + if err != nil { + panic(err) + } + loader.RunBenchmark(benchmark) +} diff --git a/cmd/tsbs_load_tdenginestmt2/main.go b/cmd/tsbs_load_tdenginestmt2/main.go new file mode 100644 index 000000000..6db7ce176 --- /dev/null +++ b/cmd/tsbs_load_tdenginestmt2/main.go @@ -0,0 +1,96 @@ +package main + +import ( + "fmt" + "log" + _ "net/http/pprof" + "os" + "runtime/pprof" + + "github.com/blagojts/viper" + "github.com/spf13/pflag" + "github.com/timescale/tsbs/internal/utils" + "github.com/timescale/tsbs/load" + "github.com/timescale/tsbs/pkg/data/source" + "github.com/timescale/tsbs/pkg/targets" + "github.com/timescale/tsbs/pkg/targets/tdengine" + "github.com/timescale/tsbs/pkg/targets/tdenginestmt2" +) + +func initProgramOptions() (*tdengine.LoadingOptions, load.BenchmarkRunner, *load.BenchmarkRunnerConfig) { + target := tdengine.NewTarget() + loaderConf := load.BenchmarkRunnerConfig{} + loaderConf.AddToFlagSet(pflag.CommandLine) + target.TargetSpecificFlags("", pflag.CommandLine) + pflag.Parse() + err := utils.SetupConfigFile() + + if err != nil { + panic(fmt.Errorf("fatal error config file: %s", err)) + } + + if err := viper.Unmarshal(&loaderConf); err != nil { + panic(fmt.Errorf("unable to decode config: %s", err)) + } + opts := tdengine.LoadingOptions{} + viper.SetTypeByDefaultValue(true) + opts.User = viper.GetString("user") + opts.Pass = viper.GetString("pass") + opts.Host = viper.GetString("host") + opts.Port = viper.GetInt("port") + vgroups := viper.GetInt("vgroups") + if vgroups > 0 { + opts.VGroups = vgroups + } + buffer := viper.GetInt("buffer") + if buffer > 0 { + opts.Buffer = buffer + } + pages := viper.GetInt("pages") + if pages > 0 { + opts.Pages = pages + } + sttTrigger := viper.GetInt("stt_trigger") + if sttTrigger > 0 { + opts.SttTrigger = sttTrigger + } + if viper.IsSet("wal_fsync_period") { + walFsyncPeriod := viper.GetInt("wal_fsync_period") + opts.WalFsyncPeriod = &walFsyncPeriod + } + if viper.IsSet("wal_level") { + walLevel := viper.GetInt("wal_level") + opts.WalLevel = &walLevel + } + if viper.IsSet("db_parameters") { + opts.DBParameters = viper.GetString("db_parameters") + } + loaderConf.HashWorkers = true + loaderConf.NoFlowControl = true + loaderConf.ChannelCapacity = 50 + loader := load.GetBenchmarkRunner(loaderConf) + return &opts, loader, &loaderConf +} +func main() { + //go http.ListenAndServe(":3666", nil) + + f, err := os.Create("./cpu.prof") + if err != nil { + log.Fatal("could not create CPU profile: ", err) + } + if err := pprof.StartCPUProfile(f); err != nil { + log.Fatal("could not start CPU profile: ", err) + } + defer pprof.StopCPUProfile() + opts, loader, loaderConf := initProgramOptions() + + benchmark, err := tdenginestmt2.NewBenchmark(loaderConf.DBName, opts, &source.DataSourceConfig{ + Type: source.FileDataSourceType, + File: &source.FileDataSourceConfig{Location: loaderConf.FileName}, + }) + benchmark.(targets.ConfigurableBenchmark).SetConfig(loaderConf.BatchSize, loaderConf.Workers) + if err != nil { + panic(err) + } + loader.RunBenchmark(benchmark) +} diff --git a/cmd/tsbs_run_queries_tdengine/main.go b/cmd/tsbs_run_queries_tdengine/main.go new file mode 100644 index 000000000..39d6d0a46 --- /dev/null +++ b/cmd/tsbs_run_queries_tdengine/main.go @@ -0,0 +1,122 @@ +package main + +import ( + "database/sql/driver" + "fmt" + "strings" + "time" + + "github.com/blagojts/viper" + "github.com/spf13/pflag" + "github.com/timescale/tsbs/internal/utils" + "github.com/timescale/tsbs/pkg/query" + "github.com/timescale/tsbs/pkg/targets/tdengine/async" + "github.com/timescale/tsbs/pkg/targets/tdengine/commonpool" +) + +var ( + user string + pass string + host string + port int + runner *query.BenchmarkRunner +) + +func init() { + var config query.BenchmarkRunnerConfig + config.AddToFlagSet(pflag.CommandLine) + + pflag.String("user", "root", "User to connect to TDengine") + pflag.String("pass", "taosdata", "Password for the user connecting to TDengine") + pflag.String("host", "", "TDengine host") + pflag.Int("port", 6030, "TDengine Port") + pflag.Parse() + err := utils.SetupConfigFile() + + if err != nil { + panic(fmt.Errorf("fatal error config file: %s", err)) + } + if err := viper.Unmarshal(&config); err != nil { + panic(fmt.Errorf("unable to decode config: %s", err)) + } + user = viper.GetString("user") + pass = viper.GetString("pass") + host = viper.GetString("host") + port = viper.GetInt("port") + runner = query.NewBenchmarkRunner(config) +} +func main() { + runner.Run(&query.TDenginePool, newProcessor) +} + +type queryExecutorOptions struct { + debug bool + printResponse bool +} + +type processor struct { + db *commonpool.Conn + opts *queryExecutorOptions +} + +func (p *processor) Init(workerNum int) { + async.Init() + db, err := commonpool.GetConnection(user, pass, host, port) + if err != nil { + panic(err) + } + dbName := runner.DatabaseName() + err = async.GlobalAsync.TaosExecWithoutResult(db.TaosConnection, "use "+dbName) + if err != nil { + panic(err) + } + p.db = db + p.opts = &queryExecutorOptions{ + debug: runner.DebugLevel() > 0, + printResponse: runner.DoPrintResponses(), + } +} + +func (p *processor) ProcessQuery(q query.Query, _ bool) ([]*query.Stat, error) { + tq := q.(*query.TDengine) + + start := time.Now() + qry := string(tq.SqlQuery) + if p.opts.debug { + fmt.Println(qry) + } + querys := strings.Split(qry, ";") + if len(querys) > 1 { + var preQuerys []string + for i := 0; i < len(querys); i++ { + if len(querys[i]) > 0 { + preQuerys = append(preQuerys, querys[i]) + } + } + if len(preQuerys) > 1 { + for i := 0; i < len(preQuerys)-1; i++ { + err := async.GlobalAsync.TaosExecWithoutResult(p.db.TaosConnection, preQuerys[i]) + if err != nil { + return nil, err + } + } + } + qry = querys[len(preQuerys)-1] + } + data, err := async.GlobalAsync.TaosExec(p.db.TaosConnection, qry, func(ts int64, precision int) driver.Value { + return ts + }) + if err != nil { + return nil, err + } + if p.opts.printResponse { + fmt.Printf("%#v\n", data) + } + took := float64(time.Since(start).Nanoseconds()) / 1e6 + stat := query.GetStat() + stat.Init(q.HumanLabelName(), took) + + return []*query.Stat{stat}, err +} + +func newProcessor() query.Processor { return &processor{} } diff --git a/docs/tdengine.md b/docs/tdengine.md new file mode 100644 index 000000000..42fbefccb --- /dev/null +++ b/docs/tdengine.md @@ -0,0 +1,143 @@ +# Installation +**`Notice`** + +TDengine uses the Go connector to perform data writing and querying in the +TSBS test. The installation process relies on the C client library. +Therefore, before compiling TSBS, the Taos client must be installed to +provide the necessary library files. For instructions on how to install the +Taos client and server, refer to the [TDengine documentation](https://docs.tdengine.com/get-started/deploy-from-package/). + +# TSBS Supplemental Guide: TDengine + +TDengine is an open-source time-series database with high performance, scalability, and SQL support. +This supplemental guide explains how +the data generated for TSBS is stored, additional flags available when +using the data importer (`tsbs_load_tdenginestmt2` or `tsbs_load_tdengine`), and additional flags +available for the query runner (`tsbs_run_queries_tdengine`). + +**This guide should be read *after* the main README.** + +--- +## Generate Data + +TDengine supports two different write modes for data ingestion: SQL mode and `stmt2` mode. Each mode corresponds to a specific data format generated by `tsbs_generate_data`. It is important to choose the appropriate mode and data format based on your use case. + +### Data Format + +The data format generated by `tsbs_generate_data` depends on the `--format` parameter: + +#### 1. SQL mode: **`--format="TDengine"`**: +Data is serialized in a "pseudo-CSV" format. This format is based on the SQL way of inserting. Each reading consists of a row, where the first item represents the operation type, indicated by 1, 2, or 3. +- **1**: Insert data. The format is: + - `1,common table name,field count,insert data` +- **2**: Create a super table. The format is: + - `2,super table name,common table name,create table SQL` +- **3**: Create a regular table. The format is: + - `3,super table name,common table name,create table SQL` +* Example for the `cpu-only` use case: +```text +2,cpu,host_0,create table cpu (ts timestamp,usage_user bigint,usage_system bigint,usage_idle bigint,usage_nice bigint,usage_iowait bigint,usage_irq bigint,usage_softirq bigint,usage_steal bigint,usage_guest bigint,usage_guest_nice bigint) tags (hostname binary(30),region binary(30),datacenter binary(30),rack binary(30),os binary(30),arch binary(30),team binary(30),service binary(30),service_version binary(30),service_environment binary(30)) +3,cpu,host_0,create table host_0 using cpu (hostname,region,datacenter,rack,os,arch,team,service,service_version,service_environment) tags ('host_0','eu-central-1','eu-central-1a','6','Ubuntu15.10','x86','SF','19','1','test') +1,host_0,10,(1451606400000,58,2,24,61,22,63,6,44,80,38) +``` + +#### 2. `stmt2` mode: **`--format="TDengineStmt2"`**: + +Data is serialized in a binary format optimized for the stmt2 binding method. +This format is more efficient for large-scale data ingestion and is recommended for high-performance scenarios. + +* Example: +The binary format cannot be directly visualized as text but is designed for efficient parsing and insertion using tsbs_load_tdenginestmt2. + +## Load Data +The `tsbs_load_tdengine` and `tsbs_load_tdenginestmt2` tools are used for importing data into TDengine, but they differ in their data insertion methods: + +- **`tsbs_load_tdengine`**: Uses SQL string concatenation to insert data. This method is straightforward and easy to understand but may not achieve the highest performance. +- **`tsbs_load_tdenginestmt2`**: Uses the `stmt2` binding method for data insertion. This approach is more efficient and faster, especially for large-scale data ingestion. + + +## `tsbs_load_tdengine` and `tsbs_load_tdenginestmt2` Additional Flags + +### Connection-Related Flags + +#### `-user` (type: `string`, default: `root`) + +Specifies the username for connecting to TDengine. + +#### `-pass` (type: `string`, default: `taosdata`) + +Specifies the password for the user connecting to TDengine. + +#### `-host` (type: `string`) + +Specifies the hostname of the TDengine server. + +#### `-port` (type: `int`, default: `6030`) + +Specifies the port of the TDengine server. + +### TDengine Database-Related Flags + +TDengine provides several parameters for creating a database. See the [TDengine documentation][conn-str] for more details. + +#### `-buffer` (type: `int`, default: `256`) + +Specifies the size (in MB) of the write buffer for each vnode. Enter a value between 3 and 16384. + +#### `-pages` (type: `int`, default: `256`) + +Specifies the number of pages in the metadata storage engine cache on each vnode. Enter a value greater than or equal to 64. + +#### `-stt_trigger` (type: `int`, default: `1`) + +Specifies the number of file merges triggered by flushed files. The value ranges from 1 to 16. + +#### `-wal_level` (type: `int`, default: `1`) + +Specifies whether fsync is enabled for the Write-Ahead Log (WAL). The default value is 1. + +- **1**: WAL is enabled, but fsync is disabled. +- **2**: Both WAL and fsync are enabled. + +#### `-wal_fsync_period` (type: `int`, default: `3000`) + +Specifies the interval (in milliseconds) at which data is written from the WAL to disk. This parameter is effective only when the WAL parameter is set to 2. Enter a value between 0 and 180000. + +#### `-vgroups` (type: `int`, default: `2`) + +Specifies the initial number of vgroups when a database is created. The value ranges from 0 to 4096. + +#### `-db-name` (type: `string`, default: `benchmark`) + +Specifies the name of the database. + +### Miscellaneous Flags + +#### `-hash-workers` (type: `boolean`, default: `false`) + +Determines whether to consistently hash data across multiple insert workers based on the value of the primary (first) tag. For datasets with a large number of devices, this option helps improve data locality on disk, which can lead to better query performance. For datasets with a small number of devices, this option is typically unnecessary. + +[conn-str]: https://docs.tdengine.com/tdengine-reference/sql-manual/manage-databases/ + +--- + +## `tsbs_run_queries_tdengine` Additional Flags + +### Connection-Related Flags + +#### `-user` (type: `string`, default: `root`) + +Specifies the username for connecting to TDengine. + +#### `-pass` (type: `string`, default: `taosdata`) + +Specifies the password for the user connecting to TDengine. + +#### `-host` (type: `string`) + +Specifies the hostname of the TDengine server. + +#### `-port` (type: `int`, default: `6030`) + +Specifies the port of the TDengine server. + diff --git a/go.mod b/go.mod index 1106116ed..8d4f0a0cf 100644 --- a/go.mod +++ b/go.mod @@ -23,8 +23,11 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/common v0.13.0 github.com/shirou/gopsutil v3.21.3+incompatible + github.com/silenceper/pool v1.0.0 + github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 + github.com/taosdata/driver-go/v3 v3.6.0 github.com/timescale/promscale v0.0.0-20201006153045-6a66a36f5c84 github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/transceptor-technology/go-qpack v0.0.0-20190116123619-49a14b216a45 diff --git a/go.sum b/go.sum index 6b3ac40ef..c7d7ca04b 100644 --- a/go.sum +++ b/go.sum @@ -416,6 +416,8 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= @@ -432,6 +434,8 @@ github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -572,6 +576,8 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= @@ -676,9 +682,12 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.2.2 h1:dxe5oCinTXiTIcfgmZecdCzPmAJKd46KsCWc35r0TV4= github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= @@ -828,8 +837,6 @@ github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs= github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= -github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= -github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v3.21.3+incompatible h1:uenXGGa8ESCQq+dbgtl916dmg6PSAz2cXov0uORQ9v8= github.com/shirou/gopsutil v3.21.3+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= @@ -843,6 +850,8 @@ github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/shurcooL/vfsgen v0.0.0-20200627165143-92b8a710ab6c/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= +github.com/silenceper/pool v1.0.0 h1:JTCaA+U6hJAA0P8nCx+JfsRCHMwLTfatsm5QXelffmU= +github.com/silenceper/pool v1.0.0/go.mod h1:3DN13bqAbq86Lmzf6iUXWEPIWFPOSYVfaoceFvilKKI= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= @@ -857,6 +866,7 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k github.com/sonatard/noctx v0.0.1/go.mod h1:9D2D/EoULe8Yy2joDHJj7bv3sZoq9AaSb8B4lqBjiZI= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/sourcegraph/go-diff v0.6.0/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= @@ -890,8 +900,12 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/taosdata/driver-go/v3 v3.6.0 h1:4dRXMl01DhIS5xBXUvtkkB+MjL8g64zN674xKd+ojTE= +github.com/taosdata/driver-go/v3 v3.6.0/go.mod h1:H2vo/At+rOPY1aMzUV9P49SVX7NlXb3LAbKw+MCLrmU= github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM= github.com/testcontainers/testcontainers-go v0.5.1/go.mod h1:Oc/G02bjZiX0p3lzyh6b1GCELP0e4/6Cg3ciU/LnFvU= github.com/tetafro/godot v0.4.8/go.mod h1:/7NLHhv08H1+8DNj0MElpAACw1ajsCuf3TKNQxA5S+0= @@ -902,6 +916,7 @@ github.com/timescale/promscale v0.0.0-20201006153045-6a66a36f5c84/go.mod h1:rkhy github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= +github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA= github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= diff --git a/internal/inputs/generator_data.go b/internal/inputs/generator_data.go index ee639659c..c42dcb98d 100644 --- a/internal/inputs/generator_data.go +++ b/internal/inputs/generator_data.go @@ -136,10 +136,17 @@ func (g *DataGenerator) getSerializer(sim common.Simulator, target targets.Imple case constants.FormatTimescaleDB: g.writeHeader(sim.Headers()) } - return target.Serializer(), nil + serializer := target.Serializer() + if cs, ok := serializer.(serialize.ConfigurableSerializer); ok { + err := cs.Config(g.config, g.bufOut) + if err != nil { + return nil, err + } + } + return serializer, nil } -//TODO should be implemented in targets package +// TODO should be implemented in targets package func (g *DataGenerator) writeHeader(headers *common.GeneratedDataHeaders) { g.bufOut.WriteString("tags") diff --git a/pkg/data/serialize/point_serializer.go b/pkg/data/serialize/point_serializer.go index e632de797..5f5c5ae24 100644 --- a/pkg/data/serialize/point_serializer.go +++ b/pkg/data/serialize/point_serializer.go @@ -1,11 +1,18 @@ package serialize import ( - "github.com/timescale/tsbs/pkg/data" "io" + + "github.com/timescale/tsbs/pkg/data" + "github.com/timescale/tsbs/pkg/data/usecases/common" ) // PointSerializer serializes a Point for writing type PointSerializer interface { Serialize(p *data.Point, w io.Writer) error } + +type ConfigurableSerializer interface { + PointSerializer + Config(*common.DataGeneratorConfig, io.Writer) error +} diff --git a/pkg/query/factories/init_factories.go b/pkg/query/factories/init_factories.go index ff3faf47d..4deae56f3 100644 --- a/pkg/query/factories/init_factories.go +++ b/pkg/query/factories/init_factories.go @@ -9,6 +9,7 @@ import ( "github.com/timescale/tsbs/cmd/tsbs_generate_queries/databases/mongo" "github.com/timescale/tsbs/cmd/tsbs_generate_queries/databases/questdb" "github.com/timescale/tsbs/cmd/tsbs_generate_queries/databases/siridb" + "github.com/timescale/tsbs/cmd/tsbs_generate_queries/databases/tdengine" "github.com/timescale/tsbs/cmd/tsbs_generate_queries/databases/timescaledb" "github.com/timescale/tsbs/cmd/tsbs_generate_queries/databases/timestream" "github.com/timescale/tsbs/cmd/tsbs_generate_queries/databases/victoriametrics" @@ -39,5 +40,6 @@ func InitQueryFactories(config *config.QueryGeneratorConfig) map[string]interfac DBName: config.DbName, } factories[constants.FormatQuestDB] = &questdb.BaseGenerator{} + factories[constants.FormatTDengine] = &tdengine.BaseGenerator{} return factories } diff --git a/pkg/query/stats.go b/pkg/query/stats.go index 007f41a45..d71914df9 100644 --- a/pkg/query/stats.go +++ b/pkg/query/stats.go @@ -65,6 +65,9 @@ type statGroup struct { latencyHDRHistogram *hdrhistogram.Histogram sum float64 count int64 + intervalHistogram *hdrhistogram.Histogram + intervalSum float64 + intervalCount int64 } // newStatGroup returns a new StatGroup with an initial size @@ -77,9 +80,14 @@ func newStatGroup(size uint64) *statGroup { // - 10 millisecond (or better) from 10 millisecond up to 10 seconds, // - 1 second (or better) from 10 second up to 3600 seconds, lH := hdrhistogram.New(1, 3600000000, 4) + iH := hdrhistogram.New(1, 3600000000, 4) return &statGroup{ - count: 0, latencyHDRHistogram: lH, + sum: 0, + count: 0, + intervalHistogram: iH, + intervalSum: 0, + intervalCount: 0, } } @@ -88,11 +96,14 @@ func (s *statGroup) push(n float64) { s.latencyHDRHistogram.RecordValue(int64(n * hdrScaleFactor)) s.sum += n s.count++ + s.intervalHistogram.RecordValue(int64(n * hdrScaleFactor)) + s.intervalSum += n + s.intervalCount++ } // string makes a simple description of a statGroup. func (s *statGroup) string() string { - return fmt.Sprintf("min: %8.2fms, med: %8.2fms, mean: %8.2fms, max: %7.2fms, stddev: %8.2fms, sum: %5.1fsec, count: %d", + vl := fmt.Sprintf("min: %8.2fms, med: %8.2fms, mean: %8.2fms, max: %7.2fms, stddev: %8.2fms, sum: %5.1fsec, count: %d", s.Min(), s.Median(), s.Mean(), @@ -100,6 +111,21 @@ func (s *statGroup) string() string { s.StdDev(), s.sum/hdrScaleFactor, s.count) + vi := "" + if s.intervalCount != 0 { + vi = fmt.Sprintf("\nmin: %8.2fms, med: %8.2fms, mean: %8.2fms, max: %7.2fms, stddev: %8.2fms, sum: %5.1fsec, count: %d", + s.MinInterval(), + s.MedianInterval(), + s.MeanInterval(), + s.MaxInterval(), + s.StdDevInterval(), + s.intervalSum/hdrScaleFactor, + s.intervalCount) + } + s.intervalSum = 0 + s.intervalCount = 0 + s.intervalHistogram.Reset() + return vl + vi } func (s *statGroup) write(w io.Writer) error { @@ -132,6 +158,26 @@ func (s *statGroup) StdDev() float64 { return float64(s.latencyHDRHistogram.StdDev()) / hdrScaleFactor } +func (s *statGroup) MedianInterval() float64 { + return float64(s.intervalHistogram.ValueAtQuantile(50.0)) / hdrScaleFactor +} + +func (s *statGroup) MeanInterval() float64 { + return float64(s.intervalHistogram.Mean()) / hdrScaleFactor +} + +func (s *statGroup) MaxInterval() float64 { + return float64(s.intervalHistogram.Max()) / hdrScaleFactor +} + +func (s *statGroup) MinInterval() float64 { + return float64(s.intervalHistogram.Min()) / hdrScaleFactor +} + +func (s *statGroup) StdDevInterval() float64 { + return float64(s.intervalHistogram.StdDev()) / hdrScaleFactor +} + // writeStatGroupMap writes a map of StatGroups in an ordered fashion by // key that they are stored by func writeStatGroupMap(w io.Writer, statGroups map[string]*statGroup) error { diff --git a/pkg/query/tdengine.go b/pkg/query/tdengine.go new file mode 100644 index 000000000..a8f842c53 --- /dev/null +++ b/pkg/query/tdengine.go @@ -0,0 +1,59 @@ +package query + +import ( + "fmt" + "sync" +) + +type TDengine struct { + id uint64 + HumanLabel []byte + HumanDescription []byte + Hypertable []byte + SqlQuery []byte +} + +var TDenginePool = sync.Pool{ + New: func() interface{} { + return &TDengine{ + HumanLabel: make([]byte, 0, 1024), + HumanDescription: make([]byte, 0, 1024), + Hypertable: make([]byte, 0, 1024), + SqlQuery: make([]byte, 0, 1024), + } + }, +} + +func NewTDengine() *TDengine { + return TDenginePool.Get().(*TDengine) +} + +func (q *TDengine) Release() { + q.HumanLabel = q.HumanLabel[:0] + q.HumanDescription = q.HumanDescription[:0] + q.id = 0 + + q.Hypertable = q.Hypertable[:0] + q.SqlQuery = q.SqlQuery[:0] + TDenginePool.Put(q) +} + +func (q *TDengine) HumanLabelName() []byte { + return q.HumanLabel +} + +func (q *TDengine) HumanDescriptionName() []byte { + return q.HumanDescription +} + +func (q *TDengine) GetID() uint64 { + return q.id +} + +func (q *TDengine) SetID(n uint64) { + q.id = n +} + +func (q *TDengine) String() string { + return fmt.Sprintf("HumanLabel: %s, HumanDescription: %s, Hypertable: %s, Query: %s", q.HumanLabel, q.HumanDescription, q.Hypertable, q.SqlQuery) +} diff --git a/pkg/targets/constants/constants.go b/pkg/targets/constants/constants.go index 516093d09..e628a1000 100644 --- a/pkg/targets/constants/constants.go +++ b/pkg/targets/constants/constants.go @@ -14,6 +14,8 @@ const ( FormatVictoriaMetrics = "victoriametrics" FormatTimestream = "timestream" FormatQuestDB = "questdb" + FormatTDengine = "TDengine" + FormatTDengineStmt2 = "TDengineStmt2" ) func SupportedFormats() []string { @@ -30,5 +32,7 @@ func SupportedFormats() []string { FormatVictoriaMetrics, FormatTimestream, FormatQuestDB, + FormatTDengine, + FormatTDengineStmt2, } } diff --git a/pkg/targets/initializers/target_initializers.go b/pkg/targets/initializers/target_initializers.go index a7a31677e..f42b24600 100644 --- a/pkg/targets/initializers/target_initializers.go +++ b/pkg/targets/initializers/target_initializers.go @@ -2,6 +2,8 @@ package initializers import ( "fmt" + "strings" + "github.com/timescale/tsbs/pkg/targets" "github.com/timescale/tsbs/pkg/targets/akumuli" "github.com/timescale/tsbs/pkg/targets/cassandra" @@ -13,10 +15,11 @@ import ( "github.com/timescale/tsbs/pkg/targets/prometheus" "github.com/timescale/tsbs/pkg/targets/questdb" "github.com/timescale/tsbs/pkg/targets/siridb" + "github.com/timescale/tsbs/pkg/targets/tdengine" + "github.com/timescale/tsbs/pkg/targets/tdenginestmt2" "github.com/timescale/tsbs/pkg/targets/timescaledb" "github.com/timescale/tsbs/pkg/targets/timestream" "github.com/timescale/tsbs/pkg/targets/victoriametrics" - "strings" ) func GetTarget(format string) targets.ImplementedTarget { @@ -45,6 +48,10 @@ func GetTarget(format string) targets.ImplementedTarget { return timestream.NewTarget() case constants.FormatQuestDB: return questdb.NewTarget() + case constants.FormatTDengine: + return tdengine.NewTarget() + case constants.FormatTDengineStmt2: + return tdenginestmt2.NewTarget() } supportedFormatsStr := strings.Join(constants.SupportedFormats(), ",") diff --git a/pkg/targets/targets.go b/pkg/targets/targets.go index 064bc5ab1..a727772cd 100644 --- a/pkg/targets/targets.go +++ b/pkg/targets/targets.go @@ -71,6 +71,11 @@ type Benchmark interface { GetDBCreator() DBCreator } +type ConfigurableBenchmark interface { + Benchmark + SetConfig(batchSize uint, workers uint) +} + type DataSource interface { NextItem() data.LoadedPoint Headers() *common.GeneratedDataHeaders diff --git a/pkg/targets/tdengine/async/handlerpool.go b/pkg/targets/tdengine/async/handlerpool.go new file mode 100644 index 000000000..1437a367e --- /dev/null +++ b/pkg/targets/tdengine/async/handlerpool.go @@ -0,0 +1,120 @@ +package async + +import ( + "container/list" + "sync" + "unsafe" + + "github.com/taosdata/driver-go/v3/wrapper/cgo" +) + +type Result struct { + Res unsafe.Pointer + N int +} +type Stmt2ExecResult struct { + Res unsafe.Pointer + Affected int + Code int +} +type Caller struct { + QueryResult chan *Result + FetchResult chan *Result + ExecResult chan *Stmt2ExecResult +} + +func NewCaller() *Caller { + return &Caller{ + QueryResult: make(chan *Result, 1), + FetchResult: make(chan *Result, 1), + ExecResult: make(chan *Stmt2ExecResult, 1), + } +} + +func (c *Caller) QueryCall(res unsafe.Pointer, code int) { + c.QueryResult <- &Result{ + Res: res, + N: code, + } +} + +func (c *Caller) FetchCall(res unsafe.Pointer, numOfRows int) { + c.FetchResult <- &Result{ + Res: res, + N: numOfRows, + } +} + +func (c *Caller) ExecCall(res unsafe.Pointer, affected int, code int) { + c.ExecResult <- &Stmt2ExecResult{ + Res: res, + Affected: affected, + Code: code, + } +} + +type poolReq struct { + idleHandler *Handler +} + +type HandlerPool struct { + mu sync.RWMutex + count int + handlers chan *Handler + reqList *list.List +} + +type Handler struct { + Handler cgo.Handle + Caller *Caller +} + +func NewHandlerPool(count int) *HandlerPool { + c := &HandlerPool{ + count: count, + handlers: make(chan *Handler, count), + reqList: list.New(), + } + for i := 0; i < count; i++ { + caller := NewCaller() + c.handlers <- &Handler{ + Handler: cgo.NewHandle(caller), + Caller: caller, + } + } + return c +} + +func (c *HandlerPool) Get() *Handler { + for { + select { + case wrapConn := <-c.handlers: + return wrapConn + default: + c.mu.Lock() + req := make(chan poolReq, 1) + c.reqList.PushBack(req) + c.mu.Unlock() + ret := <-req + return ret.idleHandler + } + } +} + +func (c *HandlerPool) Put(handler *Handler) { + c.mu.Lock() + e := c.reqList.Front() + if e != nil { + req := e.Value.(chan poolReq) + c.reqList.Remove(e) + req <- poolReq{ + idleHandler: handler, + } + c.mu.Unlock() + return + } else { + c.handlers <- handler + c.mu.Unlock() + return + } +} diff --git a/pkg/targets/tdengine/async/row.go b/pkg/targets/tdengine/async/row.go new file mode 100644 index 000000000..bd3e39013 --- /dev/null +++ b/pkg/targets/tdengine/async/row.go @@ -0,0 +1,133 @@ +package async + +import ( + "database/sql/driver" + "errors" + "sync" + "unsafe" + + "github.com/taosdata/driver-go/v3/common/parser" + tErrors "github.com/taosdata/driver-go/v3/errors" + "github.com/taosdata/driver-go/v3/wrapper" + "github.com/timescale/tsbs/pkg/targets/tdengine/thread" +) + +var FetchRowError = errors.New("fetch row error") +var GlobalAsync *Async + +type Async struct { + HandlerPool *HandlerPool +} + +func NewAsync(handlerPool *HandlerPool) *Async { + return &Async{HandlerPool: handlerPool} +} + +func (a *Async) TaosExec(taosConnect unsafe.Pointer, sql string, timeFormat parser.FormatTimeFunc) (*ExecResult, error) { + handler := a.HandlerPool.Get() + defer a.HandlerPool.Put(handler) + result, err := a.TaosQuery(taosConnect, sql, handler) + defer func() { + if result != nil && result.Res != nil { + thread.Lock() + wrapper.TaosFreeResult(result.Res) + thread.Unlock() + } + }() + if err != nil { + return nil, err + } + res := result.Res + code := wrapper.TaosError(res) + if code != int(tErrors.SUCCESS) { + errStr := wrapper.TaosErrorStr(res) + return nil, tErrors.NewError(code, errStr) + } + isUpdate := wrapper.TaosIsUpdateQuery(res) + execResult := &ExecResult{} + if isUpdate { + affectRows := wrapper.TaosAffectedRows(res) + execResult.AffectedRows = affectRows + return execResult, nil + } + fieldsCount := wrapper.TaosNumFields(res) + execResult.FieldCount = fieldsCount + var rowsHeader *wrapper.RowsHeader + rowsHeader, err = wrapper.ReadColumn(res, fieldsCount) + if err != nil { + return nil, err + } + execResult.Header = rowsHeader + precision := wrapper.TaosResultPrecision(res) + for { + result, err = a.TaosFetchRawBlockA(res, handler) + if err != nil { + return nil, err + } + if result.N == 0 { + return execResult, nil + } else if result.N < 0 { + errStr := wrapper.TaosErrorStr(result.Res) + return nil, tErrors.NewError(result.N, errStr) + } else { + res = result.Res + block := wrapper.TaosGetRawBlock(res) + values := parser.ReadBlockWithTimeFormat(block, result.N, rowsHeader.ColTypes, precision, timeFormat) + execResult.Data = append(execResult.Data, values...) + } + } +} + +func (a *Async) TaosQuery(taosConnect unsafe.Pointer, sql string, handler *Handler) (*Result, error) { + thread.Lock() + wrapper.TaosQueryA(taosConnect, sql, handler.Handler) + thread.Unlock() + r := <-handler.Caller.QueryResult + return r, nil +} + +func (a *Async) TaosFetchRawBlockA(res unsafe.Pointer, handler *Handler) (*Result, error) { + thread.Lock() + wrapper.TaosFetchRawBlockA(res, handler.Handler) + thread.Unlock() + r := <-handler.Caller.FetchResult + return r, nil +} + +type ExecResult struct { + AffectedRows int + FieldCount int + Header *wrapper.RowsHeader + Data [][]driver.Value +} + +func (a *Async) TaosExecWithoutResult(taosConnect unsafe.Pointer, sql string) error { + handler := a.HandlerPool.Get() + defer a.HandlerPool.Put(handler) + result, err := a.TaosQuery(taosConnect, sql, handler) + defer func() { + if result != nil && result.Res != nil { + thread.Lock() + wrapper.TaosFreeResult(result.Res) + thread.Unlock() + } + }() + if err != nil { + return err + } + res := result.Res + code := wrapper.TaosError(res) + if code != int(tErrors.SUCCESS) { + errStr := wrapper.TaosErrorStr(res) + return tErrors.NewError(code, errStr) + } + return nil +} + +var once sync.Once + +func Init() { + once.Do(func() { + GlobalAsync = NewAsync(NewHandlerPool(10000)) + }) +} diff --git a/pkg/targets/tdengine/benchmark.go b/pkg/targets/tdengine/benchmark.go new file mode 100644 index 000000000..e25ecd12e --- /dev/null +++ b/pkg/targets/tdengine/benchmark.go @@ -0,0 +1,66 @@ +package tdengine + +import ( + "bytes" + "math" + + "github.com/timescale/tsbs/pkg/data/source" + "github.com/timescale/tsbs/pkg/targets" +) + +func NewBenchmark(dbName string, opts *LoadingOptions, dataSourceConfig *source.DataSourceConfig) (targets.Benchmark, error) { + var ds targets.DataSource + if dataSourceConfig.Type == source.FileDataSourceType { + ds = newFileDataSource(dataSourceConfig.File.Location) + } else { + panic("not implement") + } + + return &benchmark{ + opts: opts, + dataSource: ds, + dbName: dbName, + factory: NewBatchFactory(), + }, nil +} + +type benchmark struct { + opts *LoadingOptions + dataSource targets.DataSource + dbName string + batchSize uint + factory *BatchFactory +} + +func (b *benchmark) GetDataSource() targets.DataSource { + return b.dataSource +} + +func (b *benchmark) GetBatchFactory() targets.BatchFactory { + return b.factory +} + +func (b *benchmark) GetPointIndexer(maxPartitions uint) targets.PointIndexer { + if maxPartitions > 1 { + interval := uint32(math.MaxUint32 / maxPartitions) + hashEndGroups := make([]uint32, maxPartitions) + for i := 0; i < int(maxPartitions); i++ { + if i == int(maxPartitions)-1 { + hashEndGroups[i] = math.MaxUint32 + } else { + hashEndGroups[i] = interval*uint32(i+1) - 1 + } + } + prefix := []byte("1." + b.dbName + ".") + return &indexer{buffer: &bytes.Buffer{}, prefix: prefix, hashEndGroups: hashEndGroups, partitions: int(maxPartitions), tmp: map[string]uint{}} + } + return &targets.ConstantIndexer{} +} + +func (b *benchmark) GetProcessor() targets.Processor { + return newProcessor(b.opts, b.dbName) +} + +func (b *benchmark) GetDBCreator() targets.DBCreator { + return &DbCreator{Opts: b.opts} +} diff --git a/pkg/targets/tdengine/commonpool/pool.go b/pkg/targets/tdengine/commonpool/pool.go new file mode 100644 index 000000000..9c87ad270 --- /dev/null +++ b/pkg/targets/tdengine/commonpool/pool.go @@ -0,0 +1,133 @@ +package commonpool + +import ( + "sync" + "unsafe" + + "github.com/silenceper/pool" + "github.com/taosdata/driver-go/v3/wrapper" + "github.com/timescale/tsbs/pkg/targets/tdengine/thread" +) + +type ConnectorPool struct { + host string + user string + password string + port int + pool pool.Pool +} + +func NewConnectorPool(user, password, host string, port int) (*ConnectorPool, error) { + a := &ConnectorPool{user: user, password: password, host: host, port: port} + poolConfig := &pool.Config{ + InitialCap: 1, + MaxCap: 10000, + MaxIdle: 10000, + Factory: a.factory, + Close: a.close, + IdleTimeout: -1, + } + p, err := pool.NewChannelPool(poolConfig) + if err != nil { + return nil, err + } + a.pool = p + return a, nil +} + +func (a *ConnectorPool) factory() (interface{}, error) { + thread.Lock() + defer thread.Unlock() + return wrapper.TaosConnect(a.host, a.user, a.password, "", a.port) +} + +func (a *ConnectorPool) close(v interface{}) error { + if v != nil { + thread.Lock() + defer thread.Unlock() + wrapper.TaosClose(v.(unsafe.Pointer)) + } + return nil +} + +func (a *ConnectorPool) Get() (unsafe.Pointer, error) { + v, err := a.pool.Get() + if err != nil { + return nil, err + } + return v.(unsafe.Pointer), nil +} + +func (a *ConnectorPool) Put(c unsafe.Pointer) error { + wrapper.TaosResetCurrentDB(c) + return a.pool.Put(c) +} + +func (a *ConnectorPool) Close(c unsafe.Pointer) error { + return a.pool.Close(c) +} + +func (a *ConnectorPool) Release() { + a.pool.Release() +} + +func (a *ConnectorPool) verifyPassword(password string) bool { + return password == a.password +} + +var connectionMap = sync.Map{} + +type Conn struct { + TaosConnection unsafe.Pointer + pool *ConnectorPool +} + +func (c *Conn) Put() error { + return c.pool.Put(c.TaosConnection) +} + +func GetConnection(user, password, host string, port int) (*Conn, error) { + p, exist := connectionMap.Load(user) + if exist { + connectionPool := p.(*ConnectorPool) + if !connectionPool.verifyPassword(password) { + newPool, err := NewConnectorPool(user, password, host, port) + if err != nil { + return nil, err + } + connectionPool.Release() + connectionMap.Store(user, newPool) + c, err := newPool.Get() + if err != nil { + return nil, err + } + return &Conn{ + TaosConnection: c, + pool: newPool, + }, nil + } else { + c, err := connectionPool.Get() + if err != nil { + return nil, err + } + return &Conn{ + TaosConnection: c, + pool: connectionPool, + }, nil + } + } else { + newPool, err := NewConnectorPool(user, password, host, port) + if err != nil { + return nil, err + } + connectionMap.Store(user, newPool) + c, err := newPool.Get() + if err != nil { + return nil, err + } + return &Conn{ + TaosConnection: c, + pool: newPool, + }, nil + } +} diff --git a/pkg/targets/tdengine/creator.go b/pkg/targets/tdengine/creator.go new file mode 100644 index 000000000..7439680ab --- /dev/null +++ b/pkg/targets/tdengine/creator.go @@ -0,0 +1,68 @@ +package tdengine + +import ( + "fmt" + "log" + "strconv" + + "github.com/timescale/tsbs/pkg/targets/tdengine/async" + "github.com/timescale/tsbs/pkg/targets/tdengine/commonpool" +) + +var fatal = log.Fatalf + +type DbCreator struct { + Opts *LoadingOptions + Db *commonpool.Conn +} + +func (d *DbCreator) Init() { + async.Init() + db, err := commonpool.GetConnection(d.Opts.User, d.Opts.Pass, d.Opts.Host, d.Opts.Port) + if err != nil { + panic("TDengine can not get connection") + } + d.Db = db +} + +func (d *DbCreator) DBExists(dbName string) bool { + err := async.GlobalAsync.TaosExecWithoutResult(d.Db.TaosConnection, "use "+dbName) + return err == nil +} + +func (d *DbCreator) CreateDB(dbName string) error { + sql := fmt.Sprintf("create database %s precision 'ms'", dbName) + if d.Opts.Buffer > 0 { + sql += " buffer " + strconv.Itoa(d.Opts.Buffer) + } + if d.Opts.Pages > 0 { + sql += " pages " + strconv.Itoa(d.Opts.Pages) + } + if d.Opts.VGroups > 0 { + sql += " vgroups " + strconv.Itoa(d.Opts.VGroups) + } + if d.Opts.SttTrigger > 0 { + sql += " stt_trigger " + strconv.Itoa(d.Opts.SttTrigger) + } + if d.Opts.WalFsyncPeriod != nil { + sql += " wal_fsync_period " + strconv.Itoa(*d.Opts.WalFsyncPeriod) + } + if d.Opts.WalLevel != nil { + sql += " wal_level " + strconv.Itoa(*d.Opts.WalLevel) + } + if d.Opts.DBParameters != "" { + sql += " " + d.Opts.DBParameters + } + return async.GlobalAsync.TaosExecWithoutResult(d.Db.TaosConnection, sql) +} + +func (d *DbCreator) RemoveOldDB(dbName string) error { + sql := fmt.Sprintf("drop database %s", dbName) + return async.GlobalAsync.TaosExecWithoutResult(d.Db.TaosConnection, sql) +} + +func (d *DbCreator) Close() { + if d.Db != nil { + d.Db.Put() + } +} diff --git a/pkg/targets/tdengine/file_data_source.go b/pkg/targets/tdengine/file_data_source.go new file mode 100644 index 000000000..3a3530de2 --- /dev/null +++ b/pkg/targets/tdengine/file_data_source.go @@ -0,0 +1,65 @@ +package tdengine + +import ( + "bufio" + "strconv" + "strings" + + "github.com/timescale/tsbs/load" + "github.com/timescale/tsbs/pkg/data" + "github.com/timescale/tsbs/pkg/data/usecases/common" + "github.com/timescale/tsbs/pkg/targets" +) + +func newFileDataSource(fileName string) targets.DataSource { + br := load.GetBufferedReader(fileName) + + return &fileDataSource{scanner: bufio.NewScanner(br)} +} + +type fileDataSource struct { + scanner *bufio.Scanner + headers *common.GeneratedDataHeaders +} + +func (d *fileDataSource) Headers() *common.GeneratedDataHeaders { + return nil +} + +func (d *fileDataSource) NextItem() data.LoadedPoint { + ok := d.scanner.Scan() + if !ok && d.scanner.Err() == nil { // nothing scanned & no error = EOF + return data.LoadedPoint{} + } else if !ok { + fatal("scan error: %v", d.scanner.Err()) + return data.LoadedPoint{} + } + p := &point{} + line := d.scanner.Text() + p.sqlType = line[0] + switch line[0] { + case Insert: + parts := strings.SplitN(line, ",", 4) + p.subTable = parts[1] + p.fieldCount, _ = strconv.Atoi(parts[2]) + p.sql = strings.TrimSpace(parts[3]) + case CreateSTable: + parts := strings.SplitN(line, ",", 4) + p.superTable = parts[1] + p.subTable = parts[2] + p.sql = parts[3] + case CreateSubTable: + parts := strings.SplitN(line, ",", 4) + p.superTable = parts[1] + p.subTable = parts[2] + p.sql = parts[3][12:] + //case Modify: + // parts := strings.SplitN(line, ",", 4) + // p.superTable = parts[1] + // p.subTable = parts[2] + // p.sql = parts[3] + default: + panic(line) + } + return data.NewLoadedPoint(p) +} diff --git a/pkg/targets/tdengine/implemented_target.go b/pkg/targets/tdengine/implemented_target.go new file mode 100644 index 000000000..042a984dc --- /dev/null +++ b/pkg/targets/tdengine/implemented_target.go @@ -0,0 +1,58 @@ +package tdengine + +import ( + "bytes" + + "github.com/blagojts/viper" + "github.com/spf13/pflag" + "github.com/timescale/tsbs/pkg/data/serialize" + "github.com/timescale/tsbs/pkg/data/source" + "github.com/timescale/tsbs/pkg/targets" + "github.com/timescale/tsbs/pkg/targets/constants" +) + +func NewTarget() targets.ImplementedTarget { + return &tdengineTarget{} +} + +type tdengineTarget struct { +} + +func (t *tdengineTarget) TargetSpecificFlags(flagPrefix string, flagSet *pflag.FlagSet) { + TargetSpecificFlags(flagPrefix, flagSet) +} + +func TargetSpecificFlags(flagPrefix string, flagSet *pflag.FlagSet) { + flagSet.String(flagPrefix+"user", "root", "User to connect to TDengine") + flagSet.String(flagPrefix+"pass", "taosdata", "Password for user connecting to TDengine") + flagSet.String(flagPrefix+"host", "", "TDengine host") + flagSet.Int(flagPrefix+"port", 6030, "TDengine Port") + flagSet.Int(flagPrefix+"vgroups", 0, "TDengine DB vgroups") + flagSet.Int(flagPrefix+"pages", 0, "TDengine DB pages") + flagSet.Int(flagPrefix+"buffer", 0, "TDengine DB buffer") + flagSet.Int(flagPrefix+"stt_trigger", 0, "TDengine DB stt_trigger") + flagSet.Int(flagPrefix+"wal_fsync_period", 3000, "TDengine DB wal_fsync_period") + flagSet.Int(flagPrefix+"wal_level", 1, "TDengine DB wal_level") + flagSet.String(flagPrefix+"db_parameters", "", "TDengine DB parameters") +} + +func (t *tdengineTarget) TargetName() string { + return constants.FormatTDengine +} + +func (t *tdengineTarget) Serializer() serialize.PointSerializer { + return &Serializer{ + tableMap: map[string]struct{}{}, + superTable: map[string]*Table{}, + tmpBuf: &bytes.Buffer{}, + } +} + +func (t *tdengineTarget) Benchmark(targetDB string, dataSourceConfig *source.DataSourceConfig, v *viper.Viper, +) (targets.Benchmark, error) { + var loadingOptions LoadingOptions + if err := v.Unmarshal(&loadingOptions); err != nil { + return nil, err + } + return NewBenchmark(targetDB, &loadingOptions, dataSourceConfig) +} diff --git a/pkg/targets/tdengine/process.go b/pkg/targets/tdengine/process.go new file mode 100644 index 000000000..64def336a --- /dev/null +++ b/pkg/targets/tdengine/process.go @@ -0,0 +1,225 @@ +package tdengine + +import ( + "bytes" + "context" + "fmt" + "io/ioutil" + "sync" + + "github.com/timescale/tsbs/pkg/targets" + "github.com/timescale/tsbs/pkg/targets/tdengine/async" + "github.com/timescale/tsbs/pkg/targets/tdengine/commonpool" +) + +type syncCSI struct { + m sync.Map //table:ctx +} + +const Size1M = 1 * 1024 * 1024 + +type Ctx struct { + c context.Context + cancel context.CancelFunc +} + +var globalSCI = &syncCSI{} + +type processor struct { + opts *LoadingOptions + dbName string + sci *syncCSI + _db *commonpool.Conn + wg *sync.WaitGroup + buf *bytes.Buffer +} + +func newProcessor(opts *LoadingOptions, dbName string) *processor { + return &processor{opts: opts, dbName: dbName, sci: globalSCI, wg: &sync.WaitGroup{}, buf: &bytes.Buffer{}} +} + +func (p *processor) Init(_ int, doLoad, _ bool) { + if !doLoad { + return + } + p.buf.Grow(Size1M) + var err error + p._db, err = commonpool.GetConnection(p.opts.User, p.opts.Pass, p.opts.Host, p.opts.Port) + if err != nil { + panic(err) + } + err = async.GlobalAsync.TaosExecWithoutResult(p._db.TaosConnection, "use "+p.dbName) + if err != nil { + panic(err) + } +} + +func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (metricCount, rowCount uint64) { + batches := b.(*hypertableArr) + rowCnt := uint64(0) + metricCnt := batches.totalMetric + if !doLoad { + for _, sqls := range batches.m { + rowCnt += uint64(len(sqls)) + } + return metricCnt, rowCnt + } + p.buf.Reset() + p.buf.WriteString("create table") + for _, row := range batches.createSql { + switch row.sqlType { + case CreateSTable: + c, cancel := context.WithCancel(context.Background()) + ctx := &Ctx{ + c: c, + cancel: cancel, + } + actual, _ := p.sci.m.LoadOrStore(row.superTable, ctx) + err := async.GlobalAsync.TaosExecWithoutResult(p._db.TaosConnection, row.sql) + if err != nil { + fmt.Println(row.sql) + panic(err) + } + GlobalTable.Store(row.subTable, nothing) + actual.(*Ctx).cancel() + case CreateSubTable: + c, cancel := context.WithCancel(context.Background()) + ctx := &Ctx{ + c: c, + cancel: cancel, + } + actual, _ := p.sci.m.LoadOrStore(row.subTable, ctx) + + //check super table created + _, ok := GlobalTable.Load(row.superTable) + if !ok { + v, ok := p.sci.m.Load(row.superTable) + if ok { + <-v.(*Ctx).c.Done() + if p.buf.Len()+len(row.sql) > Size1M { + sql := p.buf.String() + err := async.GlobalAsync.TaosExecWithoutResult(p._db.TaosConnection, sql) + if err != nil { + fmt.Println(sql) + panic(err) + } + p.buf.Reset() + p.buf.WriteString("create table") + } + p.buf.WriteString(row.sql) + GlobalTable.Store(row.subTable, nothing) + actual.(*Ctx).cancel() + continue + } + // wait for super table created + superTableC, superTableCancel := context.WithCancel(context.Background()) + superTableCtx := &Ctx{ + c: superTableC, + cancel: superTableCancel, + } + superTableActual, _ := p.sci.m.LoadOrStore(row.superTable, superTableCtx) + <-superTableActual.(*Ctx).c.Done() + } + if p.buf.Len()+len(row.sql) > Size1M { + sql := p.buf.String() + err := async.GlobalAsync.TaosExecWithoutResult(p._db.TaosConnection, sql) + if err != nil { + fmt.Println(sql) + panic(err) + } + p.buf.Reset() + p.buf.WriteString("create table") + } + p.buf.WriteString(row.sql) + GlobalTable.Store(row.subTable, nothing) + actual.(*Ctx).cancel() + default: + panic("impossible") + } + } + if p.buf.Len() > 12 { + sql := p.buf.String() + err := async.GlobalAsync.TaosExecWithoutResult(p._db.TaosConnection, sql) + if err != nil { + fmt.Println(sql) + panic(err) + } + } + p.buf.Reset() + p.wg.Add(len(batches.m)) + for tableName := range batches.m { + tableName := tableName + go func() { + defer p.wg.Done() + _, ok := GlobalTable.Load(tableName) + if ok { + return + } + v, ok := p.sci.m.Load(tableName) + if ok { + <-v.(*Ctx).c.Done() + return + } + c, cancel := context.WithCancel(context.Background()) + ctx := &Ctx{ + c: c, + cancel: cancel, + } + actual, _ := p.sci.m.LoadOrStore(tableName, ctx) + <-actual.(*Ctx).c.Done() + return + }() + } + p.wg.Wait() + p.buf.WriteString("insert into ") + for tableName, sqls := range batches.m { + rowCnt += uint64(len(sqls)) + if p.buf.Len()+len(sqls[0])+len(tableName)+7 > Size1M { + sql := p.buf.String() + err := async.GlobalAsync.TaosExecWithoutResult(p._db.TaosConnection, sql) + if err != nil { + ioutil.WriteFile("wrongsql.txt", []byte(sql), 0755) + fmt.Println(sql) + panic(err) + } + p.buf.Reset() + p.buf.WriteString("insert into ") + } + p.buf.WriteString(tableName) + p.buf.WriteString(" values") + for i := 0; i < len(sqls); i++ { + if p.buf.Len()+len(sqls[i]) > Size1M { + sql := p.buf.String() + err := async.GlobalAsync.TaosExecWithoutResult(p._db.TaosConnection, sql) + if err != nil { + ioutil.WriteFile("wrongsql.txt", []byte(sql), 0755) + fmt.Println(sql) + panic(err) + } + p.buf.Reset() + p.buf.WriteString("insert into ") + p.buf.WriteString(tableName) + p.buf.WriteString(" values") + } + p.buf.WriteString(sqls[i]) + } + } + if p.buf.Len() > 0 { + sql := p.buf.String() + err := async.GlobalAsync.TaosExecWithoutResult(p._db.TaosConnection, sql) + if err != nil { + fmt.Println(sql) + panic(err) + } + p.buf.Reset() + } + + batches.Reset() + return metricCnt, rowCnt +} + +func (p *processor) Close(doLoad bool) { + if doLoad { + p._db.Put() + } +} diff --git a/pkg/targets/tdengine/program_options.go b/pkg/targets/tdengine/program_options.go new file mode 100644 index 000000000..164a5ccab --- /dev/null +++ b/pkg/targets/tdengine/program_options.go @@ -0,0 +1,15 @@ +package tdengine + +type LoadingOptions struct { + User string + Pass string + Host string + Port int + VGroups int + Buffer int + Pages int + SttTrigger int + WalFsyncPeriod *int + WalLevel *int + DBParameters string +} diff --git a/pkg/targets/tdengine/scan.go b/pkg/targets/tdengine/scan.go new file mode 100644 index 000000000..3789dddf3 --- /dev/null +++ b/pkg/targets/tdengine/scan.go @@ -0,0 +1,95 @@ +package tdengine + +import ( + "bytes" + "sync" + + "github.com/spaolacci/murmur3" + "github.com/timescale/tsbs/pkg/data" + "github.com/timescale/tsbs/pkg/targets" +) + +// indexer is used to consistently send the same hostnames to the same worker +type indexer struct { + buffer *bytes.Buffer + prefix []byte + partitions int + hashEndGroups []uint32 + tmp map[string]uint +} + +func (i *indexer) GetIndex(item data.LoadedPoint) uint { + p := item.Data.(*point) + idx, exist := i.tmp[p.subTable] + if exist { + return idx + } + i.buffer.Write(i.prefix) + i.buffer.WriteString(p.subTable) + hash := murmur3.Sum32WithSeed(i.buffer.Bytes(), 0x12345678) + i.buffer.Reset() + for j := 0; j < i.partitions; j++ { + if hash <= i.hashEndGroups[j] { + idx = uint(j) + break + } + } + i.tmp[p.subTable] = idx + return idx +} + +// point is a single row of data keyed by which superTable it belongs +type point struct { + sqlType byte + superTable string + subTable string + fieldCount int + sql string +} + +var GlobalTable = sync.Map{} + +type hypertableArr struct { + createSql []*point + m map[string][]string + totalMetric uint64 + cnt uint +} + +func (ha *hypertableArr) Len() uint { + return ha.cnt +} + +func (ha *hypertableArr) Append(item data.LoadedPoint) { + that := item.Data.(*point) + if that.sqlType == Insert { + ha.m[that.subTable] = append(ha.m[that.subTable], that.sql) + ha.totalMetric += uint64(that.fieldCount) + ha.cnt++ + } else { + ha.createSql = append(ha.createSql, that) + } +} + +func (ha *hypertableArr) Reset() { + ha.m = map[string][]string{} + ha.cnt = 0 + ha.createSql = ha.createSql[:0] +} + +type BatchFactory struct { + pool *sync.Pool +} + +func (b *BatchFactory) New() targets.Batch { + return b.pool.Get().(*hypertableArr) +} + +func NewBatchFactory() *BatchFactory { + pool := &sync.Pool{New: func() interface{} { + return &hypertableArr{ + m: map[string][]string{}, + } + }} + return &BatchFactory{pool: pool} +} diff --git a/pkg/targets/tdengine/serializer.go b/pkg/targets/tdengine/serializer.go new file mode 100644 index 000000000..0e248a1de --- /dev/null +++ b/pkg/targets/tdengine/serializer.go @@ -0,0 +1,232 @@ +package tdengine + +import ( + "bytes" + "fmt" + "io" + "strconv" + "strings" + + "github.com/timescale/tsbs/pkg/data" +) + +type Serializer struct { + tmpBuf *bytes.Buffer + tableMap map[string]struct{} + superTable map[string]*Table +} + +var nothing = struct{}{} + +type Table struct { + columns map[string]struct{} + tags map[string]struct{} +} + +func FastFormat(buf *bytes.Buffer, v interface{}) string { + switch v.(type) { + case int: + buf.WriteString(strconv.Itoa(v.(int))) + return "bigint" + case int64: + buf.WriteString(strconv.FormatInt(v.(int64), 10)) + return "bigint" + case float64: + buf.WriteString(strconv.FormatFloat(v.(float64), 'f', -1, 64)) + return "double" + case float32: + buf.WriteString(strconv.FormatFloat(float64(v.(float32)), 'f', -1, 32)) + return "double" + case bool: + buf.WriteString(strconv.FormatBool(v.(bool))) + return "bool" + case []byte: + buf.WriteByte('\'') + buf.WriteString(string(v.([]byte))) + buf.WriteByte('\'') + return "binary(30)" + case string: + buf.WriteByte('\'') + buf.WriteString(v.(string)) + buf.WriteByte('\'') + return "binary(30)" + case nil: + buf.WriteString("null") + return "null" + default: + panic(fmt.Sprintf("unknown field type for %#v", v)) + } +} + +var tmpMD5 = map[string]string{} +var tmpIndex = 0 + +func calculateTable(src []byte) string { + key := string(src) + v, exist := tmpMD5[key] + if exist { + return v + } + tmpIndex += 1 + v = fmt.Sprintf("t_%d", tmpIndex) + tmpMD5[key] = v + return v +} + +const ( + Insert = '1' + CreateSTable = '2' + CreateSubTable = '3' + Modify = '4' +) + +type tbNameRule struct { + tag string + prefix string + nilValue string +} + +var tbRuleMap = map[string]*tbNameRule{ + "cpu": { + tag: "hostname", + nilValue: "host_null", + }, + "readings": { + tag: "name", + prefix: "r_", + nilValue: "r_truck_null", + }, + "diagnostics": { + tag: "name", + prefix: "d_", + nilValue: "d_truck_null", + }, +} + +func (s *Serializer) Serialize(p *data.Point, w io.Writer) error { + var fieldKeys []string + var fieldValues []string + var fieldTypes []string + var tagValues []string + var tagKeys []string + var tagTypes []string + tKeys := p.TagKeys() + tValues := p.TagValues() + fKeys := p.FieldKeys() + fValues := p.FieldValues() + superTable := string(p.MeasurementName()) + for i, value := range fValues { + fType := FastFormat(s.tmpBuf, value) + fieldKeys = append(fieldKeys, convertKeywords(string(fKeys[i]))) + fieldTypes = append(fieldTypes, fType) + fieldValues = append(fieldValues, s.tmpBuf.String()) + s.tmpBuf.Reset() + } + + rule := tbRuleMap[superTable] + fixedName := "" + for i, value := range tValues { + tType := FastFormat(s.tmpBuf, value) + if rule != nil && len(fixedName) == 0 && string(tKeys[i]) == rule.tag { + str, is := value.(string) + if is { + fixedName = str + } + } + tagKeys = append(tagKeys, convertKeywords(string(tKeys[i]))) + tagTypes = append(tagTypes, tType) + tagValues = append(tagValues, s.tmpBuf.String()) + s.tmpBuf.Reset() + } + + subTable := "" + if rule != nil { + if len(fixedName) != 0 { + if len(rule.prefix) == 0 { + subTable = fixedName + } else { + s.tmpBuf.WriteString(rule.prefix) + s.tmpBuf.WriteString(fixedName) + subTable = s.tmpBuf.String() + s.tmpBuf.Reset() + } + } else { + subTable = rule.nilValue + } + } else { + s.tmpBuf.WriteString(superTable) + for i, v := range tagValues { + s.tmpBuf.WriteByte(',') + s.tmpBuf.WriteString(tagKeys[i]) + s.tmpBuf.WriteByte('=') + s.tmpBuf.WriteString(v) + } + subTable = calculateTable(s.tmpBuf.Bytes()) + s.tmpBuf.Reset() + } + stable, exist := s.superTable[superTable] + if !exist { + for i := 0; i < len(fieldTypes); i++ { + s.tmpBuf.WriteByte(',') + s.tmpBuf.WriteString(fieldKeys[i]) + s.tmpBuf.WriteByte(' ') + s.tmpBuf.WriteString(fieldTypes[i]) + } + fieldStr := s.tmpBuf.String() + s.tmpBuf.Reset() + for i := 0; i < len(tagTypes); i++ { + s.tmpBuf.WriteString(tagKeys[i]) + s.tmpBuf.WriteByte(' ') + s.tmpBuf.WriteString(tagTypes[i]) + if i != len(tagTypes)-1 { + s.tmpBuf.WriteByte(',') + } + } + tagStr := s.tmpBuf.String() + fmt.Fprintf(w, "%c,%s,%s,create table %s (ts timestamp%s) tags (%s)\n", CreateSTable, superTable, subTable, superTable, fieldStr, tagStr) + s.tmpBuf.Reset() + table := &Table{ + columns: map[string]struct{}{}, + tags: map[string]struct{}{}, + } + for _, key := range fieldKeys { + table.columns[key] = nothing + } + for _, key := range tagKeys { + table.tags[key] = nothing + } + s.superTable[superTable] = table + } else { + for _, key := range fieldKeys { + if _, exist = stable.columns[key]; !exist { + fmt.Fprintf(w, "%c,%s,%s,alter table %s add COLUMN %s double\n", Modify, superTable, subTable, superTable, key) + stable.columns[key] = nothing + } + } + for _, key := range tagKeys { + if _, exist = stable.tags[key]; !exist { + fmt.Fprintf(w, "%c,%s,%s,alter table %s add TAG %s binary(30)\n", Modify, superTable, subTable, superTable, key) + stable.tags[key] = nothing + } + } + } + _, exist = s.tableMap[subTable] + if !exist { + fmt.Fprintf(w, "%c,%s,%s,create table %s using %s (%s) tags (%s)\n", CreateSubTable, superTable, subTable, subTable, superTable, strings.Join(tagKeys, ","), strings.Join(tagValues, ",")) + s.tableMap[subTable] = nothing + } + + fmt.Fprintf(w, "%c,%s,%d,(%d,%s)\n", Insert, subTable, len(fieldValues), p.TimestampInUnixMs(), strings.Join(fieldValues, ",")) + return nil +} + +var keyWords = map[string]bool{ + "port": true, +} + +func convertKeywords(s string) string { + if is := keyWords[s]; is { + return fmt.Sprintf("`%s`", s) + } + return s +} diff --git a/pkg/targets/tdengine/thread/locker.go b/pkg/targets/tdengine/thread/locker.go new file mode 100644 index 000000000..2ad485075 --- /dev/null +++ b/pkg/targets/tdengine/thread/locker.go @@ -0,0 +1,33 @@ +package thread + +import "runtime" + +type Locker struct { + c chan struct{} +} + +func NewLocker(count int) *Locker { + return &Locker{c: make(chan struct{}, count)} +} + +func (l *Locker) Lock() { + l.c <- struct{}{} +} + +func (l *Locker) Unlock() { + <-l.c +} + +var c chan struct{} + +func Lock() { + c <- struct{}{} +} + +func Unlock() { + <-c +} + +func init() { + c = make(chan struct{}, runtime.NumCPU()) +} diff --git a/pkg/targets/tdengine/thread/locker_test.go b/pkg/targets/tdengine/thread/locker_test.go new file mode 100644 index 000000000..f49be5c33 --- /dev/null +++ b/pkg/targets/tdengine/thread/locker_test.go @@ -0,0 +1,41 @@ +package thread + +import ( + "testing" +) + +// @author: xftan +// @date: 2021/12/14 15:16 +// @description: test NewLocker +func TestNewLocker(t *testing.T) { + type args struct { + count int + } + tests := []struct { + name string + args args + }{ + { + name: "test", + args: args{ + count: 1, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + locker := NewLocker(tt.args.count) + locker.Lock() + locker.Unlock() + }) + } +} + +// @author: xftan +// @date: 2021/12/14 15:16 +// @description: test DefaultLocker +func TestDefaultLocker(t *testing.T) { + Lock() + t.Log("success") + defer Unlock() +} diff --git a/pkg/targets/tdenginestmt2/benchmark.go b/pkg/targets/tdenginestmt2/benchmark.go new file mode 100644 index 000000000..2038b34ae --- /dev/null +++ b/pkg/targets/tdenginestmt2/benchmark.go @@ -0,0 +1,105 @@ +package tdenginestmt2 + +import "C" +import ( + "math" + + "github.com/timescale/tsbs/pkg/data/source" + "github.com/timescale/tsbs/pkg/targets" + "github.com/timescale/tsbs/pkg/targets/tdengine" +) + +func NewBenchmark(dbName string, opts *tdengine.LoadingOptions, dataSourceConfig *source.DataSourceConfig) (targets.Benchmark, error) { + var ds targets.DataSource + if dataSourceConfig.Type == source.FileDataSourceType { + ds = newFileDataSource(dataSourceConfig.File.Location) + useCase, scale := ds.(*fileDataSource).readHeader() + return &benchmark{ + opts: opts, + dataSource: ds, + dbName: dbName, + factory: NewBatchFactory(), + useCase: useCase, + scale: scale, + }, nil + } else { + panic("not implement") + } +} + +type benchmark struct { + opts *tdengine.LoadingOptions + dataSource targets.DataSource + dbName string + batchSize uint + factory targets.BatchFactory + indexer targets.PointIndexer + useCase byte + scale uint32 + tableOffset [3][]uint32 + partitionTables [3][][]uint32 +} + +func (b *benchmark) GetDataSource() targets.DataSource { + return b.dataSource +} + +func (b *benchmark) GetBatchFactory() targets.BatchFactory { + return b.factory +} + +func (b *benchmark) GetPointIndexer(uint) targets.PointIndexer { + return b.indexer +} + +func (b *benchmark) GetProcessor() targets.Processor { + return newProcessor(b.opts, b.dbName, b.batchSize, b.useCase, b.scale, b.partitionTables, b.tableOffset) +} + +func (b *benchmark) GetDBCreator() targets.DBCreator { + return &DbCreator{ + DbCreator: tdengine.DbCreator{Opts: b.opts}, + useCase: b.useCase, + ds: b.dataSource.(*fileDataSource), + } +} + +func (b *benchmark) SetConfig(batchSize uint, workers uint) { + b.batchSize = batchSize + + b.dataSource.(*fileDataSource).SetConfig(int(workers), int(batchSize), int(b.scale)) + factory := b.factory.(*BatchFactory) + factory.batchSize = batchSize + if workers > 1 { + interval := uint32(math.MaxUint32 / workers) + hashEndGroups := make([]uint32, workers) + for i := 0; i < int(workers); i++ { + if i == int(workers)-1 { + hashEndGroups[i] = math.MaxUint32 + } else { + hashEndGroups[i] = interval*uint32(i+1) - 1 + } + } + prefix := []byte("1." + b.dbName + ".") + idx, tableOffset, hostTableIndex, readingsTableIndex, diagnosticsTableIndex := NewIndexer(prefix, int(workers), hashEndGroups, b.useCase, b.scale) + b.tableOffset = tableOffset + b.partitionTables = [3][][]uint32{hostTableIndex, readingsTableIndex, diagnosticsTableIndex} + b.indexer = idx + } else { + b.indexer = &targets.ConstantIndexer{} + tmp := make([]uint32, b.scale+1) + for i := uint32(0); i < b.scale+1; i++ { + tmp[i] = i + } + switch b.useCase { + case CpuCase: + b.partitionTables[SuperTableHost] = [][]uint32{tmp} + b.tableOffset[SuperTableHost] = tmp + case IoTCase: + b.partitionTables[SuperTableReadings] = [][]uint32{tmp} + b.tableOffset[SuperTableReadings] = tmp + b.partitionTables[SuperTableDiagnostics] = [][]uint32{tmp} + b.tableOffset[SuperTableDiagnostics] = tmp + } + } +} diff --git a/pkg/targets/tdenginestmt2/cb.go b/pkg/targets/tdenginestmt2/cb.go new file mode 100644 index 000000000..62a3f57bf --- /dev/null +++ b/pkg/targets/tdenginestmt2/cb.go @@ -0,0 +1,23 @@ +package tdenginestmt2 + +/* +#cgo CFLAGS: -I/usr/include +#cgo linux LDFLAGS: -L/usr/lib -ltaos +#include +#include +#include +#include +*/ +import "C" +import ( + "unsafe" + + "github.com/taosdata/driver-go/v3/wrapper" + "github.com/taosdata/driver-go/v3/wrapper/cgo" +) + +//export QueryCallback2 +func QueryCallback2(p unsafe.Pointer, res *C.TAOS_RES, code C.int) { + caller := (*(*cgo.Handle)(p)).Value().(wrapper.Caller) + caller.QueryCall(unsafe.Pointer(res), int(code)) +} diff --git a/pkg/targets/tdenginestmt2/creator.go b/pkg/targets/tdenginestmt2/creator.go new file mode 100644 index 000000000..64ff3aa2e --- /dev/null +++ b/pkg/targets/tdenginestmt2/creator.go @@ -0,0 +1,36 @@ +package tdenginestmt2 + +import ( + "github.com/timescale/tsbs/pkg/targets/tdengine" + "github.com/timescale/tsbs/pkg/targets/tdengine/async" +) + +type DbCreator struct { + tdengine.DbCreator + useCase byte + ds *fileDataSource +} + +func (d *DbCreator) PostCreateDB(dbName string) error { + err := async.GlobalAsync.TaosExecWithoutResult(d.Db.TaosConnection, "use "+dbName) + if err != nil { + return err + } + switch d.useCase { + case CpuCase: + err = async.GlobalAsync.TaosExecWithoutResult(d.Db.TaosConnection, CpuSql) + if err != nil { + return err + } + case IoTCase: + err = async.GlobalAsync.TaosExecWithoutResult(d.Db.TaosConnection, ReadingsSql) + if err != nil { + return err + } + err = async.GlobalAsync.TaosExecWithoutResult(d.Db.TaosConnection, DiagnosticsSql) + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/targets/tdenginestmt2/file_data_source.go b/pkg/targets/tdenginestmt2/file_data_source.go new file mode 100644 index 000000000..f38795d42 --- /dev/null +++ b/pkg/targets/tdenginestmt2/file_data_source.go @@ -0,0 +1,198 @@ +package tdenginestmt2 + +import ( + "bufio" + "encoding/binary" + "io" + "log" + "os" + "reflect" + "unsafe" + + "github.com/timescale/tsbs/pkg/data" + "github.com/timescale/tsbs/pkg/data/usecases/common" + "github.com/timescale/tsbs/pkg/targets" +) + +var fatal = log.Fatalf + +func newFileDataSource(fileName string) targets.DataSource { + br := GetBufferedReader(fileName) + return &fileDataSource{br: br, exchange: make(chan struct{}, 1), exchangeStatus: make(chan int, 1)} +} + +const ( + defaultReadSize = 4 << 20 // 4 MB +) + +func GetBufferedReader(fileName string) *bufio.Reader { + if len(fileName) == 0 { + // Read from STDIN + return bufio.NewReaderSize(os.Stdin, defaultReadSize) + } + // Read from specified file + file, err := os.Open(fileName) + if err != nil { + fatal("cannot open file for read %s: %v", fileName, err) + return nil + } + return bufio.NewReaderSize(file, defaultReadSize) +} + +type fileDataSource struct { + br *bufio.Reader + cache []*[]byte + cacheIndex int + cacheSize int + scale int + exchange chan struct{} + exchangeStatus chan int + readDirect bool +} + +/* + fixed header + +| version(1 byte) | case (1 byte) | scale (4 bytes) + +*/ + +func (d *fileDataSource) readHeader() (byte, uint32) { + buf := make([]byte, 6) + _, err := io.ReadFull(d.br, buf) + if err != nil { + fatal("cannot read header: %v", err) + } + if buf[0] != 1 { + fatal("invalid version: %d", buf[0]) + } + scale := binary.LittleEndian.Uint32(buf[2:]) + d.scale = int(scale) + return buf[1], scale +} + +func (d *fileDataSource) SetConfig(worker int, batchSize int, scale int) { + globalSlicePool = NewSlicePool(worker, batchSize, scale) +} + +func (d *fileDataSource) Headers() *common.GeneratedDataHeaders { + return nil +} + +var globalLoadedPoint = data.NewLoadedPoint(nil) + +type SlicePool struct { + smallPool chan []*[]byte +} + +var globalSlicePool *SlicePool + +func NewSlicePool(worker, batchSize, scale int) *SlicePool { + totalItem := (worker*batchSize)*500 + scale*2 + 2 + arrayLen := worker * 2 + pool := make(chan []*[]byte, arrayLen*50) + totalBytes := make([]*[]byte, totalItem) + for i := 0; i < totalItem; i++ { + bs := make([]byte, 0, 256) + totalBytes[i] = &bs + } + bss := SplitBytes(totalBytes, arrayLen) + for i := 0; i < arrayLen; i++ { + pool <- bss[i] + } + return &SlicePool{smallPool: pool} +} + +func (p *SlicePool) Get() []*[]byte { + //return <-p.smallPool + select { + case b := <-p.smallPool: + return b + default: + panic("no more slice") + } +} + +func (p *SlicePool) Put(b []*[]byte) { + select { + case p.smallPool <- b: + default: + return + } +} + +func (d *fileDataSource) NextItem() data.LoadedPoint { + if d.cacheIndex == d.cacheSize { + d.cache = globalSlicePool.Get() + d.cacheSize = len(d.cache) + d.cacheIndex = 0 + } + u8length, err := d.br.ReadByte() + if err != nil { + if err == io.EOF { + globalLoadedPoint.Data = nil + return globalLoadedPoint + } + panic(err) + } + if u8length < 128 { + length := int(u8length) + ptr := d.cache[d.cacheIndex] + (*reflect.SliceHeader)(unsafe.Pointer(ptr)).Len = length + s := *ptr + nn, err := d.br.Read(s) + if err != nil { + panic(err) + } + if nn != length { + n := nn + for n < length && err == nil { + nn, err = d.br.Read(s[n:]) + n += nn + } + } + d.cacheIndex++ + globalLoadedPoint.Data = ptr + return globalLoadedPoint + } else { + tmp, err := d.br.ReadByte() + if err != nil { + panic(err) + } + u16Length := int(u8length&0x7f) + int(tmp<<7) + ptr := d.cache[d.cacheIndex] + (*reflect.SliceHeader)(unsafe.Pointer(ptr)).Len = u16Length + s := *ptr + nn, err := d.br.Read(s) + if err != nil { + panic(err) + } + if nn != u16Length { + n := nn + for n < u16Length && err == nil { + nn, err = d.br.Read(s[n:]) + n += nn + } + } + d.cacheIndex++ + globalLoadedPoint.Data = ptr + return globalLoadedPoint + } +} + +func SplitBytes(arr []*[]byte, n int) [][]*[]byte { + if n <= 0 { + return nil + } + subArraySize := (len(arr) + n - 1) / n + result := make([][]*[]byte, 0, n) + for i := 0; i < len(arr); i += subArraySize { + end := i + subArraySize + if end > len(arr) { + end = len(arr) + } + result = append(result, arr[i:end]) + } + + return result +} diff --git a/pkg/targets/tdenginestmt2/implemented_target.go b/pkg/targets/tdenginestmt2/implemented_target.go new file mode 100644 index 000000000..68d4b3fa8 --- /dev/null +++ b/pkg/targets/tdenginestmt2/implemented_target.go @@ -0,0 +1,44 @@ +package tdenginestmt2 + +import ( + "bytes" + + "github.com/blagojts/viper" + "github.com/spf13/pflag" + "github.com/timescale/tsbs/pkg/data/serialize" + "github.com/timescale/tsbs/pkg/data/source" + "github.com/timescale/tsbs/pkg/targets" + "github.com/timescale/tsbs/pkg/targets/constants" + "github.com/timescale/tsbs/pkg/targets/tdengine" +) + +func NewTarget() targets.ImplementedTarget { + return &tdengineStmt2Target{} +} + +type tdengineStmt2Target struct { +} + +func (t *tdengineStmt2Target) TargetSpecificFlags(flagPrefix string, flagSet *pflag.FlagSet) { + tdengine.TargetSpecificFlags(flagPrefix, flagSet) +} + +func (t *tdengineStmt2Target) TargetName() string { + return constants.FormatTDengineStmt2 +} + +func (t *tdengineStmt2Target) Serializer() serialize.PointSerializer { + return &Serializer{ + tmpBuf: &bytes.Buffer{}, + writeBuf: &bytes.Buffer{}, + } +} + +func (t *tdengineStmt2Target) Benchmark(targetDB string, dataSourceConfig *source.DataSourceConfig, v *viper.Viper, +) (targets.Benchmark, error) { + var loadingOptions tdengine.LoadingOptions + if err := v.Unmarshal(&loadingOptions); err != nil { + return nil, err + } + return NewBenchmark(targetDB, &loadingOptions, dataSourceConfig) +} diff --git a/pkg/targets/tdenginestmt2/process.go b/pkg/targets/tdenginestmt2/process.go new file mode 100644 index 000000000..16ded2f93 --- /dev/null +++ b/pkg/targets/tdenginestmt2/process.go @@ -0,0 +1,932 @@ +package tdenginestmt2 + +/* +#cgo CFLAGS: -I/usr/include +#cgo linux LDFLAGS: -L/usr/lib -ltaos +#include +#include +#include +#include +extern void QueryCallback2(void *param,TAOS_RES *,int code); +void taos_query_a_wrapper2(TAOS *taos,const char *sql, void *param){ + return taos_query_a(taos,sql,QueryCallback2,param); +}; +*/ +import "C" +import ( + "context" + "fmt" + "reflect" + "strconv" + "sync" + "sync/atomic" + "unsafe" + + taosCommon "github.com/taosdata/driver-go/v3/common" + "github.com/taosdata/driver-go/v3/wrapper" + "github.com/taosdata/driver-go/v3/wrapper/cgo" + "github.com/timescale/tsbs/pkg/targets" + "github.com/timescale/tsbs/pkg/targets/tdengine" + "github.com/timescale/tsbs/pkg/targets/tdengine/async" + "github.com/timescale/tsbs/pkg/targets/tdengine/commonpool" +) + +type Ctx struct { + c context.Context + cancel context.CancelFunc +} + +type processor struct { + tableSlot [3][]uint32 + cBuffers [3]*bufferPointer + tableNamePointerCache [3][]unsafe.Pointer + stmt2CHandle [3]unsafe.Pointer + stmt2CBHandle [3]*async.Handler + hostTableNameCBuffer unsafe.Pointer + readingsTableNameCBuffer unsafe.Pointer + diagnosticsTableNameCBuffer unsafe.Pointer + + hostSlot [][][]byte + readingsSlot [][][]byte + diagnosticsSlot [][][]byte + hostBatchIndexer []uint32 + readingsBatchIndexer []uint32 + diagnosticsBatchIndexer []uint32 + wg *sync.WaitGroup + partitionTable [3][][]uint32 + + _db *commonpool.Conn + bufIndex int + cBufPosition unsafe.Pointer + cBuf unsafe.Pointer + caller *async.Caller + hp unsafe.Pointer + handle cgo.Handle + + hostExecCount int32 + readingsExecCount int32 + diagnosticsExecCount int32 + createWg *sync.WaitGroup + opts *tdengine.LoadingOptions + dbName string + useCase int + batchSize uint + scale uint32 + + exitSign chan struct{} + finishWg *sync.WaitGroup +} + +type bufferPointer struct { + bindVP unsafe.Pointer + tableNameP []unsafe.Pointer + bindsP [][]unsafe.Pointer + colP []unsafe.Pointer + isNullP []unsafe.Pointer +} + +const Size1M = 1 << 20 + +func newProcessor(opts *tdengine.LoadingOptions, dbName string, batchSize uint, useCase byte, scale uint32, partitionTable [3][][]uint32, tableOffset [3][]uint32) *processor { + p := &processor{ + tableSlot: tableOffset, + partitionTable: partitionTable, + opts: opts, + dbName: dbName, + batchSize: batchSize, + wg: &sync.WaitGroup{}, + createWg: &sync.WaitGroup{}, + useCase: int(useCase), + scale: scale, + exitSign: make(chan struct{}, 1), + finishWg: &sync.WaitGroup{}, + } + return p +} + +const ( + CpuHandleIndex = 0 + ReadingsHandleIndex = 1 + DiagnosticsHandleIndex = 2 +) + +func (p *processor) Init(id int, doLoad, _ bool) { + if !doLoad { + return + } + var err error + p._db, err = commonpool.GetConnection(p.opts.User, p.opts.Pass, p.opts.Host, p.opts.Port) + if err != nil { + panic(err) + } + err = async.GlobalAsync.TaosExecWithoutResult(p._db.TaosConnection, "use "+p.dbName) + if err != nil { + panic(err) + } + defer func() { + if err := recover(); err != nil { + p.Close(true) + panic(err) + } + }() + p.caller = async.NewCaller() + p.handle = cgo.NewHandle(p.caller) + p.hp = p.handle.Pointer() + p.cBuf = C.malloc(Size1M) + createTable := []byte("create table") + createTableP := unsafe.Pointer(&createTable[0]) + C.memcpy(p.cBuf, createTableP, C.size_t(12)) + p.cBufPosition = p.cBuf + p.bufIndex = 0 + + // max table name 23 + switch p.useCase { + case CpuCase: + // init stmt2 + handler := async.GlobalAsync.HandlerPool.Get() + prepareSql := "insert into ? values(?,?,?,?,?,?,?,?,?,?,?)" + stmt2Pointer := wrapper.TaosStmt2Init(p._db.TaosConnection, int64(1<<56|id), true, true, handler.Handler) + if stmt2Pointer == nil { + panic(fmt.Errorf("failed to create stmt2")) + } + code := wrapper.TaosStmt2Prepare(stmt2Pointer, prepareSql) + if code != 0 { + errMsg := wrapper.TaosStmt2Error(stmt2Pointer) + panic(fmt.Errorf("failed to prepare stmt2: %s", errMsg)) + } + + // init table name c buffer + hostTableIndex := p.partitionTable[SuperTableHost][id] + p.tableNamePointerCache[SuperTableHost] = make([]unsafe.Pointer, len(hostTableIndex)) + p.tableNamePointerCache[SuperTableHost] = make([]unsafe.Pointer, len(hostTableIndex)) + p.hostTableNameCBuffer = C.calloc(C.size_t(24*(len(hostTableIndex))), 1) + cursor := p.hostTableNameCBuffer + prefix := []byte("host_") + var tableNameBytes []byte + for i := uint64(0); i < uint64(len(hostTableIndex)); i++ { + if hostTableIndex[i] == 0 { + tableNameNull := []byte("host_null") + C.memcpy(cursor, unsafe.Pointer(&tableNameNull[0]), C.size_t(len(tableNameNull))) + } else { + tableNameBytes = strconv.AppendUint(prefix, uint64(hostTableIndex[i]-1), 10) + C.memcpy(cursor, unsafe.Pointer(&tableNameBytes[0]), C.size_t(len(tableNameBytes))) + } + p.tableNamePointerCache[SuperTableHost][i] = cursor + cursor = unsafe.Pointer(uintptr(cursor) + 24) + } + + // crate stmt2 bind buffer + cBuffer := allocBuffer(len(hostTableIndex), int(p.batchSize), []int{ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + }, []int{ + taosCommon.TSDB_DATA_TYPE_TIMESTAMP, + taosCommon.TSDB_DATA_TYPE_BIGINT, + taosCommon.TSDB_DATA_TYPE_BIGINT, + taosCommon.TSDB_DATA_TYPE_BIGINT, + taosCommon.TSDB_DATA_TYPE_BIGINT, + taosCommon.TSDB_DATA_TYPE_BIGINT, + taosCommon.TSDB_DATA_TYPE_BIGINT, + taosCommon.TSDB_DATA_TYPE_BIGINT, + taosCommon.TSDB_DATA_TYPE_BIGINT, + taosCommon.TSDB_DATA_TYPE_BIGINT, + taosCommon.TSDB_DATA_TYPE_BIGINT, + }) + p.cBuffers[CpuHandleIndex] = cBuffer + p.stmt2CHandle[CpuHandleIndex] = stmt2Pointer + p.stmt2CBHandle[CpuHandleIndex] = handler + + // init slot + p.hostSlot = make([][][]byte, len(hostTableIndex)) + for i := 0; i < len(hostTableIndex); i++ { + p.hostSlot[i] = make([][]byte, 0, 10) + } + p.hostBatchIndexer = make([]uint32, 0, p.batchSize) + p.receiveHostExecResult() + + case IoTCase: + // init stmt2 + handlerReading := async.GlobalAsync.HandlerPool.Get() + prepareReadingSql := "insert into ? values(?,?,?,?,?,?,?,?)" + stmt2ReadingPointer := wrapper.TaosStmt2Init(p._db.TaosConnection, int64(2<<56|id), true, true, handlerReading.Handler) + if stmt2ReadingPointer == nil { + panic(fmt.Errorf("failed to create stmt2")) + } + code := wrapper.TaosStmt2Prepare(stmt2ReadingPointer, prepareReadingSql) + if code != 0 { + errMsg := wrapper.TaosStmt2Error(stmt2ReadingPointer) + panic(fmt.Errorf("failed to prepare stmt2: %s", errMsg)) + } + + handlerDiagnostics := async.GlobalAsync.HandlerPool.Get() + prepareDiagnosticsSql := "insert into ? values(?,?,?,?)" + stmt2DiagnosticsPointer := wrapper.TaosStmt2Init(p._db.TaosConnection, int64(3<<56|id), true, true, handlerDiagnostics.Handler) + if stmt2DiagnosticsPointer == nil { + panic(fmt.Errorf("failed to create stmt2")) + } + code = wrapper.TaosStmt2Prepare(stmt2DiagnosticsPointer, prepareDiagnosticsSql) + if code != 0 { + errMsg := wrapper.TaosStmt2Error(stmt2DiagnosticsPointer) + panic(fmt.Errorf("failed to prepare stmt2: %s", errMsg)) + } + + // init table name c buffer + readingsTableIndex := p.partitionTable[SuperTableReadings][id] + p.tableNamePointerCache[SuperTableReadings] = make([]unsafe.Pointer, len(readingsTableIndex)) + p.readingsTableNameCBuffer = C.calloc(C.size_t(24*len(readingsTableIndex)), 1) + cursor := p.readingsTableNameCBuffer + prefix := []byte("r_truck_") + var tableNameBytes []byte + for i := uint64(0); i < uint64(len(readingsTableIndex)); i++ { + if readingsTableIndex[i] == 0 { + tableNameNull := []byte("r_truck_null") + C.memcpy(cursor, unsafe.Pointer(&tableNameNull[0]), C.size_t(len(tableNameNull))) + } else { + tableNameBytes = strconv.AppendUint(prefix, uint64(readingsTableIndex[i]-1), 10) + C.memcpy(cursor, unsafe.Pointer(&tableNameBytes[0]), C.size_t(len(tableNameBytes))) + } + p.tableNamePointerCache[SuperTableReadings][i] = cursor + cursor = unsafe.Pointer(uintptr(cursor) + 24) + } + + diagnosticsTableIndex := p.partitionTable[SuperTableDiagnostics][id] + p.tableNamePointerCache[SuperTableDiagnostics] = make([]unsafe.Pointer, len(diagnosticsTableIndex)) + p.diagnosticsTableNameCBuffer = C.calloc(C.size_t(24*len(diagnosticsTableIndex)), 1) + cursor = p.diagnosticsTableNameCBuffer + prefix = []byte("d_truck_") + for i := uint64(0); i < uint64(len(diagnosticsTableIndex)); i++ { + if diagnosticsTableIndex[i] == 0 { + tableNameNull := []byte("d_truck_null") + C.memcpy(cursor, unsafe.Pointer(&tableNameNull[0]), C.size_t(len(tableNameNull))) + } else { + tableNameBytes = strconv.AppendUint(prefix, uint64(diagnosticsTableIndex[i]-1), 10) + C.memcpy(cursor, unsafe.Pointer(&tableNameBytes[0]), C.size_t(len(tableNameBytes))) + } + p.tableNamePointerCache[SuperTableDiagnostics][i] = cursor + cursor = unsafe.Pointer(uintptr(cursor) + 24) + } + + // crate stmt2 bind buffer + cBuffer := allocBuffer(len(readingsTableIndex), int(p.batchSize), []int{ + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + }, []int{ + taosCommon.TSDB_DATA_TYPE_TIMESTAMP, + taosCommon.TSDB_DATA_TYPE_DOUBLE, + taosCommon.TSDB_DATA_TYPE_DOUBLE, + taosCommon.TSDB_DATA_TYPE_DOUBLE, + taosCommon.TSDB_DATA_TYPE_DOUBLE, + taosCommon.TSDB_DATA_TYPE_DOUBLE, + taosCommon.TSDB_DATA_TYPE_DOUBLE, + taosCommon.TSDB_DATA_TYPE_DOUBLE, + }) + p.cBuffers[ReadingsHandleIndex] = cBuffer + p.stmt2CHandle[ReadingsHandleIndex] = stmt2ReadingPointer + p.stmt2CBHandle[ReadingsHandleIndex] = handlerReading + + cBuffer = allocBuffer(len(diagnosticsTableIndex), int(p.batchSize), []int{ + 8, + 8, + 8, + 8, + }, []int{ + taosCommon.TSDB_DATA_TYPE_TIMESTAMP, + taosCommon.TSDB_DATA_TYPE_DOUBLE, + taosCommon.TSDB_DATA_TYPE_DOUBLE, + taosCommon.TSDB_DATA_TYPE_BIGINT, + }) + p.cBuffers[DiagnosticsHandleIndex] = cBuffer + p.stmt2CHandle[DiagnosticsHandleIndex] = stmt2DiagnosticsPointer + p.stmt2CBHandle[DiagnosticsHandleIndex] = handlerDiagnostics + + // init slot + p.readingsSlot = make([][][]byte, len(readingsTableIndex)) + for i := 0; i < len(readingsTableIndex); i++ { + p.readingsSlot[i] = make([][]byte, 0, 10) + } + p.readingsBatchIndexer = make([]uint32, 0, p.batchSize) + + p.diagnosticsSlot = make([][][]byte, len(diagnosticsTableIndex)) + for i := 0; i < len(diagnosticsTableIndex); i++ { + p.diagnosticsSlot[i] = make([][]byte, 0, 10) + } + p.diagnosticsBatchIndexer = make([]uint32, 0, p.batchSize) + + p.receiveReadingsExecResult() + p.receiveDiagnosticsExecResult() + } +} + +func (p *processor) receiveHostExecResult() { + p.finishWg.Add(1) + go func() { + defer p.finishWg.Done() + stmtHandler := p.stmt2CHandle[CpuHandleIndex] + handle := p.stmt2CBHandle[CpuHandleIndex] + exitSign := p.exitSign + for { + select { + case result := <-handle.Caller.ExecResult: + if result.Code != 0 { + errStr := wrapper.TaosStmt2Error(stmtHandler) + panic(fmt.Errorf("failed to exec stmt2: %d:%s", result.Code, errStr)) + } + atomic.AddInt32(&p.hostExecCount, -1) + case <-exitSign: + c := atomic.LoadInt32(&p.hostExecCount) + if c == 0 { + return + } + for i := int32(0); i < c; i++ { + result := <-handle.Caller.ExecResult + if result.Code != 0 { + errStr := wrapper.TaosStmt2Error(stmtHandler) + panic(fmt.Errorf("failed to exec stmt2: %d:%s", result.Code, errStr)) + } + } + return + } + } + }() +} + +func (p *processor) receiveReadingsExecResult() { + p.finishWg.Add(1) + go func() { + defer p.finishWg.Done() + stmtHandler := p.stmt2CHandle[ReadingsHandleIndex] + handle := p.stmt2CBHandle[ReadingsHandleIndex] + exitSign := p.exitSign + for { + select { + case result := <-handle.Caller.ExecResult: + if result.Code != 0 { + errStr := wrapper.TaosStmt2Error(stmtHandler) + panic(fmt.Errorf("failed to exec stmt2: %d:%s", result.Code, errStr)) + } + atomic.AddInt32(&p.readingsExecCount, -1) + case <-exitSign: + c := atomic.LoadInt32(&p.readingsExecCount) + if c == 0 { + return + } + for i := int32(0); i < c; i++ { + result := <-handle.Caller.ExecResult + if result.Code != 0 { + errStr := wrapper.TaosStmt2Error(stmtHandler) + panic(fmt.Errorf("failed to exec stmt2: %d:%s", result.Code, errStr)) + } + } + return + } + } + }() +} + +func (p *processor) receiveDiagnosticsExecResult() { + p.finishWg.Add(1) + go func() { + defer p.finishWg.Done() + stmtHandler := p.stmt2CHandle[DiagnosticsHandleIndex] + handle := p.stmt2CBHandle[DiagnosticsHandleIndex] + exitSign := p.exitSign + for { + select { + case result := <-handle.Caller.ExecResult: + if result.Code != 0 { + errStr := wrapper.TaosStmt2Error(stmtHandler) + panic(fmt.Errorf("failed to exec stmt2: %d:%s", result.Code, errStr)) + } + atomic.AddInt32(&p.diagnosticsExecCount, -1) + case <-exitSign: + c := atomic.LoadInt32(&p.diagnosticsExecCount) + if c == 0 { + return + } + for i := int32(0); i < c; i++ { + result := <-handle.Caller.ExecResult + if result.Code != 0 { + errStr := wrapper.TaosStmt2Error(stmtHandler) + panic(fmt.Errorf("failed to exec stmt2: %d:%s", result.Code, errStr)) + } + } + return + } + } + }() +} + +//typedef struct TAOS_STMT2_BINDV { +// int count; +// char **tbnames; +// TAOS_STMT2_BIND **tags; +// TAOS_STMT2_BIND **bind_cols; +//} TAOS_STMT2_BINDV; + +// typedef struct TAOS_STMT2_BIND { +// int buffer_type; +// void *buffer; +// int32_t *length; +// char *is_null; +// int num; +//} TAOS_STMT2_BIND; + +const BindVSize = uintptr(C.sizeof_struct_TAOS_STMT2_BINDV) +const BindSize = uintptr(C.sizeof_struct_TAOS_STMT2_BIND) + +func allocBuffer(partitionTables int, batchSize int, colSize []int, colTypes []int) *bufferPointer { + colCount := len(colSize) + bufferSize := 0 + for i := 0; i < colCount; i++ { + bufferSize += colSize[i] + } + bindTables := partitionTables + if bindTables > batchSize { + bindTables = batchSize + } + maxRow := batchSize + /* + + | TAOS_STMT2_BINDV | + | tbnames pointer 1 | tbnames pointer 2 | tbnames pointer 3 | tbnames pointer ... | maxTable + | bind_cols pointer 1 | bind_cols pointer 2 | bind_cols pointer 3 | bind_cols pointer ... | maxTable * colCount + | TAOS_STMT2_BIND 1 | TAOS_STMT2_BIND 2 | TAOS_STMT2_BIND 3 | TAOS_STMT2_BIND ... | maxTable * colCount + */ + + // bindv + tbname pointer + bindcols pointer + bind struct pointer + bind structs + fixedSize := + // TAOS_STMT2_BINDV + BindVSize + + // table name pointer + wrapper.PointerSize*uintptr(bindTables) + + // bind_cols pointer + wrapper.PointerSize*uintptr(bindTables*colCount) + + // bind_cols TAOS_STMT2_BIND + BindSize*uintptr(bindTables*colCount) + + dataSize := uintptr(bufferSize*maxRow) + 1 + isNullSize := uintptr(maxRow*colCount) + 1 + totalSize := fixedSize + dataSize + isNullSize + _ = totalSize + bindVPointer := unsafe.Pointer(C.calloc(C.size_t(fixedSize), 1)) + dataPointer := unsafe.Pointer(C.calloc(C.size_t(dataSize), 1)) + isNullPointer := unsafe.Pointer(C.calloc(C.size_t(isNullSize), 1)) + + // bindV + bindV := (*C.TAOS_STMT2_BINDV)(bindVPointer) + // tableName + tableNamesPointer := unsafe.Pointer(uintptr(bindVPointer) + BindVSize) + bindV.tbnames = (**C.char)(tableNamesPointer) + + // bind_cols pointer + bindStructsPointerPointer := unsafe.Pointer(uintptr(tableNamesPointer) + uintptr(bindTables)*wrapper.PointerSize) + bindV.bind_cols = (**C.TAOS_STMT2_BIND)(bindStructsPointerPointer) + + // TAOS_STMT2_BIND + bindStructsPointer := unsafe.Pointer(uintptr(bindStructsPointerPointer) + uintptr(bindTables*colCount)*wrapper.PointerSize) + + tableNamesPointers := make([]unsafe.Pointer, bindTables) + bindPointers := make([][]unsafe.Pointer, bindTables) + var bind *C.TAOS_STMT2_BIND + for tableIndex := 0; tableIndex < bindTables; tableIndex++ { + // table name pointer + tableNamesPointers[tableIndex] = tableNamesPointer + tableNamesPointer = unsafe.Pointer(uintptr(tableNamesPointer) + wrapper.PointerSize) + bindPointers[tableIndex] = make([]unsafe.Pointer, colCount) + // set bind struct pointer + *(**C.TAOS_STMT2_BIND)(unsafe.Pointer(uintptr(bindStructsPointerPointer) + uintptr(tableIndex)*wrapper.PointerSize)) = (*C.TAOS_STMT2_BIND)(bindStructsPointer) + + for colIndex := 0; colIndex < colCount; colIndex++ { + // bind struct pointer + bindPointers[tableIndex][colIndex] = bindStructsPointer + // set col type + bind = (*C.TAOS_STMT2_BIND)(bindStructsPointer) + bind.buffer_type = (C.int)(colTypes[colIndex]) + bindStructsPointer = unsafe.Pointer(uintptr(bindStructsPointer) + BindSize) + } + } + + tableColDataPointers := make([]unsafe.Pointer, colCount) + isNullPointers := make([]unsafe.Pointer, colCount) + for colIndex := 0; colIndex < colCount; colIndex++ { + tableColDataPointers[colIndex] = dataPointer + dataPointer = unsafe.Pointer(uintptr(dataPointer) + uintptr(colSize[colIndex]*maxRow)) + isNullPointers[colIndex] = isNullPointer + isNullPointer = unsafe.Pointer(uintptr(isNullPointer) + uintptr(maxRow)) + } + + buffer := &bufferPointer{ + bindVP: bindVPointer, + tableNameP: tableNamesPointers, + bindsP: bindPointers, + colP: tableColDataPointers, + isNullP: isNullPointers, + } + return buffer +} + +func (p *processor) ProcessBatch(b targets.Batch, doLoad bool) (metricCount, rowCount uint64) { + batches := b.(*hypertableArr) + metricCnt := batches.totalMetric + if !doLoad { + return metricCnt, uint64(batches.cnt) + } + rowCount = uint64(batches.cnt) + if len(batches.createSql) > 0 { + p.createWg.Add(1) + go func() { + sqls := batches.createSql + p.cBufPosition = unsafe.Pointer(uintptr(p.cBuf) + 12) + p.bufIndex = 12 + for i := range sqls { + sh := (*reflect.SliceHeader)(unsafe.Pointer(sqls[i])) + createSize := sh.Len - 6 + if p.bufIndex+createSize >= Size1M { + *(*C.char)(p.cBufPosition) = C.char(0) + C.taos_query_a_wrapper2(p._db.TaosConnection, (*C.char)(p.cBuf), p.hp) + res := <-p.caller.QueryResult + if res.N != 0 { + panic(wrapper.TaosErrorStr(res.Res)) + } + go func(result unsafe.Pointer) { + C.taos_free_result(result) + }(res.Res) + p.cBufPosition = unsafe.Pointer(uintptr(p.cBuf) + 12) + p.bufIndex = 12 + } + C.memcpy(p.cBufPosition, unsafe.Pointer(sh.Data+6), C.size_t(createSize)) + p.cBufPosition = unsafe.Pointer(uintptr(p.cBufPosition) + uintptr(createSize)) + p.bufIndex += createSize + } + if p.bufIndex > 12 { + *(*C.char)(p.cBufPosition) = C.char(0) + C.taos_query_a_wrapper2(p._db.TaosConnection, (*C.char)(p.cBuf), p.hp) + res := <-p.caller.QueryResult + if res.N != 0 { + panic(wrapper.TaosErrorStr(res.Res)) + } + go func(result unsafe.Pointer) { + C.taos_free_result(result) + }(res.Res) + } + p.createWg.Done() + }() + } + + switch p.useCase { + case CpuCase: + if len(batches.data) > 0 { + var bind *C.TAOS_STMT2_BIND + hostTableIndex := 0 + hostCBuffers := p.cBuffers[CpuHandleIndex] + var nullByte byte + var currentRowData []byte + var dataPointer unsafe.Pointer + currentIsNullPointer := [11]unsafe.Pointer{ + hostCBuffers.isNullP[0], + hostCBuffers.isNullP[1], + hostCBuffers.isNullP[2], + hostCBuffers.isNullP[3], + hostCBuffers.isNullP[4], + hostCBuffers.isNullP[5], + hostCBuffers.isNullP[6], + hostCBuffers.isNullP[7], + hostCBuffers.isNullP[8], + hostCBuffers.isNullP[9], + hostCBuffers.isNullP[10], + } + currentDataPointer := [11]unsafe.Pointer{ + hostCBuffers.colP[0], + hostCBuffers.colP[1], + hostCBuffers.colP[2], + hostCBuffers.colP[3], + hostCBuffers.colP[4], + hostCBuffers.colP[5], + hostCBuffers.colP[6], + hostCBuffers.colP[7], + hostCBuffers.colP[8], + hostCBuffers.colP[9], + hostCBuffers.colP[10], + } + for i := 0; i < len(batches.data); i++ { + row := *(batches.data[i]) + slotID := p.tableSlot[SuperTableHost][*(*uint32)(unsafe.Pointer(&row[2]))] + p.hostBatchIndexer = append(p.hostBatchIndexer, slotID) + p.hostSlot[slotID] = append(p.hostSlot[slotID], row) + } + for _, slotID := range p.hostBatchIndexer { + rowData := p.hostSlot[slotID] + if len(rowData) == 0 { + continue + } + p.hostSlot[slotID] = p.hostSlot[slotID][:0] + rowLen := len(rowData) + for i := 0; i < len(rowData); i++ { + tmp := rowData[i] + currentRowData = tmp[7:len(tmp):len(tmp)] + nullByte = currentRowData[0] + dataPointer = unsafe.Pointer(¤tRowData[2]) + + *(*C.int64_t)(currentDataPointer[0]) = *(*C.int64_t)(dataPointer) + + currentDataPointer[0] = unsafe.Pointer(uintptr(currentDataPointer[0]) + 8) + dataPointer = unsafe.Pointer(uintptr(dataPointer) + 8) + + // set col data and is_null + for colIndex := 1; colIndex < 11; colIndex++ { + if colIndex == 8 { + nullByte = currentRowData[1] + } + isNullP := currentIsNullPointer[colIndex] + dataP := currentDataPointer[colIndex] + if nullByte&(1<<(7-(colIndex&7))) != 0 { + *(*C.char)(currentIsNullPointer[colIndex]) = C.char(0) + } else { + *(*C.char)(isNullP) = C.char(0) + *(*C.int64_t)(dataP) = *(*C.int64_t)(dataPointer) + } + + currentIsNullPointer[colIndex] = unsafe.Pointer(uintptr(isNullP) + 1) + currentDataPointer[colIndex] = unsafe.Pointer(uintptr(dataP) + 8) + dataPointer = unsafe.Pointer(uintptr(dataPointer) + 8) + } + + // first row, set buffer and is_null pointer + if i == 0 { + for colIndex := 0; colIndex < 11; colIndex++ { + bind = (*C.TAOS_STMT2_BIND)(hostCBuffers.bindsP[hostTableIndex][colIndex]) + bind.buffer = unsafe.Pointer(uintptr(currentDataPointer[colIndex]) - 8) + bind.is_null = (*C.char)(unsafe.Pointer(uintptr(currentIsNullPointer[colIndex]) - 1)) + bind.num = (C.int)(rowLen) + } + } + } + *(**C.char)(hostCBuffers.tableNameP[hostTableIndex]) = (*C.char)(p.tableNamePointerCache[SuperTableHost][slotID]) + hostTableIndex += 1 + } + p.hostBatchIndexer = p.hostBatchIndexer[:0] + + // stmt2 bind + bindv := (*C.TAOS_STMT2_BINDV)(hostCBuffers.bindVP) + bindv.count = C.int(hostTableIndex) + handler := p.stmt2CHandle[CpuHandleIndex] + if len(batches.createSql) > 0 { + p.createWg.Wait() + } + code := int(C.taos_stmt2_bind_param(handler, bindv, C.int32_t(-1))) + if code != 0 { + errStr := wrapper.TaosStmt2Error(handler) + panic(fmt.Errorf("failed to bind param stmt2: %d:%s", code, errStr)) + } + code = wrapper.TaosStmt2Exec(handler) + if code != 0 { + errStr := wrapper.TaosStmt2Error(handler) + panic(fmt.Errorf("failed to exec stmt2: %d:%s", code, errStr)) + } + atomic.AddInt32(&p.hostExecCount, 1) + } + + case IoTCase: + var row []byte + for i := 0; i < len(batches.data); i++ { + row = *batches.data[i] + if row[1] == SuperTableReadings { + slotID := p.tableSlot[SuperTableReadings][*(*uint32)(unsafe.Pointer(&row[2]))] + p.readingsBatchIndexer = append(p.readingsBatchIndexer, slotID) + p.readingsSlot[slotID] = append(p.readingsSlot[slotID], row) + } else { + slotID := p.tableSlot[SuperTableDiagnostics][*(*uint32)(unsafe.Pointer(&row[2]))] + p.diagnosticsBatchIndexer = append(p.diagnosticsBatchIndexer, slotID) + p.diagnosticsSlot[slotID] = append(p.diagnosticsSlot[slotID], row) + } + } + p.wg.Add(2) + go func() { + if len(p.readingsBatchIndexer) > 0 { + var bind *C.TAOS_STMT2_BIND + readingTableIndex := 0 + readingCBuffers := p.cBuffers[ReadingsHandleIndex] + var nullByte byte + var currentRowData []byte + var dataPointer unsafe.Pointer + currentIsNullPointer := [8]unsafe.Pointer{ + readingCBuffers.isNullP[0], + readingCBuffers.isNullP[1], + readingCBuffers.isNullP[2], + readingCBuffers.isNullP[3], + readingCBuffers.isNullP[4], + readingCBuffers.isNullP[5], + readingCBuffers.isNullP[6], + readingCBuffers.isNullP[7], + } + currentDataPointer := [8]unsafe.Pointer{ + readingCBuffers.colP[0], + readingCBuffers.colP[1], + readingCBuffers.colP[2], + readingCBuffers.colP[3], + readingCBuffers.colP[4], + readingCBuffers.colP[5], + readingCBuffers.colP[6], + readingCBuffers.colP[7], + } + + for _, slotID := range p.readingsBatchIndexer { + rowData := p.readingsSlot[slotID] + if len(rowData) == 0 { + continue + } + p.readingsSlot[slotID] = p.readingsSlot[slotID][:0] + rowLen := len(rowData) + for i := 0; i < len(rowData); i++ { + tmp := rowData[i] + currentRowData = tmp[7:len(tmp):len(tmp)] + nullByte = currentRowData[0] + dataPointer = unsafe.Pointer(¤tRowData[1]) + + *(*C.int64_t)(currentDataPointer[0]) = *(*C.int64_t)(dataPointer) + + currentDataPointer[0] = unsafe.Pointer(uintptr(currentDataPointer[0]) + 8) + dataPointer = unsafe.Pointer(uintptr(dataPointer) + 8) + + for colIndex := 1; colIndex < 8; colIndex++ { + if nullByte&(1<<(7-colIndex)) != 0 { + *(*C.char)(currentIsNullPointer[colIndex]) = C.char(1) + } else { + *(*C.char)(currentIsNullPointer[colIndex]) = C.char(0) + *(*C.int64_t)(currentDataPointer[colIndex]) = *(*C.int64_t)(dataPointer) + } + + currentIsNullPointer[colIndex] = unsafe.Pointer(uintptr(currentIsNullPointer[colIndex]) + 1) + currentDataPointer[colIndex] = unsafe.Pointer(uintptr(currentDataPointer[colIndex]) + 8) + dataPointer = unsafe.Pointer(uintptr(dataPointer) + 8) + } + + if i == 0 { + // first row set buffer and is_null pointer + for colIndex := 0; colIndex < 8; colIndex++ { + bind = (*C.TAOS_STMT2_BIND)(readingCBuffers.bindsP[readingTableIndex][colIndex]) + bind.buffer = unsafe.Pointer(uintptr(currentDataPointer[colIndex]) - 8) + bind.is_null = (*C.char)(unsafe.Pointer(uintptr(currentIsNullPointer[colIndex]) - 1)) + bind.num = (C.int)(rowLen) + } + } + } + *(**C.char)(readingCBuffers.tableNameP[readingTableIndex]) = (*C.char)(p.tableNamePointerCache[SuperTableReadings][slotID]) + readingTableIndex += 1 + } + p.readingsBatchIndexer = p.readingsBatchIndexer[:0] + bindv := (*C.TAOS_STMT2_BINDV)(readingCBuffers.bindVP) + bindv.count = C.int(readingTableIndex) + handler := p.stmt2CHandle[ReadingsHandleIndex] + + if len(batches.createSql) > 0 { + p.createWg.Wait() + } + code := int(C.taos_stmt2_bind_param(handler, bindv, C.int32_t(-1))) + if code != 0 { + errStr := wrapper.TaosStmt2Error(handler) + panic(fmt.Errorf("failed to bind param stmt2: %d:%s", code, errStr)) + } + code = wrapper.TaosStmt2Exec(handler) + if code != 0 { + errStr := wrapper.TaosStmt2Error(handler) + panic(fmt.Errorf("failed to exec stmt2: %d:%s", code, errStr)) + } + atomic.AddInt32(&p.readingsExecCount, 1) + } + p.wg.Done() + }() + go func() { + if len(p.diagnosticsBatchIndexer) > 0 { + var bind *C.TAOS_STMT2_BIND + diagnosticsTableIndex := 0 + diagnosticsCBuffers := p.cBuffers[DiagnosticsHandleIndex] + var nullByte byte + var currentRowData []byte + var dataPointer unsafe.Pointer + currentIsNullPointer := [4]unsafe.Pointer{ + diagnosticsCBuffers.isNullP[0], + diagnosticsCBuffers.isNullP[1], + diagnosticsCBuffers.isNullP[2], + diagnosticsCBuffers.isNullP[3], + } + currentDataPointer := [4]unsafe.Pointer{ + diagnosticsCBuffers.colP[0], + diagnosticsCBuffers.colP[1], + diagnosticsCBuffers.colP[2], + diagnosticsCBuffers.colP[3], + } + + for _, slotID := range p.diagnosticsBatchIndexer { + rowData := p.diagnosticsSlot[slotID] + if len(rowData) == 0 { + continue + } + p.diagnosticsSlot[slotID] = p.diagnosticsSlot[slotID][:0] + rowLen := len(rowData) + for i := 0; i < len(rowData); i++ { + tmp := rowData[i] + currentRowData = tmp[7:len(tmp):len(tmp)] + nullByte = currentRowData[0] + dataPointer = unsafe.Pointer(¤tRowData[1]) + + *(*C.int64_t)(currentDataPointer[0]) = *(*C.int64_t)(dataPointer) + + currentDataPointer[0] = unsafe.Pointer(uintptr(currentDataPointer[0]) + 8) + dataPointer = unsafe.Pointer(uintptr(dataPointer) + 8) + + // col1 + if nullByte&(1<<6) != 0 { + *(*C.char)(currentIsNullPointer[1]) = C.char(1) + } else { + *(*C.char)(currentIsNullPointer[1]) = C.char(0) + *(*C.double)(currentDataPointer[1]) = *(*C.double)(dataPointer) + } + currentIsNullPointer[1] = unsafe.Pointer(uintptr(currentIsNullPointer[1]) + 1) + currentDataPointer[1] = unsafe.Pointer(uintptr(currentDataPointer[1]) + 8) + dataPointer = unsafe.Pointer(uintptr(dataPointer) + 8) + + // col2 + if nullByte&(1<<5) != 0 { + *(*C.char)(currentIsNullPointer[2]) = C.char(1) + } else { + *(*C.char)(currentIsNullPointer[2]) = C.char(0) + *(*C.double)(currentDataPointer[2]) = *(*C.double)(dataPointer) + } + currentIsNullPointer[2] = unsafe.Pointer(uintptr(currentIsNullPointer[2]) + 1) + currentDataPointer[2] = unsafe.Pointer(uintptr(currentDataPointer[2]) + 8) + dataPointer = unsafe.Pointer(uintptr(dataPointer) + 8) + + // col3 + if nullByte&(1<<4) != 0 { + *(*C.char)(currentIsNullPointer[3]) = C.char(1) + } else { + *(*C.char)(currentIsNullPointer[3]) = C.char(0) + *(*C.int64_t)(currentDataPointer[3]) = *(*C.int64_t)(dataPointer) + } + currentIsNullPointer[3] = unsafe.Pointer(uintptr(currentIsNullPointer[3]) + 1) + currentDataPointer[3] = unsafe.Pointer(uintptr(currentDataPointer[3]) + 8) + if i == 0 { + // first row set buffer and is_null pointer + for colIndex := 0; colIndex < 4; colIndex++ { + bind = (*C.TAOS_STMT2_BIND)(diagnosticsCBuffers.bindsP[diagnosticsTableIndex][colIndex]) + bind.buffer = unsafe.Pointer(uintptr(currentDataPointer[colIndex]) - 8) + bind.is_null = (*C.char)(unsafe.Pointer(uintptr(currentIsNullPointer[colIndex]) - 1)) + bind.num = (C.int)(rowLen) + } + } + } + *(**C.char)(diagnosticsCBuffers.tableNameP[diagnosticsTableIndex]) = (*C.char)(p.tableNamePointerCache[SuperTableDiagnostics][slotID]) + diagnosticsTableIndex += 1 + } + p.diagnosticsBatchIndexer = p.diagnosticsBatchIndexer[:0] + bindv := (*C.TAOS_STMT2_BINDV)(diagnosticsCBuffers.bindVP) + bindv.count = C.int(diagnosticsTableIndex) + handler := p.stmt2CHandle[DiagnosticsHandleIndex] + if len(batches.createSql) > 0 { + p.createWg.Wait() + } + code := int(C.taos_stmt2_bind_param(handler, bindv, C.int32_t(-1))) + if code != 0 { + errStr := wrapper.TaosStmt2Error(handler) + panic(fmt.Errorf("failed to bind param stmt2: %d:%s", code, errStr)) + } + code = wrapper.TaosStmt2Exec(handler) + if code != 0 { + errStr := wrapper.TaosStmt2Error(handler) + panic(fmt.Errorf("failed to exec stmt2: %d:%s", code, errStr)) + } + atomic.AddInt32(&p.diagnosticsExecCount, 1) + } + + p.wg.Done() + }() + p.wg.Wait() + } + go func() { + globalSlicePool.Put(batches.data) + }() + return metricCnt, rowCount +} + +func (p *processor) Close(doLoad bool) { + if doLoad { + close(p.exitSign) + p.finishWg.Wait() + } +} diff --git a/pkg/targets/tdenginestmt2/program_options.go b/pkg/targets/tdenginestmt2/program_options.go new file mode 100644 index 000000000..cd8664c2f --- /dev/null +++ b/pkg/targets/tdenginestmt2/program_options.go @@ -0,0 +1,5 @@ +package tdenginestmt2 + +import "github.com/timescale/tsbs/pkg/targets/tdengine" + +type LoadingOptions tdengine.LoadingOptions diff --git a/pkg/targets/tdenginestmt2/scan.go b/pkg/targets/tdenginestmt2/scan.go new file mode 100644 index 000000000..635905214 --- /dev/null +++ b/pkg/targets/tdenginestmt2/scan.go @@ -0,0 +1,207 @@ +package tdenginestmt2 + +import ( + "bytes" + "fmt" + "unsafe" + + "github.com/spaolacci/murmur3" + "github.com/timescale/tsbs/pkg/data" + "github.com/timescale/tsbs/pkg/targets" +) + +// Indexer is used to consistently send the same hostnames to the same worker +type Indexer struct { + cache [3][]uint +} + +func NewIndexer(prefix []byte, partitions int, hashEndGroups []uint32, useCase byte, scale uint32) (_ *Indexer, _ [3][]uint32, hostTableIndex [][]uint32, readingsTableIndex [][]uint32, diagnosticsTableIndex [][]uint32) { + cache := [3][]uint{} + buf := &bytes.Buffer{} + var idx uint32 + switch useCase { + case CpuCase: + cache[SuperTableHost] = make([]uint, scale+1) + hostTableIndex = make([][]uint32, partitions) + //partitionIndex := make([]uint32, partitions) + tableOffset := make([]uint32, scale+1) + buf.Write(prefix) + buf.WriteString("host_null") + hash := murmur3.Sum32WithSeed(buf.Bytes(), 0x12345678) + buf.Reset() + for j := 0; j < partitions; j++ { + if hash <= hashEndGroups[j] { + cache[SuperTableHost][0] = uint(j) + tableOffset[0] = uint32(len(hostTableIndex[j])) + hostTableIndex[j] = append(hostTableIndex[j], 0) + break + } + } + tbPrefix := append(prefix, []byte("host_")...) + for i := uint32(0); i < scale; i++ { + buf.Write(tbPrefix) + _, err := fmt.Fprintf(buf, "%d", i) + if err != nil { + panic(err) + } + hash = murmur3.Sum32WithSeed(buf.Bytes(), 0x12345678) + buf.Reset() + for j := 0; j < partitions; j++ { + if hash <= hashEndGroups[j] { + idx = i + 1 + cache[SuperTableHost][idx] = uint(j) + tableOffset[idx] = uint32(len(hostTableIndex[j])) + hostTableIndex[j] = append(hostTableIndex[j], idx) + break + } + } + } + return &Indexer{ + cache: cache, + }, + [3][]uint32{ + tableOffset, + }, + hostTableIndex, + nil, + nil + case IoTCase: + cache[SuperTableReadings] = make([]uint, scale+1) + readingsTableIndex = make([][]uint32, partitions) + rTableOffset := make([]uint32, scale+1) + buf.Write(prefix) + buf.WriteString("r_truck_null") + hash := murmur3.Sum32WithSeed(buf.Bytes(), 0x12345678) + buf.Reset() + for j := 0; j < partitions; j++ { + if hash <= hashEndGroups[j] { + cache[SuperTableReadings][0] = uint(j) + rTableOffset[0] = uint32(len(readingsTableIndex[j])) + readingsTableIndex[j] = append(readingsTableIndex[j], 0) + break + } + } + tbPrefix := append(prefix, []byte("r_truck_")...) + for i := uint32(0); i < scale; i++ { + buf.Write(tbPrefix) + _, err := fmt.Fprintf(buf, "%d", i) + if err != nil { + panic(err) + } + hash := murmur3.Sum32WithSeed(buf.Bytes(), 0x12345678) + buf.Reset() + for j := 0; j < partitions; j++ { + if hash <= hashEndGroups[j] { + idx = i + 1 + cache[SuperTableReadings][idx] = uint(j) + rTableOffset[idx] = uint32(len(readingsTableIndex[j])) + readingsTableIndex[j] = append(readingsTableIndex[j], idx) + break + } + } + } + + cache[SuperTableDiagnostics] = make([]uint, scale+1) + diagnosticsTableIndex = make([][]uint32, partitions) + dTableOffset := make([]uint32, scale+1) + buf.Write(prefix) + buf.WriteString("d_truck_null") + hash = murmur3.Sum32WithSeed(buf.Bytes(), 0x12345678) + buf.Reset() + for j := 0; j < partitions; j++ { + if hash <= hashEndGroups[j] { + cache[SuperTableDiagnostics][0] = uint(j) + dTableOffset[0] = uint32(len(diagnosticsTableIndex[j])) + diagnosticsTableIndex[j] = append(diagnosticsTableIndex[j], 0) + break + } + } + tbPrefix = append(prefix, []byte("d_truck_")...) + for i := uint32(0); i < scale; i++ { + buf.Write(tbPrefix) + _, err := fmt.Fprintf(buf, "%d", i) + if err != nil { + panic(err) + } + hash := murmur3.Sum32WithSeed(buf.Bytes(), 0x12345678) + buf.Reset() + for j := 0; j < partitions; j++ { + if hash <= hashEndGroups[j] { + idx = i + 1 + cache[SuperTableDiagnostics][idx] = uint(j) + dTableOffset[idx] = uint32(len(diagnosticsTableIndex[j])) + diagnosticsTableIndex[j] = append(diagnosticsTableIndex[j], idx) + break + } + } + } + + return &Indexer{ + cache: cache, + }, + [3][]uint32{ + nil, + rTableOffset, + dTableOffset, + }, + nil, + readingsTableIndex, + diagnosticsTableIndex + default: + panic(fmt.Sprintf("invalid use case: %d", useCase)) + } +} + +func (i *Indexer) GetIndex(item data.LoadedPoint) uint { + p := *item.Data.(*[]byte) + return i.cache[p[1]][*(*uint32)(unsafe.Pointer(&p[2]))] +} + +type hypertableArr struct { + data []*[]byte + createSql []*[]byte + totalMetric uint64 + cnt uint +} + +func (ha *hypertableArr) Len() uint { + return ha.cnt +} + +func (ha *hypertableArr) Append(item data.LoadedPoint) { + p := item.Data.(*[]byte) + s := *p + _ = s[7] + if s[0] == InsertData { + if s[6] != 1 { + ha.data = append(ha.data, p) + } + switch s[1] { + case SuperTableHost: + ha.totalMetric += 10 + case SuperTableReadings: + ha.totalMetric += 7 + case SuperTableDiagnostics: + ha.totalMetric += 3 + default: + fatal("invalid table type:%d", s[1]) + } + ha.cnt++ + } else { + ha.createSql = append(ha.createSql, p) + } +} + +type BatchFactory struct { + batchSize uint +} + +func (b *BatchFactory) New() targets.Batch { + return &hypertableArr{ + data: make([]*[]byte, 0, b.batchSize), + } +} + +func NewBatchFactory() targets.BatchFactory { + return &BatchFactory{} +} diff --git a/pkg/targets/tdenginestmt2/serializer.go b/pkg/targets/tdenginestmt2/serializer.go new file mode 100644 index 000000000..a6ad5a6e6 --- /dev/null +++ b/pkg/targets/tdenginestmt2/serializer.go @@ -0,0 +1,558 @@ +package tdenginestmt2 + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "log" + "math" + "strconv" + "strings" + "unsafe" + + "github.com/timescale/tsbs/pkg/data" + "github.com/timescale/tsbs/pkg/data/usecases/common" +) + +const ( + SuperTableHost = 0 + SuperTableReadings = 1 + SuperTableDiagnostics = 2 +) + +type Serializer struct { + tmpBuf *bytes.Buffer + writeBuf *bytes.Buffer + useCase byte + scale uint32 + cpuTableList []int64 + readingsTableList []int64 + diagnosticsTableList []int64 + superTable [3]*Table +} + +type Table struct { + sortColumns map[string]int + sortTags map[string]int + tagBase []string + nullBit []byte + colOffset []int + colBufferLength int +} + +func FastFormat(buf *bytes.Buffer, v interface{}) { + switch v := v.(type) { + case int: + buf.WriteString(strconv.Itoa(v)) + case int64: + buf.WriteString(strconv.FormatInt(v, 10)) + case float64: + buf.WriteString(strconv.FormatFloat(v, 'f', -1, 64)) + case float32: + buf.WriteString(strconv.FormatFloat(float64(v), 'f', -1, 32)) + case bool: + buf.WriteString(strconv.FormatBool(v)) + case []byte: + buf.WriteByte('\'') + buf.Write(v) + buf.WriteByte('\'') + case string: + buf.WriteByte('\'') + buf.WriteString(v) + buf.WriteByte('\'') + case nil: + buf.WriteString("null") + default: + panic(fmt.Sprintf("unknown field type for %#v", v)) + } +} + +type tbNameRule struct { + tag string + prefix string + nilValue string +} + +var tbRule = []*tbNameRule{ + { + tag: "hostname", + nilValue: "host_null", + }, + { + tag: "name", + prefix: "r_", + nilValue: "r_truck_null", + }, + { + tag: "name", + prefix: "d_", + nilValue: "d_truck_null", + }, +} + +const ( + CpuCase = 1 + IoTCase = 2 +) + +/* + fixed header + +| version(1 byte) | case (1 byte) | scale (4 bytes) + +*/ + +func (s *Serializer) Config(config *common.DataGeneratorConfig, w io.Writer) error { + if config.Scale > math.MaxUint32-1 { + return fmt.Errorf("scale %d is too large", config.Scale) + } + s.scale = uint32(config.Scale) + useCase := s.prepare(config.BaseConfig.Use) + if useCase == 0 { + return fmt.Errorf("use case %s not supported", config.BaseConfig.Use) + } + bs := make([]byte, 6) + bs[0] = 1 // version + bs[1] = useCase + binary.LittleEndian.PutUint32(bs[2:], s.scale) + _, err := w.Write(bs) + if err != nil { + return err + } + s.useCase = useCase + switch useCase { + case CpuCase: + s.cpuTableList = make([]int64, config.BaseConfig.Scale+1) + case IoTCase: + s.readingsTableList = make([]int64, config.BaseConfig.Scale+1) + s.diagnosticsTableList = make([]int64, config.BaseConfig.Scale+1) + default: + return fmt.Errorf("use case %s not supported", config.BaseConfig.Use) + } + return nil +} + +var cpuTable = &Table{ + sortColumns: map[string]int{ + "usage_user": 1, + "usage_system": 2, + "usage_idle": 3, + "usage_nice": 4, + "usage_iowait": 5, + "usage_irq": 6, + "usage_softirq": 7, + "usage_steal": 8, + "usage_guest": 9, + "usage_guest_nice": 10, + }, + sortTags: map[string]int{ + "hostname": 0, + "region": 1, + "datacenter": 2, + "rack": 3, + "os": 4, + "arch": 5, + "team": 6, + "service": 7, + "service_version": 8, + "service_environment": 9, + }, + tagBase: []string{ + "null", + "null", + "null", + "null", + "null", + "null", + "null", + "null", + "null", + "null", + }, + nullBit: []byte{0b01111111, 0b11100000}, + colOffset: []int{ + 0, + 8, + 16, + 24, + 32, + 40, + 48, + 56, + 64, + 72, + 80, + }, + colBufferLength: 88, +} +var readingsTable = &Table{ + sortColumns: map[string]int{ + "latitude": 1, + "longitude": 2, + "elevation": 3, + "velocity": 4, + "heading": 5, + "grade": 6, + "fuel_consumption": 7, + }, + sortTags: map[string]int{ + "name": 0, + "fleet": 1, + "driver": 2, + "model": 3, + "device_version": 4, + "load_capacity": 5, + "fuel_capacity": 6, + "nominal_fuel_consumption": 7, + }, + tagBase: []string{ + "null", + "null", + "null", + "null", + "null", + "null", + "null", + "null", + }, + nullBit: []byte{0b01111111}, + colOffset: []int{ + 0, + 8, + 16, + 24, + 32, + 40, + 48, + 56, + }, + colBufferLength: 64, +} +var diagnosticsTable = &Table{ + sortColumns: map[string]int{ + "fuel_state": 1, + "current_load": 2, + "status": 3, + }, + sortTags: map[string]int{ + "name": 0, + "fleet": 1, + "driver": 2, + "model": 3, + "device_version": 4, + "load_capacity": 5, + "fuel_capacity": 6, + "nominal_fuel_consumption": 7, + }, + tagBase: []string{ + "null", + "null", + "null", + "null", + "null", + "null", + "null", + "null", + }, + nullBit: []byte{0b01110000}, + colOffset: []int{ + 0, + 8, + 16, + 24, + }, + colBufferLength: 32, +} + +func (s *Serializer) prepare(use string) uint8 { + switch use { + case common.UseCaseCPUOnly, common.UseCaseCPUSingle: + s.superTable[SuperTableHost] = cpuTable + return CpuCase + case common.UseCaseIoT: + s.superTable[SuperTableReadings] = readingsTable + s.superTable[SuperTableDiagnostics] = diagnosticsTable + return IoTCase + default: + return 0 + } +} + +const ( + CreateTable = 1 + InsertData = 2 +) + +/* + + create table sql + | length (1 or 2 byte) + | type (1 byte,1) | table type(1 byte) | table index (uint32 4 bytes) | sql buffer | +*/ + +/* + insert data + | length (1 byte) + | type (1 byte,2) | table type(1 byte) | table index (uint32 4 bytes) + | duplicate (bool 1 byte)| + | is null bit | column data| +*/ + +func (s *Serializer) Serialize(p *data.Point, w io.Writer) error { + tmpBuf := s.tmpBuf + tagRawKeys := p.TagKeys() + tagRawValues := p.TagValues() + colRawKeys := p.FieldKeys() + colRawValues := p.FieldValues() + superTable := p.MeasurementName() + + var tableList []int64 + superTableType := byte(0) + switch superTable[0] { + case 'c': + superTableType = SuperTableHost + tableList = s.cpuTableList + case 'r': + superTableType = SuperTableReadings + tableList = s.readingsTableList + case 'd': + superTableType = SuperTableDiagnostics + tableList = s.diagnosticsTableList + default: + log.Fatalf("super table: %s not found", superTable) + } + + // generate sub table name + rule := tbRule[superTableType] + fixedName := "" + for index, value := range tagRawValues { + if len(fixedName) == 0 && BytesToString(tagRawKeys[index]) == rule.tag { + str, is := value.(string) + if is { + fixedName = str + } + } + } + + subTable := "" + tbNameIndex := uint32(0) + + if len(fixedName) != 0 { + if len(rule.prefix) == 0 { + subTable = fixedName + } else { + tmpBuf.WriteString(rule.prefix) + tmpBuf.WriteString(fixedName) + subTable = tmpBuf.String() + tmpBuf.Reset() + } + sl := strings.Split(fixedName, "_") + id, err := strconv.ParseInt(sl[len(sl)-1], 10, 32) + if err != nil { + log.Fatalf("table: %s id parse failed", subTable) + } + if id < 0 { + log.Fatalf("table: %s id is negative", subTable) + } + if id > int64(s.scale) { + log.Fatalf("table: %s id is too large", subTable) + } + tbNameIndex = uint32(id + 1) + } else { + // use nil table name + subTable = rule.nilValue + tbNameIndex = 0 + } + + stable := s.superTable[superTableType] + + // check sub table exist + lastTs := tableList[tbNameIndex] + if lastTs == 0 { + var tagValues = make([]string, len(stable.sortTags)) + copy(tagValues, stable.tagBase) + for index, key := range tagRawKeys { + tagIndex, exist := stable.sortTags[BytesToString(key)] + if !exist { + log.Fatalf("stable: %s tag: %s not found", superTable, key) + } + tmpBuf.Reset() + FastFormat(tmpBuf, tagRawValues[index]) + tagValues[tagIndex] = tmpBuf.String() + } + tmpBuf.Reset() + for index, value := range tagValues { + tmpBuf.WriteString(value) + if index != len(tagValues)-1 { + tmpBuf.WriteByte(',') + } + } + //|length | type (1 byte,1) | table type(1 byte) | table index (uint32 4 bytes) | sql buffer | + + sqlBuf := fmt.Sprintf(" %s using %s tags (%s)", subTable, superTable, tmpBuf.Bytes()) + s.writeBuf.Reset() + tmpBuf.Reset() + length := 6 + len(sqlBuf) + + if length < 128 { + s.writeBuf.WriteByte(byte(length)) + } else { + s.writeBuf.WriteByte(byte(length&0x7f | 0x80)) + s.writeBuf.WriteByte(byte(length >> 7)) + } + + // type + s.writeBuf.WriteByte(CreateTable) + // table type + s.writeBuf.WriteByte(superTableType) + // table index + bs := make([]byte, 4) + binary.LittleEndian.PutUint32(bs, tbNameIndex) + s.writeBuf.Write(bs) + // sql length + s.writeBuf.WriteString(sqlBuf) + _, err := w.Write(s.writeBuf.Bytes()) + if err != nil { + return err + } + } + nullBit := make([]byte, len(stable.nullBit)) + copy(nullBit, stable.nullBit) + colBuffer := make([]byte, stable.colBufferLength) + ts := p.TimestampInUnixMs() + duplicate := false + tableList[tbNameIndex] = ts + if ts <= lastTs { + duplicate = true + } + binary.LittleEndian.PutUint64(colBuffer, uint64(ts)) + for index, key := range colRawKeys { + colIndex, exist := stable.sortColumns[BytesToString(key)] + if !exist { + log.Fatalf("stable: %s column: %s not found", superTable, key) + } + value := colRawValues[index] + if value == nil { + continue + } + switch v := value.(type) { + case int: + binary.LittleEndian.PutUint64(colBuffer[stable.colOffset[colIndex]:], uint64(v)) + case int64: + binary.LittleEndian.PutUint64(colBuffer[stable.colOffset[colIndex]:], uint64(v)) + case float64: + binary.LittleEndian.PutUint64(colBuffer[stable.colOffset[colIndex]:], math.Float64bits(v)) + case float32: + binary.LittleEndian.PutUint64(colBuffer[stable.colOffset[colIndex]:], math.Float64bits(float64(v))) + default: + log.Fatalf("stable: %s column: %s type: %T not supported", superTable, key, v) + } + pos := CharOffset(colIndex) + nullBit[pos] = BMUnSetNull(nullBit[pos], colIndex) + } + //insert data + //| length (1 byte) + //| type (1 byte,2) | table type(1 byte) | table index (uint32 4 bytes) + //| duplicate (bool 1 byte)| + //| is null bit | column data| + s.writeBuf.Reset() + length := 7 + len(nullBit) + len(colBuffer) + if length >= 128 { + log.Fatalf("length %d is too large", length) + } + s.writeBuf.WriteByte(byte(length)) + // type + s.writeBuf.WriteByte(InsertData) + // table type + s.writeBuf.WriteByte(superTableType) + // table index + bs := make([]byte, 4) + binary.LittleEndian.PutUint32(bs, tbNameIndex) + s.writeBuf.Write(bs) + // duplicate + if duplicate { + s.writeBuf.WriteByte(1) + } else { + s.writeBuf.WriteByte(0) + } + // is null bit + s.writeBuf.Write(nullBit) + // column data + s.writeBuf.Write(colBuffer) + _, err := w.Write(s.writeBuf.Bytes()) + return err +} + +func BytesToString(b []byte) string { + return *(*string)(unsafe.Pointer(&b)) +} + +func BitPos(n int) int { + return n & (7) +} + +func CharOffset(n int) int { + return n >> 3 +} + +func BMUnSetNull(c byte, n int) byte { + return c - (1 << (7 - BitPos(n))) +} + +const CpuSql = "create stable cpu (" + + "ts timestamp," + + "usage_user bigint," + + "usage_system bigint," + + "usage_idle bigint," + + "usage_nice bigint," + + "usage_iowait bigint," + + "usage_irq bigint," + + "usage_softirq bigint," + + "usage_steal bigint," + + "usage_guest bigint," + + "usage_guest_nice bigint) " + + "tags (" + + "hostname varchar(30)," + + "region varchar(30)," + + "datacenter varchar(30)," + + "rack varchar(30)," + + "os varchar(30)," + + "arch varchar(30)," + + "team varchar(30)," + + "service varchar(30)," + + "service_version varchar(30)," + + "service_environment varchar(30))" +const ReadingsSql = "CREATE STABLE `readings` (" + + "`ts` TIMESTAMP, " + + "`latitude` DOUBLE, " + + "`longitude` DOUBLE, " + + "`elevation` DOUBLE, " + + "`velocity` DOUBLE, " + + "`heading` DOUBLE, " + + "`grade` DOUBLE, " + + "`fuel_consumption` DOUBLE) " + + "TAGS (" + + "`name` VARCHAR(30), " + + "`fleet` VARCHAR(30), " + + "`driver` VARCHAR(30), " + + "`model` VARCHAR(30), " + + "`device_version` VARCHAR(30), " + + "`load_capacity` DOUBLE, " + + "`fuel_capacity` DOUBLE, " + + "`nominal_fuel_consumption` DOUBLE)" + +const DiagnosticsSql = "CREATE STABLE `diagnostics` (" + + "`ts` TIMESTAMP, " + + "`fuel_state` DOUBLE, " + + "`current_load` DOUBLE, " + + "`status` BIGINT) " + + "TAGS (" + + "`name` VARCHAR(30), " + + "`fleet` VARCHAR(30), " + + "`driver` VARCHAR(30), " + + "`model` VARCHAR(30), " + + "`device_version` VARCHAR(30), " + + "`load_capacity` DOUBLE, " + + "`fuel_capacity` DOUBLE, " + + "`nominal_fuel_consumption` DOUBLE)" diff --git a/pkg/targets/timescaledb/creator.go b/pkg/targets/timescaledb/creator.go index ca36181af..195a39be5 100644 --- a/pkg/targets/timescaledb/creator.go +++ b/pkg/targets/timescaledb/creator.go @@ -116,6 +116,7 @@ func (d *dbCreator) PostCreateDB(dbName string) error { } r = MustQuery(dbBench, checkTableQuery) } + return nil } } return nil diff --git a/scripts/full_cycle_minitest/full_cycle_minitest_tdengine.sh b/scripts/full_cycle_minitest/full_cycle_minitest_tdengine.sh new file mode 100755 index 000000000..64de2865a --- /dev/null +++ b/scripts/full_cycle_minitest/full_cycle_minitest_tdengine.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# showcases the ftsb 3 phases for TDengine +# - 1) data and query generation +# - 2) data loading/insertion +# - 3) query execution + +MAX_QUERIES=${MAX_QUERIES:-"1000"} + +mkdir -p /tmp/bulk_data + +# generate data +$GOPATH/bin/tsbs_generate_data --format TDengine --use-case cpu-only --scale 10 --seed 123 --file /tmp/bulk_data/tdengine_data + +# generate queries +$GOPATH/bin/tsbs_generate_queries --queries=${MAX_QUERIES} --format TDengine --use-case cpu-only --scale 10 --seed 123 --query-type lastpoint --file /tmp/bulk_data/tdengine_query_lastpoint +$GOPATH/bin/tsbs_generate_queries --queries=${MAX_QUERIES} --format TDengine --use-case cpu-only --scale 10 --seed 123 --query-type cpu-max-all-1 --file /tmp/bulk_data/tdengine_query_cpu-max-all-1 +$GOPATH/bin/tsbs_generate_queries --queries=${MAX_QUERIES} --format TDengine --use-case cpu-only --scale 10 --seed 123 --query-type high-cpu-1 --file /tmp/bulk_data/tdengine_query_high-cpu-1 + +# insert benchmark +$GOPATH/bin/tsbs_load_tdengine --db-name=benchmark --workers=1 --file=/tmp/bulk_data/tdengine_data --results-file="tdengine_load_results.json" + +# queries benchmark +$GOPATH/bin/tsbs_run_queries_tdengine --db-name=benchmark --workers=1 --max-queries=${MAX_QUERIES} --file=/tmp/bulk_data/tdengine_query_lastpoint --results-file="tdengine_query_lastpoint_results.json" +$GOPATH/bin/tsbs_run_queries_tdengine --db-name=benchmark --workers=1 --max-queries=${MAX_QUERIES} --file=/tmp/bulk_data/tdengine_query_cpu-max-all-1 --results-file="tdengine_query_cpu-max-all-1_results.json" +$GOPATH/bin/tsbs_run_queries_tdengine --db-name=benchmark --workers=1 --max-queries=${MAX_QUERIES} --file=/tmp/bulk_data/tdengine_query_high-cpu-1 --results-file="tdengine_query_high-cpu-1_results.json" \ No newline at end of file diff --git a/scripts/load/load_tdengine.sh b/scripts/load/load_tdengine.sh new file mode 100755 index 000000000..5c39deb2e --- /dev/null +++ b/scripts/load/load_tdengine.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Ensure loader is available +EXE_FILE_NAME_LOAD_DATA=${EXE_FILE_NAME_LOAD_DATA:-$(which tsbs_load_tdengine)} +if [[ -z "$EXE_FILE_NAME_LOAD_DATA" ]]; then + echo "tsbs_load_tdengine is not available. It is not specified explicitly and not found in \$PATH" + exit 1 +fi + +# Load parameters - common +DATA_FILE_NAME=${DATA_FILE_NAME:-TDengine-data.gz} +DATABASE_NAME=${DATABASE_NAME:-benchmark} +DATABASE_HOST=${DATABASE_HOST:-localhost} +DATABASE_TAOS_PORT=${DATABASE_TAOS_PORT:-6030} +DATABASE_TAOS_PWD=${DATABASE_TAOS_PWD:-taosdata} + +# Load parameters - personal +VGROUPS=${VGROUPS:-"2"} +BUFFER=${BUFFER:-"256"} +PAGES=${PAGES:-"256"} +TRIGGER=${TRIGGER:-"1"} +WALFSYNCPERIOD=${WALFSYNCPERIOD:-"3000"} +WAL_LEVEL=${WAL_LEVEL:-"1"} +HASH_WORKERS=${HASH_WORKERS:-false} + +EXE_DIR=${EXE_DIR:-$(dirname $0)} +source ${EXE_DIR}/load_common.sh + +cat ${DATA_FILE} | gunzip | $EXE_FILE_NAME_LOAD_DATA \ + --db-name=${DATABASE_NAME} \ + --host=${DATABASE_HOST} \ + --port=${DATABASE_TAOS_PORT} \ + --pass=${DATABASE_TAOS_PWD} \ + --workers=${NUM_WORKERS} \ + --batch-size=${BATCH_SIZE} \ + --vgroups=${VGROUPS} \ + --buffer=${BUFFER} \ + --pages=${PAGES} \ + --hash-workers=${HASH_WORKERS} \ + --stt_trigger=${TRIGGER} \ + --wal_level=${WAL_LEVEL} \ + --wal_fsync_period=${WALFSYNCPERIOD} diff --git a/scripts/load/load_tdenginestmt2.sh b/scripts/load/load_tdenginestmt2.sh new file mode 100755 index 000000000..531c98ae2 --- /dev/null +++ b/scripts/load/load_tdenginestmt2.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Ensure loader is available +EXE_FILE_NAME_LOAD_DATA=${EXE_FILE_NAME_LOAD_DATA:-$(which tsbs_load_tdenginestmt2)} +if [[ -z "$EXE_FILE_NAME_LOAD_DATA" ]]; then + echo "tsbs_load_tdenginestmt2 is not available. It is not specified explicitly and not found in \$PATH" + exit 1 +fi + +# Load parameters - common +DATA_FILE_NAME=${DATA_FILE_NAME:-TDengineStmt2-data.gz} +DATABASE_NAME=${DATABASE_NAME:-benchmark} +DATABASE_HOST=${DATABASE_HOST:-localhost} +DATABASE_TAOS_PORT=${DATABASE_TAOS_PORT:-6030} +DATABASE_TAOS_PWD=${DATABASE_TAOS_PWD:-taosdata} + +# Load parameters - personal +VGROUPS=${VGROUPS:-"2"} +BUFFER=${BUFFER:-"256"} +PAGES=${PAGES:-"256"} +TRIGGER=${TRIGGER:-"1"} +WALFSYNCPERIOD=${WALFSYNCPERIOD:-"3000"} +WAL_LEVEL=${WAL_LEVEL:-"1"} +HASH_WORKERS=${HASH_WORKERS:-false} + +EXE_DIR=${EXE_DIR:-$(dirname $0)} +source ${EXE_DIR}/load_common.sh + +cat ${DATA_FILE} | gunzip | $EXE_FILE_NAME_LOAD_DATA \ + --db-name=${DATABASE_NAME} \ + --host=${DATABASE_HOST} \ + --port=${DATABASE_TAOS_PORT} \ + --pass=${DATABASE_TAOS_PWD} \ + --workers=${NUM_WORKERS} \ + --batch-size=${BATCH_SIZE} \ + --vgroups=${VGROUPS} \ + --buffer=${BUFFER} \ + --pages=${PAGES} \ + --hash-workers=${HASH_WORKERS} \ + --stt_trigger=${TRIGGER} \ + --wal_level=${WAL_LEVEL} \ + --wal_fsync_period=${WALFSYNCPERIOD} diff --git a/scripts/run_queries/run_queries_tdengine.sh b/scripts/run_queries/run_queries_tdengine.sh new file mode 100755 index 000000000..d0bc0ba00 --- /dev/null +++ b/scripts/run_queries/run_queries_tdengine.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Ensure runner is available +EXE_FILE_NAME=${EXE_FILE_NAME:-$(which tsbs_run_queries_tdengine)} +if [[ -z "$EXE_FILE_NAME" ]]; then + echo "tsbs_run_queries_tdengine not available. It is not specified explicitly and not found in \$PATH" + exit 1 +fi + +# Default queries folder +BULK_DATA_DIR=${BULK_DATA_DIR:-"/tmp/bulk_queries"} +MAX_QUERIES=${MAX_QUERIES:-"0"} +# How many concurrent worker would run queries - match num of cores, or default to 4 +NUM_WORKERS=${NUM_WORKERS:-$(grep -c ^processor /proc/cpuinfo 2> /dev/null || echo 4)} + +# +# Run test for one file +# +function run_file() +{ + # $FULL_DATA_FILE_NAME: /full/path/to/file_with.ext + # $DATA_FILE_NAME: file_with.ext + # $DIR: /full/path/to + # $EXTENSION: ext + # NO_EXT_DATA_FILE_NAME: file_with + FULL_DATA_FILE_NAME=$1 + DATA_FILE_NAME=$(basename -- "${FULL_DATA_FILE_NAME}") + DIR=$(dirname "${FULL_DATA_FILE_NAME}") + EXTENSION="${DATA_FILE_NAME##*.}" + NO_EXT_DATA_FILE_NAME="${DATA_FILE_NAME%.*}" + + # Several options on how to name results file + #OUT_FULL_FILE_NAME="${DIR}/result_${DATA_FILE_NAME}" + OUT_FULL_FILE_NAME="${DIR}/result_${NO_EXT_DATA_FILE_NAME}.out" + #OUT_FULL_FILE_NAME="${DIR}/${NO_EXT_DATA_FILE_NAME}.out" + + if [ "${EXTENSION}" == "gz" ]; then + GUNZIP="gunzip" + else + GUNZIP="cat" + fi + + echo "Running ${DATA_FILE_NAME}" + cat $FULL_DATA_FILE_NAME \ + | $GUNZIP \ + | $EXE_FILE_NAME \ + --max-queries $MAX_QUERIES \ + --workers $NUM_WORKERS \ + | tee $OUT_FULL_FILE_NAME +} + +if [ "$#" -gt 0 ]; then + echo "Have $# files specified as params" + for FULL_DATA_FILE_NAME in "$@"; do + run_file $FULL_DATA_FILE_NAME + done +else + echo "Do not have any files specified - run from default queries folder as ${BULK_DATA_DIR}/queries_TDengine*" + for FULL_DATA_FILE_NAME in "${BULK_DATA_DIR}/queries_TDengine"*; do + run_file $FULL_DATA_FILE_NAME + done +fi