diff --git a/README.md b/README.md index 622c9fde0..476040b0f 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,13 @@ TSP is a Time Series Patterns search engine. It is a backend system behind the [ TSP is a distributed compute system implemented in Modern Scala. For more information, refer to [Documentation](https://clover-group.github.io/tsp/). +## Build +To compile TSP you need to have [github.token](https://docs.github.com/en/github/authenticating-to-github/keeping-your-account-and-data-secure/creating-a-personal-access-token) (requires the ```read:package``` grant) set in ~/.gitconfig : +``` +[github] + token = TOKEN_DATA +``` + ## Profiling ![YourKit logo](https://www.yourkit.com/images/yklogo.png) diff --git a/build.sbt b/build.sbt index 841313caf..96d41b39e 100644 --- a/build.sbt +++ b/build.sbt @@ -8,7 +8,6 @@ dockerUsername in Docker := Some("clovergrp") dockerUpdateLatest := true dockerAlias in Docker := dockerAlias.value.withTag(dockerAlias.value.tag.map(_.replace("+", "_"))) -// Flink currently does not work with Scala 2.12.8+ scalaVersion in ThisBuild := "2.12.10" resolvers in ThisBuild ++= Seq( "Apache Development Snapshot Repository" at "https://repository.apache.org/content/repositories/snapshots/", @@ -139,7 +138,7 @@ case "git.properties" => MergeStrategy.first lazy val runTask = taskKey[Unit]("App runner") //runTask := { -// (http/runMain ${TSP_LAUNCHER:-ru.itclover.tsp.http.Launcher} ${TSP_LAUNCHER_ARGS:-flink-local}) +// (http/runMain ${TSP_LAUNCHER:-ru.itclover.tsp.http.Launcher} ${TSP_LAUNCHER_ARGS:-spark-local}) //} lazy val root = (project in file(".")) @@ -147,8 +146,8 @@ lazy val root = (project in file(".")) .settings(commonSettings) .settings(githubRelease := Utils.defaultGithubRelease.evaluated) - .aggregate(core, config, http, flink, spark, dsl, itValid) - .dependsOn(core, config, http, flink, spark, dsl, itValid) + .aggregate(core, config, http, spark, dsl, itValid) + .dependsOn(core, config, http, spark, dsl, itValid) lazy val core = project.in(file("core")) .settings(commonSettings) @@ -162,26 +161,18 @@ lazy val config = project.in(file("config")) .settings(commonSettings) .settings( buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion, - "flinkVersion" -> Version.flink, "sparkVersion" -> Version.spark), + "sparkVersion" -> Version.spark), buildInfoPackage := "ru.itclover.tsp" ) .dependsOn(core) -lazy val flink = project.in(file("flink")) - .settings(commonSettings) - .settings( - libraryDependencies ++= Library.flink ++ Library.scalaTest ++ Library.dbDrivers ++ Library.redisson ++ Library.logging, - dependencyOverrides += "com.fasterxml.jackson.core" % "jackson-core" % "2.10.0" - ) - .dependsOn(core, config, dsl) - lazy val http = project.in(file("http")) .settings(commonSettings) .settings( - libraryDependencies ++= Library.scalaTest ++ Library.flink ++ Library.akka ++ + libraryDependencies ++= Library.scalaTest ++ Library.akka ++ Library.akkaHttp ++ Library.sparkDeps ++ Library.logging ) - .dependsOn(core, config, flink, spark, dsl) + .dependsOn(core, config, spark, dsl) lazy val dsl = project.in(file("dsl")) .settings(commonSettings) @@ -193,7 +184,7 @@ lazy val dsl = project.in(file("dsl")) lazy val spark = project.in(file("spark")) .settings(commonSettings) .settings( - libraryDependencies ++= Library.scalaTest ++ Library.sparkDeps ++ Library.logging, + libraryDependencies ++= Library.scalaTest ++ Library.dbDrivers ++ Library.sparkDeps ++ Library.logging, dependencyOverrides += "com.fasterxml.jackson.core" % "jackson-core" % "2.10.0" ) .dependsOn(core, config, dsl) @@ -201,15 +192,15 @@ lazy val spark = project.in(file("spark")) lazy val itValid = project.in(file("integration/correctness")) .settings(commonSettings) .settings( - libraryDependencies ++= Library.flink ++ Library.scalaTest ++ Library.dbDrivers ++ Library.testContainers ++ Library.logging, + libraryDependencies ++= Library.scalaTest ++ Library.dbDrivers ++ Library.testContainers ++ Library.logging, dependencyOverrides += "com.fasterxml.jackson.core" % "jackson-databind" % "2.10.0" ) - .dependsOn(core, flink, http, config) + .dependsOn(core, http, config) lazy val itPerf = project.in(file("integration/performance")) .settings(commonSettings) .settings( - libraryDependencies ++= Library.flink ++ Library.scalaTest ++ Library.dbDrivers ++ Library.testContainers ++ Library.logging + libraryDependencies ++= Library.scalaTest ++ Library.dbDrivers ++ Library.testContainers ++ Library.logging ) .dependsOn(itValid) diff --git a/core/src/main/scala/ru/itclover/tsp/core/Incident.scala b/core/src/main/scala/ru/itclover/tsp/core/Incident.scala index eb9e4d448..49f9dd97a 100644 --- a/core/src/main/scala/ru/itclover/tsp/core/Incident.scala +++ b/core/src/main/scala/ru/itclover/tsp/core/Incident.scala @@ -15,12 +15,8 @@ case class Incident( patternId: Int, maxWindowMs: Long, segment: Segment, - @deprecated("Unused and scheduled to be removed in 0.17", since = "TSP 0.16") - forwardedFields: Seq[(String, String)], patternUnit: Int, patternSubunit: Int, - @deprecated("Unused and scheduled to be removed in 0.17", since = "TSP 0.16") - patternPayload: Seq[(String, String)] ) extends Product with Serializable @@ -39,10 +35,8 @@ object IncidentInstances { b.patternId, b.maxWindowMs, Segment(from, to), - b.forwardedFields, b.patternUnit, b.patternSubunit, - b.patternPayload ) } } diff --git a/core/src/main/scala/ru/itclover/tsp/core/RawPattern.scala b/core/src/main/scala/ru/itclover/tsp/core/RawPattern.scala index f2c0b15c4..f33d771a2 100644 --- a/core/src/main/scala/ru/itclover/tsp/core/RawPattern.scala +++ b/core/src/main/scala/ru/itclover/tsp/core/RawPattern.scala @@ -3,7 +3,5 @@ package ru.itclover.tsp.core case class RawPattern( id: Int, sourceCode: String, - payload: Option[Map[String, String]] = None, subunit: Option[Int] = None, - forwardedFields: Option[Seq[Symbol]] = None ) extends Serializable diff --git a/core/src/test/scala/ru/itclover/tsp/core/patterns/FoundedPatternTest.scala b/core/src/test/scala/ru/itclover/tsp/core/patterns/FoundedPatternTest.scala index 8490ab425..8d1dae09c 100644 --- a/core/src/test/scala/ru/itclover/tsp/core/patterns/FoundedPatternTest.scala +++ b/core/src/test/scala/ru/itclover/tsp/core/patterns/FoundedPatternTest.scala @@ -27,10 +27,8 @@ class FoundedPatternTest extends WordSpec with Matchers { patternId = 1, maxWindowMs = 1000, segment = firstTestSegment, - forwardedFields = Seq(("test1", "1"), ("test2", "2")), patternUnit = 13, patternSubunit = 42, - patternPayload = Seq(("test3", "3"), ("test4", "4")) ) val secondIncident = Incident( @@ -38,10 +36,8 @@ class FoundedPatternTest extends WordSpec with Matchers { patternId = 2, maxWindowMs = 4000, segment = secondTestSegment, - forwardedFields = Seq(("test1", "1"), ("test2", "2")), patternUnit = 13, patternSubunit = 42, - patternPayload = Seq(("test3", "3"), ("test4", "4")) ) val expectedIncident = Incident( @@ -52,10 +48,8 @@ class FoundedPatternTest extends WordSpec with Matchers { from = Time(1000), to = Time(4000) ), - forwardedFields = Seq(("test1", "1"), ("test2", "2")), patternUnit = 13, patternSubunit = 42, - patternPayload = Seq(("test3", "3"), ("test4", "4")) ) val actualIncident = IncidentInstances.semigroup.combine(firstIncident, secondIncident) diff --git a/flink/src/main/java/ru/itclover/tsp/DebugTsViolationHandler.java b/flink/src/main/java/ru/itclover/tsp/DebugTsViolationHandler.java deleted file mode 100644 index b4afa9dab..000000000 --- a/flink/src/main/java/ru/itclover/tsp/DebugTsViolationHandler.java +++ /dev/null @@ -1,17 +0,0 @@ -package ru.itclover.tsp; - -import org.apache.flink.streaming.api.functions.timestamps.AscendingTimestampExtractor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -public final class DebugTsViolationHandler implements AscendingTimestampExtractor.MonotonyViolationHandler { - private static final long serialVersionUID = 2L; - - private static final Logger LOG = LoggerFactory.getLogger(AscendingTimestampExtractor.class); - - @Override - public void handleViolation(long elementTimestamp, long lastTimestamp) { - LOG.debug("Timestamp monotony violated: {} < {}", elementTimestamp, lastTimestamp); - } -} diff --git a/flink/src/main/java/ru/itclover/tsp/JDBCInputFormatProps.java b/flink/src/main/java/ru/itclover/tsp/JDBCInputFormatProps.java deleted file mode 100644 index d4d338399..000000000 --- a/flink/src/main/java/ru/itclover/tsp/JDBCInputFormatProps.java +++ /dev/null @@ -1,358 +0,0 @@ -package ru.itclover.tsp; - -import org.apache.flink.annotation.VisibleForTesting; -import org.apache.flink.api.common.io.DefaultInputSplitAssigner; -import org.apache.flink.api.common.io.InputFormat; -import org.apache.flink.api.common.io.RichInputFormat; -import org.apache.flink.api.common.io.statistics.BaseStatistics; -import org.apache.flink.api.java.io.jdbc.split.ParameterValuesProvider; -import org.apache.flink.api.java.typeutils.ResultTypeQueryable; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.core.io.GenericInputSplit; -import org.apache.flink.core.io.InputSplit; -import org.apache.flink.core.io.InputSplitAssigner; -import org.apache.flink.types.Row; -import org.apache.flink.util.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.math.BigDecimal; -import java.sql.*; -import java.util.Arrays; -import java.util.Enumeration; -import java.util.Properties; - -public class JDBCInputFormatProps extends RichInputFormat implements ResultTypeQueryable { - - private static final long serialVersionUID = 1L; - private static final String USER_PROP = "user"; - private static final String PASSWORD_PROP = "password"; - private static final Logger LOG = LoggerFactory.getLogger(JDBCInputFormatProps.class); - - private String drivername; - private String dbURL; - private final Properties props = new Properties(); - private String queryTemplate; - private int resultSetType; - private int resultSetConcurrency; - private RowTypeInfo rowTypeInfo; - - private transient Connection dbConn; - private transient PreparedStatement statement; - private transient ResultSet resultSet; - private int fetchSize; - - private boolean hasNext; - private Object[][] parameterValues; - - public JDBCInputFormatProps() { - } - - @Override - public RowTypeInfo getProducedType() { - return rowTypeInfo; - } - - @Override - public void configure(Configuration parameters) { - //do nothing here - } - - @Override - public void openInputFormat() { - //called once per inputFormat (on open) - try { - Class.forName(drivername); - dbConn = DriverManager.getConnection(dbURL, props); - statement = dbConn.prepareStatement(queryTemplate, resultSetType, resultSetConcurrency); - if (fetchSize == Integer.MIN_VALUE || fetchSize > 0) { - statement.setFetchSize(fetchSize); - } - } catch (SQLException se) { - throw new IllegalArgumentException("open() failed." + se.getMessage(), se); - } catch (ClassNotFoundException cnfe) { - throw new IllegalArgumentException("JDBC-Class not found. - " + cnfe.getMessage(), cnfe); - } - } - - @Override - public void closeInputFormat() { - //called once per inputFormat (on close) - try { - if (statement != null) { - statement.close(); - } - } catch (SQLException se) { - LOG.info("Inputformat Statement couldn't be closed - " + se.getMessage()); - } finally { - statement = null; - } - - try { - if (dbConn != null) { - dbConn.close(); - } - } catch (SQLException se) { - LOG.info("Inputformat couldn't be closed - " + se.getMessage()); - } finally { - dbConn = null; - } - - parameterValues = null; - } - - /** - * Connects to the source database and executes the query in a parallel - * fashion if - * this {@link InputFormat} is built using a parameterized query (i.e. using - * a {@link PreparedStatement}) - * and a proper {@link ParameterValuesProvider}, in a non-parallel - * fashion otherwise. - * - * @param inputSplit which is ignored if this InputFormat is executed as a - * non-parallel source, - * a "hook" to the query parameters otherwise (using its - * splitNumber) - * @throws IOException if there's an error during the execution of the query - */ - @Override - public void open(InputSplit inputSplit) throws IOException { - try { - if (inputSplit != null && parameterValues != null) { - for (int i = 0; i < parameterValues[inputSplit.getSplitNumber()].length; i++) { - Object param = parameterValues[inputSplit.getSplitNumber()][i]; - if (param instanceof String) { - statement.setString(i + 1, (String) param); - } else if (param instanceof Long) { - statement.setLong(i + 1, (Long) param); - } else if (param instanceof Integer) { - statement.setInt(i + 1, (Integer) param); - } else if (param instanceof Double) { - statement.setDouble(i + 1, (Double) param); - } else if (param instanceof Boolean) { - statement.setBoolean(i + 1, (Boolean) param); - } else if (param instanceof Float) { - statement.setFloat(i + 1, (Float) param); - } else if (param instanceof BigDecimal) { - statement.setBigDecimal(i + 1, (BigDecimal) param); - } else if (param instanceof Byte) { - statement.setByte(i + 1, (Byte) param); - } else if (param instanceof Short) { - statement.setShort(i + 1, (Short) param); - } else if (param instanceof Date) { - statement.setDate(i + 1, (Date) param); - } else if (param instanceof Time) { - statement.setTime(i + 1, (Time) param); - } else if (param instanceof Timestamp) { - statement.setTimestamp(i + 1, (Timestamp) param); - } else if (param instanceof Array) { - statement.setArray(i + 1, (Array) param); - } else { - //extends with other types if needed - throw new IllegalArgumentException("open() failed. Parameter " + i + " of type " + param.getClass() + " is not handled (yet)."); - } - } - if (LOG.isDebugEnabled()) { - LOG.debug(String.format("Executing '%s' with parameters %s", queryTemplate, Arrays.deepToString(parameterValues[inputSplit.getSplitNumber()]))); - } - } - resultSet = statement.executeQuery(); - hasNext = resultSet.next(); - } catch (SQLException se) { - throw new IllegalArgumentException("open() failed." + se.getMessage(), se); - } - } - - /** - * Closes all resources used. - * - * @throws IOException Indicates that a resource could not be closed. - */ - @Override - public void close() throws IOException { - if (resultSet == null) { - return; - } - try { - resultSet.close(); - } catch (SQLException se) { - LOG.info("Inputformat ResultSet couldn't be closed - " + se.getMessage()); - } - } - - /** - * Checks whether all data has been read. - * - * @return boolean value indication whether all data has been read. - * @throws IOException - */ - @Override - public boolean reachedEnd() throws IOException { - return !hasNext; - } - - /** - * Stores the next resultSet row in a tuple. - * - * @param row row to be reused. - * @return row containing next {@link Row} - * @throws java.io.IOException - */ - @Override - public Row nextRecord(Row row) throws IOException { - try { - if (!hasNext) { - return null; - } - for (int pos = 0; pos < row.getArity(); pos++) { - row.setField(pos, resultSet.getObject(pos + 1)); - } - //update hasNext after we've read the record - hasNext = resultSet.next(); - return row; - } catch (SQLException se) { - throw new IOException("Couldn't read data - " + se.getMessage(), se); - } catch (NullPointerException npe) { - throw new IOException("Couldn't access resultSet", npe); - } - } - - @Override - public BaseStatistics getStatistics(BaseStatistics cachedStatistics) throws IOException { - return cachedStatistics; - } - - @Override - public InputSplit[] createInputSplits(int minNumSplits) throws IOException { - if (parameterValues == null) { - return new GenericInputSplit[]{new GenericInputSplit(0, 1)}; - } - GenericInputSplit[] ret = new GenericInputSplit[parameterValues.length]; - for (int i = 0; i < ret.length; i++) { - ret[i] = new GenericInputSplit(i, ret.length); - } - return ret; - } - - @Override - public InputSplitAssigner getInputSplitAssigner(InputSplit[] inputSplits) { - return new DefaultInputSplitAssigner(inputSplits); - } - - @VisibleForTesting - PreparedStatement getStatement() { - return statement; - } - - /** - * A builder used to set parameters to the output format's configuration in a fluent way. - * @return builder - */ - public static JDBCInputFormatPropsBuilder buildJDBCInputFormat() { - return new JDBCInputFormatPropsBuilder(); - } - - /** - * Builder for a {@link JDBCInputFormatProps}. - */ - public static class JDBCInputFormatPropsBuilder { - private final JDBCInputFormatProps format; - - public JDBCInputFormatPropsBuilder() { - this.format = new JDBCInputFormatProps(); - //using TYPE_FORWARD_ONLY for high performance reads - this.format.resultSetType = ResultSet.TYPE_FORWARD_ONLY; - this.format.resultSetConcurrency = ResultSet.CONCUR_READ_ONLY; - } - - public JDBCInputFormatProps.JDBCInputFormatPropsBuilder addProperties(Properties props) { - Enumeration names = props.propertyNames(); - while (names.hasMoreElements()) { - String name = names.nextElement().toString(); - format.props.setProperty(name, props.getProperty(name)); - } - return this; - } - - public JDBCInputFormatProps.JDBCInputFormatPropsBuilder setUsername(String username) { - format.props.setProperty(USER_PROP, username); - return this; - } - - public JDBCInputFormatProps.JDBCInputFormatPropsBuilder setPassword(String password) { - format.props.setProperty(PASSWORD_PROP, password); - return this; - } - - public JDBCInputFormatProps.JDBCInputFormatPropsBuilder setDrivername(String drivername) { - format.drivername = drivername; - return this; - } - - public JDBCInputFormatProps.JDBCInputFormatPropsBuilder setDBUrl(String dbURL) { - format.dbURL = dbURL; - return this; - } - - public JDBCInputFormatProps.JDBCInputFormatPropsBuilder setQuery(String query) { - format.queryTemplate = query; - return this; - } - - public JDBCInputFormatProps.JDBCInputFormatPropsBuilder setResultSetType(int resultSetType) { - format.resultSetType = resultSetType; - return this; - } - - public JDBCInputFormatProps.JDBCInputFormatPropsBuilder setResultSetConcurrency(int resultSetConcurrency) { - format.resultSetConcurrency = resultSetConcurrency; - return this; - } - - public JDBCInputFormatProps.JDBCInputFormatPropsBuilder setParametersProvider(ParameterValuesProvider parameterValuesProvider) { - format.parameterValues = parameterValuesProvider.getParameterValues(); - return this; - } - - public JDBCInputFormatProps.JDBCInputFormatPropsBuilder setRowTypeInfo(RowTypeInfo rowTypeInfo) { - format.rowTypeInfo = rowTypeInfo; - return this; - } - - public JDBCInputFormatProps.JDBCInputFormatPropsBuilder setFetchSize(int fetchSize) { - Preconditions.checkArgument(fetchSize == Integer.MIN_VALUE || fetchSize > 0, - "Illegal value %s for fetchSize, has to be positive or Integer.MIN_VALUE.", fetchSize); - format.fetchSize = fetchSize; - return this; - } - - public JDBCInputFormatProps finish() { - if (format.props.getProperty(USER_PROP) == null) { - LOG.info("Username was not supplied separately."); - } - if (format.props.getProperty(PASSWORD_PROP) == null) { - LOG.info("Password was not supplied separately."); - } - if (format.dbURL == null) { - throw new IllegalArgumentException("No database URL supplied"); - } - if (format.queryTemplate == null) { - throw new IllegalArgumentException("No query supplied"); - } - if (format.drivername == null) { - throw new IllegalArgumentException("No driver supplied"); - } - if (format.rowTypeInfo == null) { - throw new IllegalArgumentException("No " + RowTypeInfo.class.getSimpleName() + " supplied"); - } - if (format.parameterValues == null) { - LOG.debug("No input splitting configured (data will be read with parallelism 1)."); - } - return format; - } - - } - -} \ No newline at end of file diff --git a/flink/src/main/resources/application.conf b/flink/src/main/resources/application.conf deleted file mode 100644 index 0088c5825..000000000 --- a/flink/src/main/resources/application.conf +++ /dev/null @@ -1,6 +0,0 @@ -flink { - # Defines maximum number of parallel tasks and num of uniq partitions for operator (keyBy) - max-parallelism = 8192 - # Empty checkpoint saving dir disables checkpoints - savepoints-dir = "" -} \ No newline at end of file diff --git a/flink/src/main/resources/log4j2.xml b/flink/src/main/resources/log4j2.xml deleted file mode 100644 index 2c9d75704..000000000 --- a/flink/src/main/resources/log4j2.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/flink/src/main/scala/ru/itclover/tsp/PatternsSearchJob.scala b/flink/src/main/scala/ru/itclover/tsp/PatternsSearchJob.scala deleted file mode 100644 index ff45ba5ff..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/PatternsSearchJob.scala +++ /dev/null @@ -1,252 +0,0 @@ -package ru.itclover.tsp - -import cats.Traverse -import cats.data.Validated -import cats.implicits._ -import com.typesafe.scalalogging.Logger -import org.apache.flink.api.common.functions.RichMapFunction -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.streaming.api.datastream.DataStreamSink -import org.apache.flink.streaming.api.scala._ -import org.apache.flink.streaming.api.windowing.assigners._ -import org.apache.flink.streaming.api.windowing.time.{Time => WindowingTime} -import org.apache.flink.streaming.api.windowing.windows.{Window => FlinkWindow} -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer -import ru.itclover.tsp.core.IncidentInstances.semigroup -import ru.itclover.tsp.core.Pattern.IdxExtractor -import ru.itclover.tsp.core.aggregators.TimestampsAdderPattern -import ru.itclover.tsp.core.io.{BasicDecoders, Extractor, TimeExtractor} -import ru.itclover.tsp.core.optimizations.Optimizer -import ru.itclover.tsp.core.{Incident, RawPattern, _} -import ru.itclover.tsp.dsl.{ASTPatternGenerator, AnyState, PatternFieldExtractor, PatternMetadata} -import ru.itclover.tsp.io.input.KafkaInputConf -import ru.itclover.tsp.io.output.{KafkaOutputConf, OutputConf} -import ru.itclover.tsp.mappers._ -import ru.itclover.tsp.transformers.SparseRowsDataAccumulator -import ru.itclover.tsp.utils.DataStreamOps.DataStreamOps -import ru.itclover.tsp.utils.ErrorsADT.{ConfigErr, InvalidPatternsCode} - -import scala.reflect.ClassTag - -case class PatternsSearchJob[In: TypeInformation, InKey, InItem]( - source: StreamSource[In, InKey, InItem], - decoders: BasicDecoders[InItem] -) { - // TODO: Restore InKey as a type parameter - - import PatternsSearchJob._ - import decoders._ - import source.{eventCreator, keyCreator, kvExtractor} - - def patternsSearchStream[OutE: TypeInformation, OutKey, S]( - rawPatterns: Seq[RawPattern], - outputConf: OutputConf[OutE], - resultMapper: RichMapFunction[Incident, OutE] - ): Either[ConfigErr, (Seq[RichPattern[In, Segment, AnyState[Segment]]], DataStreamSink[OutE])] = { - import source.{idxExtractor, transformedExtractor, transformedTimeExtractor} - preparePatterns[In, S, InKey, InItem]( - rawPatterns, - source.fieldToEKey, - source.conf.defaultToleranceFraction.getOrElse(0), - source.conf.eventsMaxGapMs.getOrElse(60000L), - source.transformedFieldsClasses.map { case (s, c) => s -> ClassTag(c) }.toMap, - source.patternFields - ).map { patterns => - val forwardFields = outputConf.forwardedFieldsIds.map(id => (id, source.fieldToEKey(id))) - val useWindowing = !source.conf.isInstanceOf[KafkaInputConf] - val incidents = cleanIncidentsFromPatterns(patterns, forwardFields, useWindowing) - val mapped = incidents.map(resultMapper) - (patterns, saveStream(mapped, outputConf)) - } - } - - def cleanIncidentsFromPatterns( - richPatterns: Seq[RichPattern[In, Segment, AnyState[Segment]]], - forwardedFields: Seq[(Symbol, InKey)], - useWindowing: Boolean - ): DataStream[Incident] = { - import source.timeExtractor - val stream = source.createStream - val singleIncidents = incidentsFromPatterns( - applyTransformation(stream.assignAscendingTimestamps(timeExtractor(_).toMillis)), - richPatterns, - forwardedFields, - useWindowing - ) - if (source.conf.defaultEventsGapMs.getOrElse(2000L) > 0L) reduceIncidents(singleIncidents) else singleIncidents - } - - def incidentsFromPatterns[T]( - stream: DataStream[In], - patterns: Seq[RichPattern[In, Segment, AnyState[Segment]]], - forwardedFields: Seq[(Symbol, InKey)], - useWindowing: Boolean - ): DataStream[Incident] = { - - import source.{transformedExtractor, idxExtractor, transformedTimeExtractor => timeExtractor} - - log.debug("incidentsFromPatterns started") - - val mappers: Seq[PatternProcessor[In, Optimizer.S[Segment], Incident]] = patterns.map { - case ((pattern, meta), rawP) => - val allForwardFields = forwardedFields ++ rawP.forwardedFields - .getOrElse(Seq.empty) - .map(id => (id, source.fieldToEKey(id))) - - val toIncidents = ToIncidentsMapper( - rawP.id, - allForwardFields.map { case (id, k) => id.toString.tail -> k }, - source.fieldToEKey(source.conf.unitIdField.get), - rawP.subunit.getOrElse(0), - rawP.payload.getOrElse(Map.empty).toSeq, - if (meta.sumWindowsMs > 0L) meta.sumWindowsMs else source.conf.defaultEventsGapMs.getOrElse(2000L), - source.conf.partitionFields.map(source.fieldToEKey) - ) - - val optimizedPattern = new Optimizer[In].optimize(pattern) - - val incidentPattern = MapWithContextPattern(optimizedPattern)(toIncidents.apply) - - PatternProcessor[In, Optimizer.S[Segment], Incident]( - incidentPattern, - source.conf.eventsMaxGapMs.getOrElse(60000L) - ) - } - val keyedStream = stream - .assignAscendingTimestamps(timeExtractor(_).toMillis) - .keyBy(source.transformedPartitioner) - val windowed = - if (useWindowing) { - keyedStream - .window( - TumblingEventTimeWindows - .of(WindowingTime.milliseconds(source.conf.chunkSizeMs.getOrElse(900000))) - .asInstanceOf[WindowAssigner[In, FlinkWindow]] - ) - } else { - // For Kafka we generate windows by processing time (not event time) every 1 second, - // so we always get the results without collecting huge window. - keyedStream - .window( - TumblingProcessingTimeWindows - .of(WindowingTime.seconds(1)) - .asInstanceOf[WindowAssigner[In, FlinkWindow]] - ) - } - val processed = windowed - .process[Incident]( - ProcessorCombinator(mappers, timeExtractor) - ) - .setMaxParallelism(source.conf.maxPartitionsParallelism) - - log.debug("incidentsFromPatterns finished") - processed - } - - def applyTransformation(dataStream: DataStream[In]): DataStream[In] = source.conf.dataTransformation match { - case Some(_) => - import source.{extractor, timeExtractor} - dataStream - .keyBy(source.partitioner) - .process( - SparseRowsDataAccumulator[In, InKey, InItem, In]( - source.asInstanceOf[StreamSource[In, InKey, InItem]], - source.patternFields - ) - ) - .setParallelism(1) // SparseRowsDataAccumulator cannot work in parallel - case _ => dataStream - } -} - -object PatternsSearchJob { - type RichSegmentedP[E] = RichPattern[E, Segment, AnyState[Segment]] - type RichPattern[E, T, S] = ((Pattern[E, S, T], PatternMetadata), RawPattern) - - val log = Logger("PatternsSearchJob") - def maxPartitionsParallelism = 8192 - - def preparePatterns[E, S, EKey, EItem]( - rawPatterns: Seq[RawPattern], - fieldsIdxMap: Symbol => EKey, - toleranceFraction: Double, - eventsMaxGapMs: Long, - fieldsTags: Map[Symbol, ClassTag[_]], - patternFields: Set[EKey] - )( - implicit extractor: Extractor[E, EKey, EItem], - getTime: TimeExtractor[E], - idxExtractor: IdxExtractor[E] /*, - dDecoder: Decoder[EItem, Double]*/ - ): Either[ConfigErr, List[RichPattern[E, Segment, AnyState[Segment]]]] = { - - log.debug("preparePatterns started") - - val filteredPatterns = rawPatterns.filter( - p => PatternFieldExtractor.extract[E, EKey, EItem](List(p))(fieldsIdxMap).subsetOf(patternFields) - ) - - val pGenerator = ASTPatternGenerator[E, EKey, EItem]()( - idxExtractor, - getTime, - extractor, - fieldsIdxMap - ) - val res = Traverse[List] - .traverse(filteredPatterns.toList)( - p => - Validated - .fromEither(pGenerator.build(p.sourceCode, toleranceFraction, eventsMaxGapMs, fieldsTags)) - .leftMap(err => List(s"PatternID#${p.id}, error: ${err.getMessage}")) - .map( - p => - ( - new TimestampsAdderPattern(SegmentizerPattern(p._1)) - .asInstanceOf[Pattern[E, AnyState[Segment], Segment]], - p._2 - ) - ) - ) - .leftMap[ConfigErr](InvalidPatternsCode(_)) - .map(_.zip(filteredPatterns)) - .toEither - - log.debug("preparePatterns finished") - - res - } - - def reduceIncidents(incidents: DataStream[Incident]) = { - log.debug("reduceIncidents started") - - val res = incidents - .assignAscendingTimestamps_withoutWarns(p => p.segment.from.toMillis) - .keyBy(p => (p.id, p.patternPayload)) - .window(EventTimeSessionWindows.withDynamicGap(new SessionWindowTimeGapExtractor[Incident] { - override def extract(element: Incident): Long = element.maxWindowMs - })) - .reduce { _ |+| _ } - .name("Uniting adjacent incidents") - - log.debug("reduceIncidents finished") - res - } - - def saveStream[E](stream: DataStream[E], outputConf: OutputConf[E]): DataStreamSink[E] = { - log.debug("saveStream started") - outputConf match { - case kafkaConf: KafkaOutputConf => - val producer = new FlinkKafkaProducer(kafkaConf.broker, kafkaConf.topic, kafkaConf.dataSerializer) - .asInstanceOf[FlinkKafkaProducer[E]] // here we know that E == Row - val res = stream.addSink(producer) - log.debug("saveStream finished") - res - - case _ => - val outputFormat = outputConf.getOutputFormat - val res = stream.writeUsingOutputFormat(outputFormat) - log.debug("saveStream finished") - res - } - } -} diff --git a/flink/src/main/scala/ru/itclover/tsp/StreamSources.scala b/flink/src/main/scala/ru/itclover/tsp/StreamSources.scala deleted file mode 100644 index 79cae4da7..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/StreamSources.scala +++ /dev/null @@ -1,535 +0,0 @@ -package ru.itclover.tsp - -import cats.syntax.either._ -import com.typesafe.scalalogging.Logger -import org.apache.flink.api.common.io.RichInputFormat -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.RowTypeInfo -import org.apache.flink.core.io.InputSplit -import org.apache.flink.streaming.api.functions.timestamps.AscendingTimestampExtractor -import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment, _} -import org.apache.flink.types.Row -import org.influxdb.dto.QueryResult -import org.redisson.client.codec.ByteArrayCodec -import ru.itclover.tsp.core.Pattern.{Idx, IdxExtractor} -import ru.itclover.tsp.core.io.{Decoder, Extractor, TimeExtractor} -import ru.itclover.tsp.io.input._ -import ru.itclover.tsp.io.{EventCreator, EventCreatorInstances} -import ru.itclover.tsp.services.{InfluxDBService, JdbcService, KafkaService} -import ru.itclover.tsp.transformers.SparseRowsDataAccumulator -import ru.itclover.tsp.utils.ErrorsADT._ -import ru.itclover.tsp.utils.RowOps.{RowIsoTimeExtractor, RowSymbolExtractor, RowTsTimeExtractor} -import ru.itclover.tsp.utils.{KeyCreator, KeyCreatorInstances} - -import scala.collection.JavaConverters._ -import scala.util.{Failure, Success} -import scala.collection.mutable - -/*sealed*/ -trait StreamSource[Event, EKey, EItem] extends Product with Serializable { - def createStream: DataStream[Event] - - def conf: InputConf[Event, EKey, EItem] - - def fieldsClasses: Seq[(Symbol, Class[_])] - - def transformedFieldsClasses: Seq[(Symbol, Class[_])] = conf.dataTransformation match { - case Some(NarrowDataUnfolding(key, value, _, mapping, _)) => - val m: Map[EKey, List[EKey]] = mapping.getOrElse(Map.empty) - val r = fieldsClasses ++ m.map { - case (col, list) => - list.map( - k => - ( - eKeyToField(k), - fieldsClasses - .find { - case (s, _) => fieldToEKey(s) == col - } - .map(_._2) - .getOrElse(defaultClass) - ) - ) - }.flatten - r - case _ => - fieldsClasses - } - - def defaultClass: Class[_] = conf.dataTransformation match { - case Some(NarrowDataUnfolding(_, value, _, _, _)) => - fieldsClasses.find { case (s, _) => fieldToEKey(s) == value }.map(_._2).getOrElse(classOf[Double]) - case _ => - classOf[Double] - } - - def fieldToEKey: Symbol => EKey - - def eKeyToField: EKey => Symbol - - def fieldsIdxMap: Map[Symbol, Int] - - def transformedFieldsIdxMap: Map[Symbol, Int] - - def partitioner: Event => String - - def transformedPartitioner: Event => String - - implicit def timeExtractor: TimeExtractor[Event] - - implicit def transformedTimeExtractor: TimeExtractor[Event] - - implicit def idxExtractor: IdxExtractor[Event] - - implicit def extractor: Extractor[Event, EKey, EItem] - - implicit def transformedExtractor: Extractor[Event, EKey, EItem] - - implicit def trivialEItemDecoder: Decoder[EItem, EItem] = (v1: EItem) => v1 - - implicit def itemToKeyDecoder: Decoder[EItem, EKey] // for narrow data widening - - implicit def kvExtractor: Event => (EKey, EItem) = conf.dataTransformation match { - case Some(NarrowDataUnfolding(key, value, _, mapping, _)) => - (r: Event) => - // TODO: Maybe optimise that by using intermediate (non-serialised) dictionary - val extractedKey = extractor.apply[EKey](r, key) - val valueColumn = mapping - .getOrElse(Map.empty[EKey, List[EKey]]) - .toSeq - .find { - case (_, list) => - list.contains(extractedKey) - } - .map(_._1) - .getOrElse(value) - val extractedValue = extractor.apply[EItem](r, valueColumn) - (extractedKey, extractedValue) // TODO: See that place better - case Some(WideDataFilling(_, _)) => - (_: Event) => sys.error("Wide data filling does not need K-V extractor") - case Some(_) => - (_: Event) => sys.error("Unsupported data transformation") - case None => - (_: Event) => sys.error("No K-V extractor without data transformation") - } - - implicit def eventCreator: EventCreator[Event, EKey] - - implicit def keyCreator: KeyCreator[EKey] - - def patternFields: Set[EKey] -} - -object StreamSource { - - def findNullField(allFields: Seq[Symbol], excludedFields: Seq[Symbol]) = - allFields.find { field => - !excludedFields.contains(field) - } -} - -case class RowWithIdx(idx: Idx, row: Row) - -object JdbcSource { - - def create(conf: JDBCInputConf, fields: Set[Symbol])( - implicit strEnv: StreamExecutionEnvironment - ): Either[Err, JdbcSource] = - for { - types <- JdbcService - .fetchFieldsTypesInfo(conf.driverName, conf.jdbcUrl, conf.query) - .toEither - .leftMap[ConfigErr](e => SourceUnavailable(Option(e.getMessage).getOrElse(e.toString))) - newFields <- checkKeysExistence(conf, fields) - source <- StreamSource.findNullField(types.map(_._1), conf.datetimeField +: conf.partitionFields) match { - case Some(nullField) => JdbcSource(conf, types, nullField, newFields).asRight - case None => InvalidRequest("Source should contain at least one non partition and datatime field.").asLeft - } - } yield source - - def checkKeysExistence(conf: JDBCInputConf, keys: Set[Symbol]): Either[GenericRuntimeErr, Set[Symbol]] = - conf.dataTransformation match { - case Some(NarrowDataUnfolding(keyColumn, _, _, _, _)) => - JdbcService - .fetchAvailableKeys(conf.driverName, conf.jdbcUrl, conf.query, keyColumn) - .toEither - .map(_.intersect(keys)) - .leftMap[GenericRuntimeErr](e => GenericRuntimeErr(e, 5099)) - case _ => Right(keys) - } -} - -// todo rm nullField and trailing nulls in queries at platform (uniting now done on Flink) after states fix -case class JdbcSource( - conf: JDBCInputConf, - fieldsClasses: Seq[(Symbol, Class[_])], - nullFieldId: Symbol, - patternFields: Set[Symbol] -)( - implicit @transient streamEnv: StreamExecutionEnvironment -) extends StreamSource[RowWithIdx, Symbol, Any] { - - import conf._ - - val stageName = "JDBC input processing stage" - val log = Logger[JdbcSource] - val fieldsIdx = fieldsClasses.map(_._1).zipWithIndex - val fieldsIdxMap = fieldsIdx.toMap - def partitionsIdx = partitionFields.filter(fieldsIdxMap.contains).map(fieldsIdxMap) - def transformedPartitionsIdx = partitionFields.map(transformedFieldsIdxMap) - - require(fieldsIdxMap.get(datetimeField).isDefined, "Cannot find datetime field, index overflow.") - require(fieldsIdxMap(datetimeField) < fieldsIdxMap.size, "Cannot find datetime field, index overflow.") - private val badPartitions = partitionFields - .map(fieldsIdxMap.get) - .find(idx => idx.getOrElse(Int.MaxValue) >= fieldsIdxMap.size) - .flatten - .map(p => fieldsClasses(p)._1) - require(badPartitions.isEmpty, s"Cannot find partition field (${badPartitions.getOrElse('unknown)}), index overflow.") - - val timeIndex = fieldsIdxMap(datetimeField) - val transformedTimeIndex = transformedFieldsIdxMap(datetimeField) - val fieldsTypesInfo: Array[TypeInformation[_]] = fieldsClasses.map(c => TypeInformation.of(c._2)).toArray - val rowTypesInfo = new RowTypeInfo(fieldsTypesInfo, fieldsClasses.map(_._1.toString.tail).toArray) - - override def createStream = { - - val stream = streamEnv.createInput(inputFormat).name(stageName) - - val ascendingExtractor = new AscendingTimestampExtractor[RowWithIdx] { - override def extractAscendingTimestamp(element: RowWithIdx): Long = timeExtractor(element).toMillis - } - - parallelism - .fold(stream)(stream.setParallelism) - .map(r => RowWithIdx(0, r)) // todo hack look in PatternProcessor for actual idx - .assignTimestampsAndWatermarks(ascendingExtractor) - } - - override def fieldToEKey = { fieldId: Symbol => - fieldId - // fieldsIdxMap(fieldId) - } - - override def eKeyToField: Symbol => Symbol = { key: Symbol => - key - } - - override def partitioner: RowWithIdx => String = { - val serializablePI = partitionsIdx - event: RowWithIdx => serializablePI.map(event.row.getField).mkString - } - - override def transformedPartitioner: RowWithIdx => String = { - val serializablePI = transformedPartitionsIdx - event: RowWithIdx => serializablePI.map(event.row.getField).mkString - } - - def tsMultiplier = timestampMultiplier.getOrElse { - log.trace("timestampMultiplier in JDBC source conf is not provided, use default = 1000.0") - 1000.0 - } - - override def timeExtractor: TimeExtractor[RowWithIdx] = { - val rowExtractor = RowTsTimeExtractor(timeIndex, tsMultiplier, datetimeField) - TimeExtractor.of(r => rowExtractor(r.row)) - } - override def extractor = RowSymbolExtractor(fieldsIdxMap).comap(_.row) - - override def transformedExtractor = RowSymbolExtractor(transformedFieldsIdxMap).comap(_.row) - - val inputFormat: RichInputFormat[Row, InputSplit] = - JDBCInputFormatProps - .buildJDBCInputFormat() - .setDrivername(driverName) - .setDBUrl(jdbcUrl) - .setUsername(userName.getOrElse("")) - .setPassword(password.getOrElse("")) - .setQuery(query) - .setRowTypeInfo(rowTypesInfo) - .finish() - - implicit override def eventCreator: EventCreator[RowWithIdx, Symbol] = - EventCreatorInstances.rowWithIdxSymbolEventCreator - - implicit override def keyCreator: KeyCreator[Symbol] = KeyCreatorInstances.symbolKeyCreator - - implicit override def itemToKeyDecoder: Decoder[Any, Symbol] = (x: Any) => Symbol(x.toString) - - override def transformedFieldsIdxMap: Map[Symbol, Int] = conf.dataTransformation match { - case Some(_) => - val acc = SparseRowsDataAccumulator[RowWithIdx, Symbol, Any, RowWithIdx](this, patternFields)( - createTypeInformation[RowWithIdx], - timeExtractor, - kvExtractor, - extractor, - eventCreator, - keyCreator - ) - acc.allFieldsIndexesMap - case None => - fieldsIdxMap - } - - implicit override def transformedTimeExtractor: TimeExtractor[RowWithIdx] = - RowTsTimeExtractor(transformedTimeIndex, tsMultiplier, datetimeField).comap(_.row) - - //todo refactor everything related to idxExtractor - implicit override def idxExtractor: IdxExtractor[RowWithIdx] = IdxExtractor.of(_.idx) -} - -object InfluxDBSource { - - def create(conf: InfluxDBInputConf, fields: Set[Symbol])( - implicit strEnv: StreamExecutionEnvironment - ): Either[ConfigErr, InfluxDBSource] = - for { - types <- InfluxDBService - .fetchFieldsTypesInfo(conf.query, conf.influxConf) - .toEither - .leftMap[ConfigErr](e => SourceUnavailable(Option(e.getMessage).getOrElse(e.toString))) - source <- StreamSource.findNullField(types.map(_._1), conf.datetimeField +: conf.partitionFields) match { - case Some(nullField) => InfluxDBSource(conf, types, nullField, fields).asRight - case None => InvalidRequest("Source should contain at least one non partition and datatime field.").asLeft - } - } yield source -} - -case class InfluxDBSource( - conf: InfluxDBInputConf, - fieldsClasses: Seq[(Symbol, Class[_])], - nullFieldId: Symbol, - patternFields: Set[Symbol] -)( - implicit @transient streamEnv: StreamExecutionEnvironment -) extends StreamSource[RowWithIdx, Symbol, Any] { - - import conf._ - - val dummyResult: Class[QueryResult.Result] = new QueryResult.Result().getClass.asInstanceOf[Class[QueryResult.Result]] - val queryResultTypeInfo: TypeInformation[QueryResult.Result] = TypeInformation.of(dummyResult) - val stageName = "InfluxDB input processing stage" - val defaultTimeoutSec = 200L - - val fieldsIdx = fieldsClasses.map(_._1).zipWithIndex - val fieldsIdxMap = fieldsIdx.toMap - def partitionsIdx = partitionFields.filter(fieldsIdxMap.contains).map(fieldsIdxMap) - def transformedPartitionsIdx = partitionFields.map(transformedFieldsIdxMap) - - require(fieldsIdxMap.get(datetimeField).isDefined, "Cannot find datetime field, index overflow.") - require(fieldsIdxMap(datetimeField) < fieldsIdxMap.size, "Cannot find datetime field, index overflow.") - private val badPartitions = partitionFields - .map(fieldsIdxMap.get) - .find(idx => idx.getOrElse(Int.MaxValue) >= fieldsIdxMap.size) - .flatten - .map(p => fieldsClasses(p)._1) - require(badPartitions.isEmpty, s"Cannot find partition field (${badPartitions.getOrElse('unknown)}), index overflow.") - - val timeIndex = fieldsIdxMap(datetimeField) - val transformedTimeIndex = transformedFieldsIdxMap(datetimeField) - val fieldsTypesInfo: Array[TypeInformation[_]] = fieldsClasses.map(c => TypeInformation.of(c._2)).toArray - val rowTypesInfo = new RowTypeInfo(fieldsTypesInfo, fieldsClasses.map(_._1.toString.tail).toArray) - - override def createStream = { - val serFieldsIdxMap = fieldsIdxMap // for task serialization - val stream = streamEnv - .createInput(inputFormat)(queryResultTypeInfo) - .flatMap(queryResult => { - // extract Flink.rows form series of points - if (queryResult == null || queryResult.getSeries == null) { - mutable.Buffer[Row]() - } else - for { - series <- queryResult.getSeries.asScala - valueSet <- series.getValues.asScala.map(_.asScala) - if valueSet != null - } yield { - val tags = if (series.getTags != null) series.getTags.asScala else Map.empty - val row = new Row(tags.size + valueSet.size) - val fieldsAndValues = tags ++ series.getColumns.asScala.zip(valueSet) - fieldsAndValues.foreach { - case (field, value) => row.setField(serFieldsIdxMap(Symbol(field)), value) - } - row - } - }) - .name(stageName) - (parallelism match { - case Some(p) => stream.setParallelism(p) - case None => stream - }).map(r => RowWithIdx(0, r)) // todo hack look in PatternProcessor for actual idx - } - - override def fieldToEKey = (fieldId: Symbol) => fieldId // fieldsIdxMap(fieldId) - - override def eKeyToField: Symbol => Symbol = (key: Symbol) => key - - override def partitioner = { - val serializablePI = partitionsIdx - event: RowWithIdx => serializablePI.map(event.row.getField).mkString - } - - override def transformedPartitioner = { - val serializablePI = transformedPartitionsIdx - event: RowWithIdx => serializablePI.map(event.row.getField).mkString - } - - override def timeExtractor = RowIsoTimeExtractor(timeIndex, datetimeField).comap(_.row) - override def extractor = RowSymbolExtractor(fieldsIdxMap).comap(_.row) - override def transformedExtractor = RowSymbolExtractor(transformedFieldsIdxMap).comap(_.row) - - val inputFormat = - InfluxDBInputFormat - .create() - .url(url) - .timeoutSec(timeoutSec.getOrElse(defaultTimeoutSec)) - .username(userName.getOrElse("")) - .password(password.getOrElse("")) - .database(dbName) - .query(query) - .and() - .buildIt() - - implicit override def eventCreator: EventCreator[RowWithIdx, Symbol] = - EventCreatorInstances.rowWithIdxSymbolEventCreator - - implicit override def keyCreator: KeyCreator[Symbol] = KeyCreatorInstances.symbolKeyCreator - - implicit override def itemToKeyDecoder: Decoder[Any, Symbol] = (x: Any) => Symbol(x.toString) - - override def transformedFieldsIdxMap: Map[Symbol, Int] = conf.dataTransformation match { - case Some(_) => - val acc = SparseRowsDataAccumulator[RowWithIdx, Symbol, Any, RowWithIdx](this, patternFields)( - createTypeInformation[RowWithIdx], - timeExtractor, - kvExtractor, - extractor, - eventCreator, - keyCreator - ) - acc.allFieldsIndexesMap - case None => - fieldsIdxMap - } - - implicit override def transformedTimeExtractor: TimeExtractor[RowWithIdx] = - RowIsoTimeExtractor(transformedTimeIndex, datetimeField).comap(_.row) - - //todo refactor everything related to idxExtractor - implicit override def idxExtractor: IdxExtractor[RowWithIdx] = IdxExtractor.of(_.idx) -} - -object KafkaSource { - - val log = Logger[KafkaSource] - - def create(conf: KafkaInputConf, fields: Set[Symbol])( - implicit strEnv: StreamExecutionEnvironment - ): Either[ConfigErr, KafkaSource] = - for { - types <- KafkaService - .fetchFieldsTypesInfo(conf) - .toEither - .leftMap[ConfigErr](e => SourceUnavailable(Option(e.getMessage).getOrElse(e.toString))) - _ = log.info(s"Kafka types found: $types") - source <- StreamSource.findNullField(types.map(_._1), conf.datetimeField +: conf.partitionFields) match { - case Some(nullField) => KafkaSource(conf, types, nullField, fields).asRight - case None => InvalidRequest("Source should contain at least one non partition and datatime field.").asLeft - } - } yield source - -} - -case class KafkaSource( - conf: KafkaInputConf, - fieldsClasses: Seq[(Symbol, Class[_])], - nullFieldId: Symbol, - patternFields: Set[Symbol] -)( - implicit @transient streamEnv: StreamExecutionEnvironment -) extends StreamSource[RowWithIdx, Symbol, Any] { - - val log = Logger[KafkaSource] - - def fieldsIdx = fieldsClasses.map(_._1).zipWithIndex - def fieldsIdxMap = fieldsIdx.toMap - - override def fieldToEKey: Symbol => Symbol = (x => x) - - override def eKeyToField: Symbol => Symbol = (key: Symbol) => key - - def timeIndex = fieldsIdxMap(conf.datetimeField) - - def tsMultiplier = conf.timestampMultiplier.getOrElse { - log.debug("timestampMultiplier in Kafka source conf is not provided, use default = 1000.0") - 1000.0 - } - - implicit def extractor: ru.itclover.tsp.core.io.Extractor[RowWithIdx, Symbol, Any] = - RowSymbolExtractor(fieldsIdxMap).comap(_.row) - implicit def timeExtractor: ru.itclover.tsp.core.io.TimeExtractor[RowWithIdx] = - RowTsTimeExtractor(timeIndex, tsMultiplier, conf.datetimeField).comap(_.row) - - val stageName = "Kafka input processing stage" - - def createStream: DataStream[RowWithIdx] = { - val consumer = KafkaService.consumer(conf, fieldsIdxMap) - // TODO: Make this parameter configurable - consumer.setStartFromGroupOffsets() - streamEnv.enableCheckpointing(5000) - streamEnv - .addSource(consumer) - // we need to set parallelism of Kafka source to 1 to persist ordering. - // If we know number of topic partitions, it's better to pass this number here. - .setParallelism(conf.numParallelSources.getOrElse(1)) - .name(stageName) - //.keyBy(_ => "nokey") - //.process(new TimeOutFunction(5000, timeIndex, fieldsIdxMap.size)) - }.map(r => RowWithIdx(0, r)) // todo hack look in PatternProcessor for actual idx - - def partitionsIdx = conf.partitionFields.filter(fieldsIdxMap.contains).map(fieldsIdxMap) - def transformedPartitionsIdx = conf.partitionFields.map(transformedFieldsIdxMap) - - def partitioner = { - val serializablePI = partitionsIdx - event: RowWithIdx => serializablePI.map(event.row.getField).mkString - } - - def transformedPartitioner = { - val serializablePI = transformedPartitionsIdx - event: RowWithIdx => serializablePI.map(event.row.getField).mkString - } - - override def transformedFieldsIdxMap: Map[Symbol, Int] = conf.dataTransformation match { - case Some(_) => - val acc = SparseRowsDataAccumulator[RowWithIdx, Symbol, Any, RowWithIdx](this, patternFields)( - createTypeInformation[RowWithIdx], - timeExtractor, - kvExtractor, - extractor, - eventCreator, - keyCreator - ) - acc.allFieldsIndexesMap - case None => - fieldsIdxMap - } - - val transformedTimeIndex = transformedFieldsIdxMap(conf.datetimeField) - - implicit override def transformedTimeExtractor: TimeExtractor[RowWithIdx] = - RowTsTimeExtractor(transformedTimeIndex, tsMultiplier, conf.datetimeField).comap(_.row) - - implicit override def transformedExtractor: Extractor[RowWithIdx, Symbol, Any] = - RowSymbolExtractor(transformedFieldsIdxMap).comap(_.row) - - implicit override def itemToKeyDecoder: Decoder[Any, Symbol] = (x: Any) => Symbol(x.toString) - - implicit override def eventCreator: EventCreator[RowWithIdx, Symbol] = - EventCreatorInstances.rowWithIdxSymbolEventCreator - - implicit override def keyCreator: KeyCreator[Symbol] = KeyCreatorInstances.symbolKeyCreator - //todo refactor everything related to idxExtractor - implicit override def idxExtractor: IdxExtractor[RowWithIdx] = IdxExtractor.of(_.idx) - -} diff --git a/flink/src/main/scala/ru/itclover/tsp/io/EventCreator.scala b/flink/src/main/scala/ru/itclover/tsp/io/EventCreator.scala deleted file mode 100644 index 2ec4f799b..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/io/EventCreator.scala +++ /dev/null @@ -1,34 +0,0 @@ -package ru.itclover.tsp.io -import org.apache.flink.types.Row -import ru.itclover.tsp.RowWithIdx - -trait EventCreator[Event, Key] extends Serializable { - def create(kv: Seq[(Key, AnyRef)]): Event -} - -object EventCreatorInstances { - implicit val rowSymbolEventCreator: EventCreator[Row, Symbol] = new EventCreator[Row, Symbol] { - override def create(kv: Seq[(Symbol, AnyRef)]): Row = { - val row = new Row(kv.length) - kv.zipWithIndex.foreach { kvWithIndex => - row.setField(kvWithIndex._2, kvWithIndex._1._2) - } - row - } - } - - implicit val rowIntEventCreator: EventCreator[Row, Int] = new EventCreator[Row, Int] { - override def create(kv: Seq[(Int, AnyRef)]): Row = { - val row = new Row(kv.length) - kv.zipWithIndex.foreach { kvWithIndex => - row.setField(kvWithIndex._2, kvWithIndex._1._2) - } - row - } - } - - //todo change it to not have effects here - implicit val rowWithIdxSymbolEventCreator: EventCreator[RowWithIdx, Symbol] = new EventCreator[RowWithIdx, Symbol] { - override def create(kv: Seq[(Symbol, AnyRef)]): RowWithIdx = RowWithIdx(0, rowSymbolEventCreator.create(kv)) - } -} diff --git a/flink/src/main/scala/ru/itclover/tsp/io/input/InfluxDBInputConf.scala b/flink/src/main/scala/ru/itclover/tsp/io/input/InfluxDBInputConf.scala deleted file mode 100644 index 0cf865de8..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/io/input/InfluxDBInputConf.scala +++ /dev/null @@ -1,50 +0,0 @@ -package ru.itclover.tsp.io.input - -import ru.itclover.tsp.RowWithIdx -import ru.itclover.tsp.services.InfluxDBService - -/** - * Source for InfluxDB - * @param sourceId simple mark to pass to sink - * @param dbName wich DB to use - * @param url to database, for example `http://localhost:8086` - * @param query Influx SQL query for data - * @param eventsMaxGapMs maximum gap by which source data will be split, i.e. result incidents will be split by these gaps - * @param defaultEventsGapMs "typical" gap between events, used to unite nearby incidents in one (sessionization) - * @param partitionFields fields by which data will be split and paralleled physically - * @param datetimeField name of datetime field, could be timestamp and regular time (will be parsed by JodaTime) - * @param userName for auth - * @param password for auth - * @param timeoutSec for DB connection - * @param parallelism of source task (not recommended to chagne) - * @param numParallelSources number of absolutely separate sources to create. Patterns also will be separated by - * equal (as much as possible) buckets by the max window in pattern (TBD by sum window size) - * @param patternsParallelism number of parallel branch nodes splitted after sink stage (node). Patterns also - * separated by approx. equal buckets by the max window in pattern (TBD by sum window size) - */ -@SerialVersionUID(91001L) -case class InfluxDBInputConf( - sourceId: Int, - dbName: String, - url: String, - query: String, - eventsMaxGapMs: Option[Long] = Some(60000L), - defaultEventsGapMs: Option[Long] = Some(2000L), - chunkSizeMs: Option[Long], - partitionFields: Seq[Symbol], - datetimeField: Symbol = 'time, - unitIdField: Option[Symbol] = None, - userName: Option[String] = None, - password: Option[String] = None, - timeoutSec: Option[Long] = None, - dataTransformation: Option[SourceDataTransformation[RowWithIdx, Symbol, Any]] = None, - defaultToleranceFraction: Option[Double] = None, - parallelism: Option[Int] = None, - numParallelSources: Option[Int] = Some(1), - patternsParallelism: Option[Int] = Some(2), - additionalTypeChecking: Option[Boolean] = Some(true) -) extends InputConf[RowWithIdx, Symbol, Any] { - - val influxConf = - InfluxDBService.InfluxConf(url, dbName, userName, password, 200L, additionalTypeChecking.getOrElse(true)) -} diff --git a/flink/src/main/scala/ru/itclover/tsp/io/input/InfluxDBInputFormat.scala b/flink/src/main/scala/ru/itclover/tsp/io/input/InfluxDBInputFormat.scala deleted file mode 100644 index 130d748c4..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/io/input/InfluxDBInputFormat.scala +++ /dev/null @@ -1,104 +0,0 @@ -package ru.itclover.tsp.io.input - -import java.io.IOException -import java.util -import java.util.concurrent.TimeUnit -import java.util.function.Consumer - -import okhttp3.OkHttpClient -import org.apache.flink.api.common.io.{GenericInputFormat, NonParallelInput} -import org.apache.flink.core.io.GenericInputSplit -import org.influxdb.InfluxDBFactory -import org.influxdb.dto.{Query, QueryResult} - -object InfluxDBInputFormat { - def create() = new Builder() - - class Builder { - val obj = new InfluxDBInputFormat() - - def url(url: String) = { - - obj.url = url - this - } - - def username(username: String) = { - - obj.username = username - this - } - - def password(password: String) = { - - obj.password = password - this - } - - def database(database: String) = { - - obj.database = database - this - } - - def timeoutSec(sec: Long) = { - - obj.timeoutSec = sec - this - } - - def query(query: String) = { - - obj.query = query - this - } - - def and() = - new Actions() - - class Actions { - - def buildIt() = - obj - - def consumeIt(consumer: Consumer[InfluxDBInputFormat]) = - consumer.accept(buildIt()) - } - } -} - -@SerialVersionUID(42L) -class InfluxDBInputFormat extends GenericInputFormat[QueryResult.Result] with NonParallelInput { - var url: String = _ - var username: String = _ - var password: String = _ - var database: String = _ - var query: String = _ - var timeoutSec: Long = _ - - var queryResult: util.ListIterator[QueryResult.Result] = _ - - // -------------------------------------------------------------------------- - - override def open(split: GenericInputSplit) = { - val extraConf = new OkHttpClient.Builder() - .readTimeout(timeoutSec, TimeUnit.SECONDS) - .writeTimeout(timeoutSec, TimeUnit.SECONDS) - .connectTimeout(timeoutSec, TimeUnit.SECONDS) - val connection = InfluxDBFactory.connect(url, username, password, extraConf) - val queryResult = connection.query(new Query(this.query, database)) - if (queryResult.hasError) { - throw new IOException("Could not execute query: " + queryResult.getError) - } - this.queryResult = queryResult.getResults.listIterator() - } - - override def reachedEnd() = !queryResult.hasNext - - def nextRecord(reuse: QueryResult.Result): QueryResult.Result = { - val result = queryResult.next() - reuse.setError(result.getError) - reuse.setSeries(result.getSeries) - reuse - } -} diff --git a/flink/src/main/scala/ru/itclover/tsp/io/input/InputConf.scala b/flink/src/main/scala/ru/itclover/tsp/io/input/InputConf.scala deleted file mode 100644 index d7cf8c1ec..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/io/input/InputConf.scala +++ /dev/null @@ -1,24 +0,0 @@ -package ru.itclover.tsp.io.input - -trait InputConf[Event, EKey, EItem] extends Serializable { - def sourceId: Int // todo .. Rm - - def datetimeField: Symbol - def partitionFields: Seq[Symbol] - def unitIdField: Option[Symbol] // Only for new sink, will be ignored for old - - def parallelism: Option[Int] // Parallelism per each source - def numParallelSources: Option[Int] // Number on parallel (separate) sources to be created - def patternsParallelism: Option[Int] // Number of parallel branches after source step - - def eventsMaxGapMs: Option[Long] - def defaultEventsGapMs: Option[Long] - def chunkSizeMs: Option[Long] // Chunk size - - def dataTransformation: Option[SourceDataTransformation[Event, EKey, EItem]] - - def defaultToleranceFraction: Option[Double] - - // Set maximum number of physically independent partitions for stream.keyBy operation - def maxPartitionsParallelism: Int = 8192 -} diff --git a/flink/src/main/scala/ru/itclover/tsp/io/input/JDBCInputConf.scala b/flink/src/main/scala/ru/itclover/tsp/io/input/JDBCInputConf.scala deleted file mode 100644 index 45c8673bc..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/io/input/JDBCInputConf.scala +++ /dev/null @@ -1,43 +0,0 @@ -package ru.itclover.tsp.io.input -import ru.itclover.tsp.RowWithIdx - -/** - * Source for anything that support JDBC connection - * - * @param sourceId mark to pass to sink - * @param jdbcUrl example - "jdbc:clickhouse://localhost:8123/default?" - * @param query SQL query for data - * @param driverName example - "ru.yandex.clickhouse.ClickHouseDriver" - * @param datetimeField name of datetime field, could be timestamp and regular time (will be parsed by JodaTime) - * @param eventsMaxGapMs maximum gap by which source data will be split, i.e. result incidents will be split by these gaps - * @param defaultEventsGapMs "typical" gap between events, used to unite nearby incidents in one (sessionization) - * @param partitionFields fields by which data will be split and paralleled physically - * @param userName for JDBC auth - * @param password for JDBC auth - * @param parallelism of source task (not recommended to chagne) - * @param numParallelSources number of absolutely separate sources to create. Patterns also will be separated by - * equal (as much as possible) buckets by the max window in pattern (TBD by sum window size) - * @param patternsParallelism number of parallel branch nodes splitted after sink stage (node). Patterns also - * separated by approx. equal buckets by the max window in pattern (TBD by sum window size) - */ -@SerialVersionUID(91002L) -case class JDBCInputConf( - sourceId: Int, - jdbcUrl: String, - query: String, - driverName: String, - datetimeField: Symbol, - eventsMaxGapMs: Option[Long] = Some(60000L), - defaultEventsGapMs: Option[Long] = Some(2000L), - chunkSizeMs: Option[Long], - partitionFields: Seq[Symbol], - unitIdField: Option[Symbol] = None, - userName: Option[String] = None, - password: Option[String] = None, - dataTransformation: Option[SourceDataTransformation[RowWithIdx, Symbol, Any]] = None, - defaultToleranceFraction: Option[Double] = None, - parallelism: Option[Int] = Some(1), - numParallelSources: Option[Int] = Some(1), - patternsParallelism: Option[Int] = Some(1), - timestampMultiplier: Option[Double] = Some(1000.0) -) extends InputConf[RowWithIdx, Symbol, Any] diff --git a/flink/src/main/scala/ru/itclover/tsp/io/input/KafkaInputConf.scala b/flink/src/main/scala/ru/itclover/tsp/io/input/KafkaInputConf.scala deleted file mode 100644 index 394b24aa9..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/io/input/KafkaInputConf.scala +++ /dev/null @@ -1,29 +0,0 @@ -package ru.itclover.tsp.io.input - -import java.util.UUID -import ru.itclover.tsp.RowWithIdx - -@SerialVersionUID(91000L) -case class KafkaInputConf( - sourceId: Int, - brokers: String, - topic: String, - group: String = UUID.randomUUID().toString, - serializer: Option[String] = Some("json"), - datetimeField: Symbol, - partitionFields: Seq[Symbol], - unitIdField: Option[Symbol] = None, - dataTransformation: Option[SourceDataTransformation[RowWithIdx, Symbol, Any]] = None, - timestampMultiplier: Option[Double] = Some(1000.0), - eventsMaxGapMs: Option[Long] = Some(90000), - chunkSizeMs: Option[Long] = Some(10L), - numParallelSources: Option[Int] = Some(1), - fieldsTypes: Map[String, String] -) extends InputConf[RowWithIdx, Symbol, Any] { - - def defaultEventsGapMs: Option[Long] = Some(0L) - def defaultToleranceFraction: Option[Double] = None - def parallelism: Option[Int] = Some(1) - def patternsParallelism: Option[Int] = Some(1) -} - diff --git a/flink/src/main/scala/ru/itclover/tsp/io/input/RedisInputConf.scala b/flink/src/main/scala/ru/itclover/tsp/io/input/RedisInputConf.scala deleted file mode 100644 index 6e19c19dc..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/io/input/RedisInputConf.scala +++ /dev/null @@ -1,36 +0,0 @@ -package ru.itclover.tsp.io.input - -import ru.itclover.tsp.RowWithIdx - -/** - * Source for Redis Input - * @param url connection string to redis - * @param datetimeField name of datetime field, could be timestamp and regular time (will be parsed by JodaTime) - * @param partitionFields fields by which data will be split and paralleled physically - * @param fieldsTypes description of field and his type - * @param key by what key get data from redis - * @param serializer format of data in redis - */ -@SerialVersionUID(4815162342L) -@deprecated("Redis support will be dropped", "0.16.0") -case class RedisInputConf( - url: String, - datetimeField: Symbol, - partitionFields: Seq[Symbol], - unitIdField: Option[Symbol] = None, - dataTransformation: Option[SourceDataTransformation[RowWithIdx, Symbol, Any]] = None, - fieldsTypes: Map[String, String], - key: String, - serializer: String = "json" -) extends InputConf[RowWithIdx, Symbol, Any] { - - def chunkSizeMs: Option[Long] = Some(10L) - def defaultEventsGapMs: Option[Long] = Some(0L) - def defaultToleranceFraction: Option[Double] = Some(0.1) - def eventsMaxGapMs: Option[Long] = Some(1L) - def numParallelSources: Option[Int] = Some(1) - def parallelism: Option[Int] = Some(1) - def patternsParallelism: Option[Int] = Some(1) - def sourceId: Int = 1 - -} diff --git a/flink/src/main/scala/ru/itclover/tsp/io/input/SourceDataTransformationConf.scala b/flink/src/main/scala/ru/itclover/tsp/io/input/SourceDataTransformationConf.scala deleted file mode 100644 index 871a5d061..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/io/input/SourceDataTransformationConf.scala +++ /dev/null @@ -1,24 +0,0 @@ -package ru.itclover.tsp.io.input - -sealed trait SourceDataTransformationConf - -abstract class SourceDataTransformation[Event, EKey, EValue](val `type`: String) extends Serializable { - val config: SourceDataTransformationConf -} - -case class NarrowDataUnfolding[Event, EKey, EValue]( - keyColumn: EKey, - defaultValueColumn: EKey, - fieldsTimeoutsMs: Map[EKey, Long], - valueColumnMapping: Option[Map[EKey, List[EKey]]] = None, - defaultTimeout: Option[Long] = None -) extends SourceDataTransformation[Event, EKey, EValue]("NarrowDataUnfolding") - with SourceDataTransformationConf { - override val config: SourceDataTransformationConf = this -} - -case class WideDataFilling[Event, EKey, EValue](fieldsTimeoutsMs: Map[EKey, Long], defaultTimeout: Option[Long] = None) - extends SourceDataTransformation[Event, EKey, EValue]("WideDataFilling") - with SourceDataTransformationConf { - override val config: SourceDataTransformationConf = this -} diff --git a/flink/src/main/scala/ru/itclover/tsp/io/output/JDBCOutput.scala b/flink/src/main/scala/ru/itclover/tsp/io/output/JDBCOutput.scala deleted file mode 100644 index 9e8f8bd4f..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/io/output/JDBCOutput.scala +++ /dev/null @@ -1,31 +0,0 @@ -package ru.itclover.tsp.io.output - -// import org.apache.flink.api.java.io.jdbc.JDBCAppendTableSink -import com.typesafe.scalalogging.Logger -import org.apache.flink.api.java.io.jdbc.JDBCOutputFormat - -object JDBCOutput { - val log = Logger("ClickhouseOutput") - val DEFAULT_BATCH_INTERVAL = 1000000 - - def getOutputFormat(config: JDBCOutputConf): JDBCOutputFormat = { - val insertQuery = getInsertQuery(config.tableName, config.rowSchema) - log.info(s"Configure ClickhouseOutput with insertQuery = `$insertQuery`") - JDBCOutputFormat - .buildJDBCOutputFormat() - .setDrivername(config.driverName) - .setDBUrl(config.jdbcUrl) - .setUsername(config.userName.getOrElse("")) - .setPassword(config.password.getOrElse("")) - .setQuery(insertQuery) - .setSqlTypes(config.rowSchema.fieldsTypes.toArray) - .setBatchInterval(config.batchInterval.getOrElse(DEFAULT_BATCH_INTERVAL)) - .finish() - } - - private def getInsertQuery(tableName: String, rowSchema: EventSchema) = { - val columns = rowSchema.fieldsNames.map(_.toString().tail) - val statements = columns.map(_ => "?").mkString(", ") - s"INSERT INTO ${tableName} (${columns.mkString(", ")}) VALUES (${statements})" - } -} diff --git a/flink/src/main/scala/ru/itclover/tsp/io/output/OutputConf.scala b/flink/src/main/scala/ru/itclover/tsp/io/output/OutputConf.scala deleted file mode 100644 index 0e9ab7da9..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/io/output/OutputConf.scala +++ /dev/null @@ -1,83 +0,0 @@ -package ru.itclover.tsp.io.output - -import org.apache.flink.api.common.io.OutputFormat -import org.apache.flink.api.common.serialization.SerializationSchema -import org.apache.flink.formats.avro.AvroOutputFormat -import org.apache.flink.types.Row -import ru.itclover.tsp.serializers.KafkaSerializers.{ArrowSerializer, JSONSerializer} - -trait OutputConf[Event] { - def forwardedFieldsIds: Seq[Symbol] - - def getOutputFormat: OutputFormat[Event] - - def parallelism: Option[Int] - - def rowSchema: EventSchema -} - -/** - * Sink for anything that support JDBC connection - * @param rowSchema schema of writing rows, __will be replaced soon__ - * @param jdbcUrl example - "jdbc:clickhouse://localhost:8123/default?" - * @param driverName example - "ru.yandex.clickhouse.ClickHouseDriver" - * @param userName for JDBC auth - * @param password for JDBC auth - * @param batchInterval batch size for writing found incidents - * @param parallelism num of parallel task to write data - */ -case class JDBCOutputConf( - tableName: String, - rowSchema: EventSchema, - jdbcUrl: String, - driverName: String, - password: Option[String] = None, - batchInterval: Option[Int] = None, - userName: Option[String] = None, - parallelism: Option[Int] = Some(1) -) extends OutputConf[Row] { - override def getOutputFormat = JDBCOutput.getOutputFormat(this) - - override def forwardedFieldsIds = rowSchema match { - case newRS: NewRowSchema => Seq.empty // no forwarded fields in new row schema - } -} - -///** -// * "Empty" sink (for example, used if one need only to report timings) -// */ -//case class EmptyOutputConf() extends OutputConf[Row] { -// override def forwardedFieldsIds: Seq[Symbol] = Seq() -// override def getOutputFormat: OutputFormat[Row] = ??? -// override def parallelism: Option[Int] = Some(1) -//} - -/** - * Sink for kafka connection - * @param broker host and port for kafka broker - * @param topic where is data located - * @param serializer format of data in kafka - * @param rowSchema schema of writing rows - * @param parallelism num of parallel task to write data - * @author Dmitry Galanin - */ -case class KafkaOutputConf( - broker: String, - topic: String, - serializer: Option[String] = Some("json"), - rowSchema: EventSchema, - parallelism: Option[Int] = Some(1) -) extends OutputConf[Row] { - override def forwardedFieldsIds = rowSchema match { - case newRS: NewRowSchema => Seq.empty // no forwarded fields in new row schema - } - - override def getOutputFormat: OutputFormat[Row] = new AvroOutputFormat(classOf[Row]) // actually not needed - - def dataSerializer: SerializationSchema[Row] = serializer.getOrElse("json") match { - case "json" => new JSONSerializer(rowSchema) - case "arrow" => new ArrowSerializer(rowSchema) - case _ => throw new IllegalArgumentException(s"No deserializer for type ${serializer}") - } - -} diff --git a/flink/src/main/scala/ru/itclover/tsp/io/output/SinkSchema.scala b/flink/src/main/scala/ru/itclover/tsp/io/output/SinkSchema.scala deleted file mode 100644 index 699cb2f74..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/io/output/SinkSchema.scala +++ /dev/null @@ -1,60 +0,0 @@ -package ru.itclover.tsp.io.output - -import java.sql.Types -import java.time.{LocalDateTime, ZonedDateTime} - -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.RowTypeInfo - -import scala.collection.mutable - -/** - * Schema for writing data to sink. - */ -trait SinkSchema extends Serializable { - def rowSchema: EventSchema -} - -//case class KafkaSegmentsSink(schemaUri: String, brokerList: String, topicId: String, rowSchema: RowSchema) { -// override def toString: String = { -// "{" + super.toString + s", fieldsIndexesMap=${rowSchema.fieldsIndexesMap}" -// } -//} - -trait EventSchema { // TODO fieldsTypesInfo to PatternsSearchJob - def fieldsTypes: List[Int] - - def fieldsNames: List[Symbol] - - def fieldsCount: Int -} - -case class NewRowSchema( - unitIdField: Symbol, - fromTsField: Symbol, - toTsField: Symbol, - appIdFieldVal: (Symbol, Int), - patternIdField: Symbol, - subunitIdField: Symbol -) extends EventSchema - with Serializable { - override def fieldsTypes: List[Int] = - List(Types.INTEGER, Types.TIMESTAMP, Types.TIMESTAMP, Types.INTEGER, Types.INTEGER, Types.INTEGER) - - override def fieldsNames: List[Symbol] = - List(unitIdField, fromTsField, toTsField, appIdFieldVal._1, patternIdField, subunitIdField) - - override def fieldsCount: Int = 6 - - val fieldsIndexesMap: mutable.LinkedHashMap[Symbol, Int] = mutable.LinkedHashMap(fieldsNames.zipWithIndex: _*) - - val unitIdInd = fieldsIndexesMap(unitIdField) - val subunitIdInd = fieldsIndexesMap(subunitIdField) - - val beginInd = fieldsIndexesMap(fromTsField) - val endInd = fieldsIndexesMap(toTsField) - - val appIdInd = fieldsIndexesMap(appIdFieldVal._1) - val patternIdInd = fieldsIndexesMap(patternIdField) - -} diff --git a/flink/src/main/scala/ru/itclover/tsp/mappers/MapWithContextPattern.scala b/flink/src/main/scala/ru/itclover/tsp/mappers/MapWithContextPattern.scala deleted file mode 100644 index 72d1f6476..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/mappers/MapWithContextPattern.scala +++ /dev/null @@ -1,42 +0,0 @@ -package ru.itclover.tsp.mappers - -import cats.syntax.foldable._ -import cats.syntax.functor._ -import cats.{Foldable, Functor, Monad} -import ru.itclover.tsp.core.PQueue.MapPQueue -import ru.itclover.tsp.core.Result._ -import ru.itclover.tsp.core.{PQueue, Pattern, Result} - -import scala.language.higherKinds - -//todo tests -// MapWithContextPattern is a pattern which can add context to the results of inner using information from head of `events`. -case class MapWithContextPattern[Event, InnerState, T1, T2](inner: Pattern[Event, InnerState, T1])( - val func: Event => T1 => T2 -) extends Pattern[Event, InnerState, T2] { - override def apply[F[_]: Monad, Cont[_]: Foldable: Functor]( - oldState: InnerState, - oldQueue: PQueue[T2], - event: Cont[Event] - ): F[(InnerState, PQueue[T2])] = { - - // we need to trim inner queue here to avoid memory leaks - val innerQueue = getInnerQueue(oldQueue) - val headOption = event.get(0).toResult - inner(oldState, innerQueue, event).map { - case (innerResult, innerQueue) => - innerResult -> MapPQueue[T1, T2](innerQueue, _.value.flatMap(t1 => headOption.map(head => func(head)(t1)))) - } - } - - private def getInnerQueue(queue: PQueue[T2]): PQueue[T1] = { - queue match { - case MapPQueue(innerQueue, _) if innerQueue.isInstanceOf[PQueue[T1]] => innerQueue.asInstanceOf[PQueue[T1]] - //this is for first call - case x if x.size == 0 => x.asInstanceOf[PQueue[T1]] - case _ => sys.error("Wrong logic! MapPattern.apply must be called only with MapPQueue") - } - } - - override def initialState(): InnerState = inner.initialState() -} diff --git a/flink/src/main/scala/ru/itclover/tsp/mappers/PatternProcessor.scala b/flink/src/main/scala/ru/itclover/tsp/mappers/PatternProcessor.scala deleted file mode 100644 index 38106c82d..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/mappers/PatternProcessor.scala +++ /dev/null @@ -1,88 +0,0 @@ -package ru.itclover.tsp.mappers - -import cats.Id -import com.typesafe.scalalogging.Logger -import org.apache.flink.util.Collector -import ru.itclover.tsp.core.io.TimeExtractor -import ru.itclover.tsp.core.{Time, _} - -import scala.collection.mutable.ListBuffer - -case class PatternProcessor[E: TimeExtractor, State, Out]( - pattern: Pattern[E, State, Out], - eventsMaxGapMs: Long -) { - - private val log = Logger("PatternLogger") - private var lastState: State = _ - private var lastTime: Time = Time(0) - private val timeExtractor = implicitly[TimeExtractor[E]] - log.debug(s"pattern: $pattern") - - def process( - elements: Iterable[E], - out: Collector[Out] - ): Unit = { - if (elements.isEmpty) { - log.info("No elements to proccess") - return - } - - val initialState = pattern.initialState() - val firstElement = elements.head - // if the last event occurred so long ago, clear the state - if (lastState == null || timeExtractor(firstElement).toMillis - lastTime.toMillis > eventsMaxGapMs) { - lastState = initialState - } - - // Split the different time sequences if they occurred in the same time window - val sequences = PatternProcessor.splitByCondition(elements.toSeq)( - (next, prev) => timeExtractor(next).toMillis - timeExtractor(prev).toMillis > eventsMaxGapMs - ) - - val machine = StateMachine[Id] - - val consume: IdxValue[Out] => Unit = x => x.value.foreach(out.collect) - - val seedStates = lastState +: Stream.continually(initialState) - - // this step has side-effect = it calls `consume` for each output event. We need to process - // events sequentually, that's why I use foldLeft here - lastState = sequences.zip(seedStates).foldLeft(initialState) { - case (_, (events, seedState)) => machine.run(pattern, events, seedState, consume) - } - - lastTime = elements.lastOption.map(timeExtractor(_)).getOrElse(Time(0)) - } -} - -object PatternProcessor { - - val currentEventTsMetric = "currentEventTs" - - /** - * Splits a list into a list of fragments, the boundaries are determined by the given predicate - * E.g. `splitByCondition(List(1,2,3,5,8,9,12), (x, y) => (x - y) > 2) == List(List(1,2,3,5),List(8,9),List(12)` - * - * @param elements initial sequence - * @param pred condition between the next and previous elements (in this order) - * @tparam T Element type - * @return List of chunks - */ - def splitByCondition[T](elements: Seq[T])(pred: (T, T) => Boolean): List[Seq[T]] = - if (elements.length < 2) { - List(elements) - } else { - val results = ListBuffer(ListBuffer(elements.head)) - elements.sliding(2).foreach { e => - val prev = e.head - val cur = e(1) - if (pred(cur, prev)) { - results += ListBuffer(cur) - } else { - results.lastOption.getOrElse(sys.error("Empty result sequence - something went wrong")) += cur - } - } - results.map(_.toSeq).toList - } -} diff --git a/flink/src/main/scala/ru/itclover/tsp/mappers/PatternsToRowMapper.scala b/flink/src/main/scala/ru/itclover/tsp/mappers/PatternsToRowMapper.scala deleted file mode 100644 index b1183457a..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/mappers/PatternsToRowMapper.scala +++ /dev/null @@ -1,50 +0,0 @@ -package ru.itclover.tsp.mappers - -import java.sql.Timestamp -import java.time.{Instant, LocalDateTime, ZoneId, ZoneOffset, ZonedDateTime} - -import org.apache.flink.api.common.functions.RichMapFunction -import org.apache.flink.types.Row -import ru.itclover.tsp.core.Incident -import ru.itclover.tsp.io.output.{EventSchema, NewRowSchema} - -import scala.util.Try - -/** - * Packer of found incident into [[org.apache.flink.types.Row]] - */ -case class PatternsToRowMapper[Event, EKey](sourceId: Int, schema: EventSchema) extends RichMapFunction[Incident, Row] { - - override def map(incident: Incident) = schema match { - case newRowSchema: NewRowSchema => - val resultRow = new Row(newRowSchema.fieldsCount) - resultRow.setField(newRowSchema.unitIdInd, incident.patternUnit) - resultRow.setField(newRowSchema.patternIdInd, incident.patternId) - resultRow.setField(newRowSchema.appIdInd, newRowSchema.appIdFieldVal._2) - resultRow.setField(newRowSchema.beginInd, Timestamp.from(Instant.ofEpochMilli(incident.segment.from.toMillis))) - resultRow.setField(newRowSchema.endInd, Timestamp.from(Instant.ofEpochMilli(incident.segment.to.toMillis))) - resultRow.setField(newRowSchema.subunitIdInd, incident.patternSubunit) - - resultRow - } - - def nowInUtcMillis: Double = { - val zonedDt = ZonedDateTime.of(LocalDateTime.now, ZoneId.systemDefault) - val utc = zonedDt.withZoneSameInstant(ZoneId.of("UTC")) - Timestamp.valueOf(utc.toLocalDateTime).getTime / 1000.0 - } - - def payloadToJson(payload: Seq[(String, Any)]): String = - payload - .map { - case (fld, value) if value.isInstanceOf[String] => s""""${fld}":"${value}"""" - case (fld, value) => s""""${fld}":$value""" - } - .mkString("{", ",", "}") - -// def findSubunit(payload: Seq[(String, Any)]): Int = { -// payload.find { case (name, _) => name.toLowerCase == "subunit" } -// .map{ case (_, value) => Try(value.toString.toInt).getOrElse(0) } -// .getOrElse(0) -// } -} diff --git a/flink/src/main/scala/ru/itclover/tsp/mappers/ProcessorCombinator.scala b/flink/src/main/scala/ru/itclover/tsp/mappers/ProcessorCombinator.scala deleted file mode 100644 index a81be1cdd..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/mappers/ProcessorCombinator.scala +++ /dev/null @@ -1,34 +0,0 @@ -package ru.itclover.tsp.mappers -import java.util.concurrent.atomic.AtomicLong - -import org.apache.flink.streaming.api.scala.function.ProcessWindowFunction -import org.apache.flink.streaming.api.windowing.windows.Window -import org.apache.flink.util.Collector -import ru.itclover.tsp.RowWithIdx -import ru.itclover.tsp.core.io.TimeExtractor - -case class ProcessorCombinator[In, S, Inner, Out]( - mappers: Seq[PatternProcessor[In, S, Out]], - timeExtractor: TimeExtractor[In] -) extends ProcessWindowFunction[In, Out, String, Window] { - - private val counter = new AtomicLong(0) - - override def process( - key: String, - context: Context, - elements: Iterable[In], - out: Collector[Out] - ): Unit = { -// todo hack!!! - val sorted = elements.toBuffer.sortBy(timeExtractor.apply) - if (sorted.head.isInstanceOf[RowWithIdx]) { - val indexed = - sorted.map(x => x.asInstanceOf[RowWithIdx].copy(idx = counter.incrementAndGet())).asInstanceOf[Iterable[In]] - mappers.foreach(_.process( /*key,*/ indexed, out)) - } else { - mappers.foreach(_.process( /*key,*/ sorted, out)) - } - } - -} diff --git a/flink/src/main/scala/ru/itclover/tsp/mappers/ToIncidentsMapper.scala b/flink/src/main/scala/ru/itclover/tsp/mappers/ToIncidentsMapper.scala deleted file mode 100644 index 28108c315..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/mappers/ToIncidentsMapper.scala +++ /dev/null @@ -1,24 +0,0 @@ -package ru.itclover.tsp.mappers - -import ru.itclover.tsp.core.io.{Decoder, Extractor} -import ru.itclover.tsp.core.{Incident, Segment} - -import scala.util.Try - -final case class ToIncidentsMapper[E, EKey, EItem]( - patternId: Int, - forwardedFields: Seq[(String, EKey)], - unitIdField: EKey, - subunit: Int, - payload: Seq[(String, String)], - sessionWindowMs: Long, - partitionFields: Seq[EKey] -)(implicit extractor: Extractor[E, EKey, EItem], decoder: Decoder[EItem, Any]) { - - def apply(event: E): Segment => Incident = { - val incidentId = s"P#$patternId;" + partitionFields.map(f => f -> extractor[Any](event, f)).mkString - val extractedFields = forwardedFields.map { case (name, k) => name -> extractor[Any](event, k).toString } - val unit = Try(extractor[Any](event, unitIdField).toString.toInt).getOrElse(Int.MinValue) - segment => Incident(incidentId, patternId, sessionWindowMs, segment, extractedFields, unit, subunit, payload) - } -} diff --git a/flink/src/main/scala/ru/itclover/tsp/serializers/KafkaSerializers.scala b/flink/src/main/scala/ru/itclover/tsp/serializers/KafkaSerializers.scala deleted file mode 100644 index 6b7bdaba0..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/serializers/KafkaSerializers.scala +++ /dev/null @@ -1,18 +0,0 @@ -package ru.itclover.tsp.serializers - -import org.apache.flink.api.common.serialization.SerializationSchema -import org.apache.flink.types.Row -import ru.itclover.tsp.io.output.EventSchema -import ru.itclover.tsp.serializers.core.{ArrowSerialization, JSONSerialization} - -object KafkaSerializers { - - class JSONSerializer(rowSchema: EventSchema) extends SerializationSchema[Row] { - override def serialize(element: Row): Array[Byte] = new JSONSerialization().serialize(element, rowSchema) - } - - class ArrowSerializer(rowSchema: EventSchema) extends SerializationSchema[Row] { - override def serialize(element: Row): Array[Byte] = new ArrowSerialization().serialize(element, rowSchema) - } - -} diff --git a/flink/src/main/scala/ru/itclover/tsp/serializers/RowAvroSerializer.scala b/flink/src/main/scala/ru/itclover/tsp/serializers/RowAvroSerializer.scala deleted file mode 100644 index e57f5006f..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/serializers/RowAvroSerializer.scala +++ /dev/null @@ -1,54 +0,0 @@ -//package ru.itclover.tsp.serializers -// -//import java.io.IOException -//import org.apache.flink.types.Row -//import org.apache.avro.generic.{GenericData, GenericDatumWriter, GenericRecord, GenericRecordBuilder} -//import org.apache.avro.io.{DatumWriter, EncoderFactory} -//import org.apache.avro.{Schema, SchemaBuilder} -//import org.apache.commons.io.output.ByteArrayOutputStream -//import org.apache.flink.api.common.serialization.SerializationSchema -//import org.apache.flink.api.common.typeinfo.TypeInformation -//import scala.concurrent.duration._ -//import cats.syntax.either._ -// -// -//case class RowAvroSerializer(fieldsIndexesMap: Map[Symbol, Int], schema: Schema) -// (implicit ty: TypeInformation[Row]) extends SerializationSchema[Row] { -// -// @transient -// private var dtw: GenericDatumWriter[GenericData.Record] = getOrCreateWriter(schema) -// -// override def serialize(element: Row): Array[Byte] = { -// ensureInitialized() -// -// val record = new GenericData.Record(schema) -// fieldsIndexesMap foreach { fieldAndInd => -// record.put(fieldAndInd._1.toString.tail, element.getField(fieldAndInd._2)) -// } -// -// val arrOutStream = new ByteArrayOutputStream() -// val encoder = EncoderFactory.get().binaryEncoder(arrOutStream, null) -// try { -// arrOutStream.reset() -// dtw.write(record, encoder) -// encoder.flush() -// arrOutStream.toByteArray -// } -// catch { -// case ex: IOException => throw new RuntimeException(s"Fail to serialize Row", ex) -// } -// } -// -// private def ensureInitialized(): Unit = { -// if (dtw == null) { -// dtw = getOrCreateWriter(schema) -// } -// } -// -// private def getOrCreateWriter(schema: Schema) = { -// if (dtw == null) { -// dtw = new GenericDatumWriter[GenericData.Record](schema) -// } -// dtw -// } -//} diff --git a/flink/src/main/scala/ru/itclover/tsp/serializers/core/ArrowSerialization.scala b/flink/src/main/scala/ru/itclover/tsp/serializers/core/ArrowSerialization.scala deleted file mode 100644 index 8948d1bf9..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/serializers/core/ArrowSerialization.scala +++ /dev/null @@ -1,110 +0,0 @@ -package ru.itclover.tsp.serializers.core - -import java.io.File -import java.nio.file.Files -import java.sql.Timestamp - -import org.apache.arrow.memory.RootAllocator -import org.apache.arrow.vector.types.{DateUnit, TimeUnit} -import org.apache.arrow.vector.types.pojo.{ArrowType, Field, Schema} -import org.apache.flink.types.Row -import ru.itclover.tsp.io.output.{EventSchema, NewRowSchema} -import ru.itclover.tsp.serializers.utils.SerializationUtils -import ru.itclover.tsp.services.FileService -import ru.itclover.tsp.utils.ArrowOps - -import scala.collection.JavaConverters._ -import scala.collection.mutable - -/** - * Serialization for Apache Arrow format - */ -// Arrow serialization heavily deals with Any's and null's (and also asInstanceOf). -// Hence, no option other than suppress these warts here. -@SuppressWarnings(Array("org.wartremover.warts.Null", "org.wartremover.warts.Any", "org.wartremover.warts.AsInstanceOf")) -class ArrowSerialization extends Serialization[Array[Byte], Row] { - - override def serialize(output: Row, eventSchema: EventSchema): Array[Byte] = { - - val tempPath = FileService.createTemporaryFile() - val tempFile = tempPath.toFile - - val schemaFields = eventSchema match { - case newRowSchema: NewRowSchema => - List( - new Field( - newRowSchema.unitIdField.name, - false, - new ArrowType.Int(32, true), - null - ), - new Field( - newRowSchema.fromTsField.name, - false, - new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC"), - null - ), - new Field( - newRowSchema.toTsField.name, - false, - new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC"), - null - ), - new Field( - newRowSchema.appIdFieldVal._1.name, - false, - new ArrowType.Int(32, true), - null - ), - new Field( - newRowSchema.patternIdField.name, - false, - new ArrowType.Utf8, - null - ), - new Field( - newRowSchema.subunitIdField.name, - false, - new ArrowType.Utf8, - null - ) - ) - } - - val schema = new Schema(schemaFields.asJava) - val allocator = new RootAllocator(1000000) - - val data = eventSchema match { - case newRowSchema: NewRowSchema => - mutable.ListBuffer( - mutable.Map( - newRowSchema.unitIdField.name -> output.getField(newRowSchema.unitIdInd).asInstanceOf[Int], - newRowSchema.fromTsField.name -> output.getField(newRowSchema.beginInd).asInstanceOf[Timestamp], - newRowSchema.toTsField.name -> output.getField(newRowSchema.endInd).asInstanceOf[Timestamp], - newRowSchema.appIdFieldVal._1.name -> output.getField(newRowSchema.appIdInd).asInstanceOf[Int], - newRowSchema.patternIdField.name -> output.getField(newRowSchema.patternIdInd).asInstanceOf[String], - newRowSchema.subunitIdField.name -> output.getField(newRowSchema.subunitIdInd).asInstanceOf[String] - ) - ) - } - - ArrowOps.writeData((tempFile, schema, data, allocator)) - - val result = Files.readAllBytes(tempPath) - tempFile.delete() - - result - - } - - override def deserialize(input: Array[Byte], fieldsIdxMap: Map[Symbol, Int]): Row = { - - val tempFile: File = FileService.convertBytes(input) - val schemaAndReader = ArrowOps.retrieveSchemaAndReader(tempFile, Integer.MAX_VALUE) - val rowData = ArrowOps.retrieveData(schemaAndReader) - tempFile.delete() - - SerializationUtils.combineRows(rowData) - - } -} diff --git a/flink/src/main/scala/ru/itclover/tsp/serializers/core/JSONSerialization.scala b/flink/src/main/scala/ru/itclover/tsp/serializers/core/JSONSerialization.scala deleted file mode 100644 index fab9dad24..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/serializers/core/JSONSerialization.scala +++ /dev/null @@ -1,64 +0,0 @@ -package ru.itclover.tsp.serializers.core - -import java.nio.charset.Charset -import java.sql.Timestamp - -import com.fasterxml.jackson.databind.ObjectMapper -import org.apache.flink.types.Row -import ru.itclover.tsp.io.output.{EventSchema, NewRowSchema} - -/** - * JSON Serialization for Redis - */ -class JSONSerialization extends Serialization[Array[Byte], Row] { - - /** - * Method for deserialize from json string - * @param input bytes array from json string - * @return flink row - */ - override def deserialize(input: Array[Byte], fieldsIdxMap: Map[Symbol, Int]): Row = { - - val inputData = new String(input) - val jsonTree = new ObjectMapper().readTree(inputData) - val row = new Row(fieldsIdxMap.size) - val mapper = new ObjectMapper() - - fieldsIdxMap.foreach { - case (elem, index) => - val rawValue = jsonTree.get(elem.name) - val fieldValue = mapper.convertValue(rawValue, classOf[java.lang.Object]) - row.setField(index, fieldValue) - } - - row - - } - - /** - * Method for serialize to json string - * @param output flink row - * @param rowSchema schema from flink row - * @return bytes array from json string - */ - override def serialize(output: Row, eventSchema: EventSchema): Array[Byte] = { - - val mapper = new ObjectMapper() - val root = mapper.createObjectNode() - - eventSchema match { - case newRowSchema: NewRowSchema => - root.put(newRowSchema.unitIdField.name, output.getField(newRowSchema.unitIdInd).asInstanceOf[Int]) - root.put(newRowSchema.fromTsField.name, output.getField(newRowSchema.beginInd).asInstanceOf[Timestamp].toString) - root.put(newRowSchema.toTsField.name, output.getField(newRowSchema.endInd).asInstanceOf[Timestamp].toString) - root.put(newRowSchema.appIdFieldVal._1.name, output.getField(newRowSchema.appIdInd).asInstanceOf[Int]) - root.put(newRowSchema.patternIdField.name, output.getField(newRowSchema.patternIdInd).asInstanceOf[String]) - root.put(newRowSchema.subunitIdField.name, output.getField(newRowSchema.subunitIdInd).asInstanceOf[String]) - } - - val jsonString = mapper.writeValueAsString(root) - - jsonString.getBytes(Charset.forName("UTF-8")) - - } -} diff --git a/flink/src/main/scala/ru/itclover/tsp/serializers/core/Serialization.scala b/flink/src/main/scala/ru/itclover/tsp/serializers/core/Serialization.scala deleted file mode 100644 index d6696848c..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/serializers/core/Serialization.scala +++ /dev/null @@ -1,16 +0,0 @@ -package ru.itclover.tsp.serializers.core - -import ru.itclover.tsp.io.output.EventSchema - -/** - * Deserialization trait for Redis - * - * @tparam INPUT input type - * @tparam OUTPUT output type - */ -trait Serialization[INPUT, OUTPUT] { - - def serialize(output: OUTPUT, rowSchema: EventSchema): INPUT - def deserialize(input: INPUT, fieldsIdxMap: Map[Symbol, Int]): OUTPUT - -} diff --git a/flink/src/main/scala/ru/itclover/tsp/serializers/utils/SerializationUtils.scala b/flink/src/main/scala/ru/itclover/tsp/serializers/utils/SerializationUtils.scala deleted file mode 100644 index 8774b445e..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/serializers/utils/SerializationUtils.scala +++ /dev/null @@ -1,32 +0,0 @@ -package ru.itclover.tsp.serializers.utils - -import org.apache.flink.types.Row - -import scala.collection.mutable - -object SerializationUtils { - - def combineRows(input: mutable.ListBuffer[Row]): Row = { - - val size = input.head.getArity * input.size - val row = new Row(size) - var counter = 0 - - input.foreach(rowInput => { - - val arity = rowInput.getArity - - (0 until arity).foreach(i => { - - row.setField(counter, rowInput.getField(i)) - counter += 1 - - }) - - }) - - row - - } - -} diff --git a/flink/src/main/scala/ru/itclover/tsp/services/FileService.scala b/flink/src/main/scala/ru/itclover/tsp/services/FileService.scala deleted file mode 100644 index 49833f62b..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/services/FileService.scala +++ /dev/null @@ -1,50 +0,0 @@ -package ru.itclover.tsp.services - -import java.io.File -import java.nio.ByteBuffer -import java.nio.channels.FileChannel -import java.nio.file.{Files, Path, StandardOpenOption} -import java.time.LocalDateTime - -import scala.util.Random - -object FileService { - - /** - * Method for creating temp file - * @return temp file path - */ - def createTemporaryFile(): Path = { - - val currentTime = LocalDateTime.now().toString - val randomInd = Random.nextInt(Integer.MAX_VALUE) - - Files.createTempFile(s"temp_${randomInd}_($currentTime)", ".temp") - - } - - /** - * Method for converting input bytes to file - * @param input bytes for convert - * @return file with input bytes - */ - def convertBytes(input: Array[Byte]): File = { - val path = FileService.createTemporaryFile() - - val options = Set( - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING, - StandardOpenOption.WRITE - ).toSeq - - val fileChannel = FileChannel.open(path, options: _*) - val buffer = ByteBuffer.wrap(input) - - fileChannel.write(buffer) - fileChannel.close() - - path.toFile - - } - -} diff --git a/flink/src/main/scala/ru/itclover/tsp/services/InfluxDBService.scala b/flink/src/main/scala/ru/itclover/tsp/services/InfluxDBService.scala deleted file mode 100644 index 60af4f2de..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/services/InfluxDBService.scala +++ /dev/null @@ -1,87 +0,0 @@ -package ru.itclover.tsp.services - -import java.util.concurrent.TimeUnit -import java.util.regex.Pattern - -import okhttp3.OkHttpClient -import org.influxdb.{InfluxDB, InfluxDBException, InfluxDBFactory} -import org.influxdb.dto.Query -import org.influxdb.{InfluxDB, InfluxDBException, InfluxDBFactory} -import ru.itclover.tsp.utils.CollectionsOps.{OptionOps, StringOps} - -import scala.collection.JavaConverters._ -import scala.util.{Failure, Success, Try} - -object InfluxDBService { - case class InfluxConf( - url: String, - dbName: String, - userName: Option[String] = None, - password: Option[String] = None, - timeoutSec: Long, - checkTypes: Boolean - ) - - def fetchFieldsTypesInfo(query: String, conf: InfluxConf): Try[Seq[(Symbol, Class[_])]] = for { - db <- connectDb(conf) - series <- fetchFirstSeries(db, query, conf.dbName) - seriesForTypes <- (if (conf.checkTypes) fetchFirstNotNullValueSeries(db, query, conf.dbName) - else fetchFirstSeries(db, query, conf.dbName)) - values <- seriesForTypes.getValues.asScala.headOption.toTry(whenNone = emptyValuesException(query)) - tags = if (series.getTags != null) series.getTags.asScala.toSeq.sortBy(_._1) else Seq.empty - } yield { - val fields = tags.map(_._1) ++ series.getColumns.asScala - val classes = tags.map(_ => classOf[String]) ++ values.asScala.map( - v => if (v != null) v.getClass else classOf[Nothing] - ) - val res = fields.map(Symbol(_)).zip(classes) - res - } - - def fetchFirstSeries(db: InfluxDB, query: String, dbName: String) = { - val influxQuery = new Query(makeLimit1Query(query), dbName) - for { - result <- Try(db.query(influxQuery)) - _ <- if (result.hasError) Failure(new InfluxDBException(result.getError)) - else if (result.getResults == null) Failure(new InfluxDBException(s"Null results of query `$influxQuery`.")) - else Success(()) - // Safely get first series - firstSeries <- result.getResults.asScala.headOption - .flatMap(r => Option(r.getSeries.asScala).flatMap(_.headOption)) - .toTry(whenNone = new InfluxDBException(s"Empty results in query - `$query`.")) - } yield firstSeries - } - - def fetchFirstNotNullValueSeries(db: InfluxDB, query: String, dbName: String) = { - val influxQuery = new Query(makeFirstNotNullQuery(query.replaceAll("(?i)fill\\(previous\\)", "fill(none)")), dbName) - for { - result <- Try(db.query(influxQuery)) - _ <- if (result.hasError) Failure(new InfluxDBException(result.getError)) - else if (result.getResults == null) Failure(new InfluxDBException(s"Null results of query `$influxQuery`.")) - else Success(()) - // Safely get first series - firstSeries <- result.getResults.asScala.headOption - .flatMap(r => Option(r.getSeries.asScala).flatMap(_.headOption)) - .toTry(whenNone = new InfluxDBException(s"Empty results in query - `$query`.")) - } yield firstSeries - } - - def connectDb(conf: InfluxConf) = { - import conf._ - val extraConf = new OkHttpClient.Builder() - .readTimeout(timeoutSec, TimeUnit.SECONDS) - .writeTimeout(timeoutSec, TimeUnit.SECONDS) - .connectTimeout(timeoutSec, TimeUnit.SECONDS) - for { - connection <- Try(InfluxDBFactory.connect(url, userName.orNull, password.orNull, extraConf)) - db <- Try(connection.setDatabase(dbName)) - } yield db - } - - def makeLimit1Query(query: String) = - query.replaceLast("""LIMIT \d+""", "", Pattern.CASE_INSENSITIVE) + " LIMIT 1" - - def makeFirstNotNullQuery(query: String) = s"SELECT first(*) from ($query)" - - def emptyValuesException(query: String) = new InfluxDBException(s"Empty/Null values or tags in query - `$query`.") -} diff --git a/flink/src/main/scala/ru/itclover/tsp/services/JdbcService.scala b/flink/src/main/scala/ru/itclover/tsp/services/JdbcService.scala deleted file mode 100644 index 33ca29b57..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/services/JdbcService.scala +++ /dev/null @@ -1,41 +0,0 @@ -package ru.itclover.tsp.services - -import java.sql.DriverManager - -import scala.util.Try - -object JdbcService { - - def fetchFieldsTypesInfo(driverName: String, jdbcUrl: String, query: String): Try[Seq[(Symbol, Class[_])]] = { - val classTry: Try[Class[_]] = Try(Class.forName(driverName)) - - val connectionTry = Try(DriverManager.getConnection(jdbcUrl)) - for { - _ <- classTry - connection <- connectionTry - resultSet <- Try(connection.createStatement().executeQuery(s"SELECT * FROM (${query}) as mainQ LIMIT 1")) - metaData <- Try(resultSet.getMetaData) - } yield { - (1 to metaData.getColumnCount).map { i: Int => - val className = metaData.getColumnClassName(i) - (Symbol(metaData.getColumnName(i)), Class.forName(className)) - } - } - } - - def fetchAvailableKeys(driverName: String, jdbcUrl: String, query: String, keyColumn: Symbol): Try[Set[Symbol]] = { - val classTry: Try[Class[_]] = Try(Class.forName(driverName)) - - val connectionTry = Try(DriverManager.getConnection(jdbcUrl)) - for { - _ <- classTry - connection <- connectionTry - resultSet <- Try(connection.createStatement().executeQuery(s"SELECT DISTINCT(${keyColumn.name}) FROM (${query})")) - } yield { - new Iterator[String] { - def hasNext = resultSet.next() - def next() = resultSet.getString(1) - }.toStream.map(Symbol(_)).toSet - } - } -} diff --git a/flink/src/main/scala/ru/itclover/tsp/services/KafkaService.scala b/flink/src/main/scala/ru/itclover/tsp/services/KafkaService.scala deleted file mode 100644 index 1fde0f726..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/services/KafkaService.scala +++ /dev/null @@ -1,117 +0,0 @@ -package ru.itclover.tsp.services - -import java.util.Properties - -import scala.util.Try -import org.apache.flink.streaming.connectors.kafka.{FlinkKafkaConsumer, KafkaDeserializationSchema} -import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor} -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.apache.flink.types.Row -import org.apache.flink.util.Collector -import org.apache.kafka.clients.consumer.ConsumerRecord -import ru.itclover.tsp.io.input.KafkaInputConf -import ru.itclover.tsp.serializers.core.{ArrowSerialization, JSONSerialization} - -class StreamEndException(message: String) extends Exception(message) - -object KafkaService { - - // Stub - def fetchFieldsTypesInfo(conf: KafkaInputConf): Try[Seq[(Symbol, Class[_])]] = Try(conf.fieldsTypes.map { - case (fieldName, fieldType) => - val fieldClass = fieldType match { - case "int8" => classOf[Byte] - case "int16" => classOf[Short] - case "int32" => classOf[Int] - case "int64" => classOf[Long] - case "float32" => classOf[Float] - case "float64" => classOf[Double] - case "boolean" => classOf[Boolean] - case "string" => classOf[String] - case _ => classOf[Any] - } - (Symbol(fieldName), fieldClass) - }.toSeq) - - // Stub - def consumer(conf: KafkaInputConf, fieldsIdxMap: Map[Symbol, Int]) = { - val props = new Properties - props.setProperty("bootstrap.servers", conf.brokers) - props.setProperty("group.id", conf.group) - // //props.setProperty("client.id", "client0") - props.setProperty("auto.offset.reset", "earliest") // Always read topic from start if no offset is provided - props.setProperty("enable.auto.commit", "true") // Enable auto committing when checkpointing is disabled - props.setProperty("auto.commit.interval.ms", "1000") // Enable auto committing when checkpointing is disabled - - val deserializer = conf.serializer.getOrElse("json") match { - case "json" => new RowDeserializationSchema(fieldsIdxMap) - case "arrow" => new ArrowRowDeserializationSchema() - case _ => throw new IllegalArgumentException(s"No deserializer for type ${conf.serializer}") - } - - new FlinkKafkaConsumer(conf.topic, deserializer, props) - } -} - -/** - * Deserialization for JSON format - * @param fieldsIdxMap mapping of types from string and scala type - */ -class RowDeserializationSchema(fieldsIdxMap: Map[Symbol, Int]) extends KafkaDeserializationSchema[Row] { - override def deserialize(record: ConsumerRecord[Array[Byte], Array[Byte]]): Row = - new JSONSerialization().deserialize(record.value(), fieldsIdxMap) - - override def isEndOfStream(nextElement: Row): Boolean = false - - override def getProducedType: TypeInformation[Row] = TypeInformation.of(classOf[Row]) -} - -/** - * Deserialization for Apache Arrow format - */ -class ArrowRowDeserializationSchema extends KafkaDeserializationSchema[Row] { - - override def deserialize(record: ConsumerRecord[Array[Byte], Array[Byte]]): Row = - new ArrowSerialization().deserialize(record.value(), null) - - override def isEndOfStream(nextElement: Row): Boolean = false - - override def getProducedType: TypeInformation[Row] = TypeInformation.of(classOf[Row]) -} - -class TimeOutFunction( // delay after which an alert flag is thrown - val timeOut: Long, - timeIndex: Int, - fieldsCount: Int -) extends ProcessFunction[Row, Row] { - // state to remember the last timer set - private var lastTimer: ValueState[Long] = _ - - override def open(conf: Configuration): Unit = { // setup timer state - val lastTimerDesc = new ValueStateDescriptor[Long]("lastTimer", classOf[Long]) - lastTimer = getRuntimeContext.getState(lastTimerDesc) - } - - override def processElement(value: Row, ctx: ProcessFunction[Row, Row]#Context, out: Collector[Row]): Unit = { // get current time and compute timeout time - val currentTime = ctx.timerService.currentProcessingTime - val timeoutTime = currentTime + timeOut - // register timer for timeout time - ctx.timerService.registerProcessingTimeTimer(timeoutTime) - // remember timeout time - lastTimer.update(timeoutTime) - // throughput the event - out.collect(value) - } - - override def onTimer(timestamp: Long, ctx: ProcessFunction[Row, Row]#OnTimerContext, out: Collector[Row]): Unit = { - // check if this was the last timer we registered - if (timestamp == lastTimer.value) { - // it was, so no data was received afterwards. - val sentinel = new Row(fieldsCount) - sentinel.setField(timeIndex, Long.MaxValue) - out.collect(sentinel) - } - } -} diff --git a/flink/src/main/scala/ru/itclover/tsp/transformers/SparseDataAccumulator.scala b/flink/src/main/scala/ru/itclover/tsp/transformers/SparseDataAccumulator.scala deleted file mode 100644 index 4430f4672..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/transformers/SparseDataAccumulator.scala +++ /dev/null @@ -1,203 +0,0 @@ -package ru.itclover.tsp.transformers - -import com.typesafe.scalalogging.Logger -import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor} -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.KeyedProcessFunction -import org.apache.flink.util.Collector -import ru.itclover.tsp.StreamSource -import ru.itclover.tsp.core.io.{Extractor, TimeExtractor} -import ru.itclover.tsp.utils.KeyCreator -//import ru.itclover.tsp.phases.NumericPhases.InKeyNumberExtractor -//import ru.itclover.tsp.EvalUtils -import ru.itclover.tsp.core.Time -//import ru.itclover.tsp.core.Time.TimeNonTransformedExtractor -import ru.itclover.tsp.io.EventCreator -import ru.itclover.tsp.io.input.{NarrowDataUnfolding, WideDataFilling} -//import ru.itclover.tsp.phases.Phases.{AnyExtractor, AnyNonTransformedExtractor} -import ru.itclover.tsp.core.io.AnyDecodersInstances.decodeToAny - -import scala.collection.mutable -import scala.util.{Success, Try} - -trait SparseDataAccumulator - -/** - * Accumulates sparse key-value format into dense Row using timeouts. - * @param fieldsKeysTimeoutsMs - indexes to collect and timeouts (milliseconds) per each (collect by-hand for now) - * @param extraFieldNames - will be added to every emitting event - */ -class SparseRowsDataAccumulator[InEvent, InKey, Value, OutEvent]( - fieldsKeysTimeoutsMs: Map[InKey, Long], - extraFieldNames: Seq[InKey], - useUnfolding: Boolean, - defaultTimeout: Option[Long] -)( - implicit extractTime: TimeExtractor[InEvent], - extractKeyAndVal: InEvent => (InKey, Value), - extractValue: Extractor[InEvent, InKey, Value], - eventCreator: EventCreator[OutEvent, InKey], - keyCreator: KeyCreator[InKey] -) extends KeyedProcessFunction[String, InEvent, OutEvent] - with Serializable { - // potential event values with receive time - val event: mutable.Map[InKey, (Value, Time)] = mutable.Map.empty - val targetKeySet: Set[InKey] = fieldsKeysTimeoutsMs.keySet - val keysIndexesMap: Map[InKey, Int] = targetKeySet.zip(0 until targetKeySet.size).toMap - - val extraFieldsIndexesMap: Map[InKey, Int] = extraFieldNames - .zip( - targetKeySet.size until - targetKeySet.size + extraFieldNames.size - ) - .toMap - val allFieldsIndexesMap: Map[InKey, Int] = keysIndexesMap ++ extraFieldsIndexesMap - val arity: Int = fieldsKeysTimeoutsMs.size + extraFieldNames.size - - val log = Logger("SparseDataAccumulator") - - var lastTimestamp = Time(Long.MinValue) - var lastEvent: OutEvent = _ - private var lastTimer: ValueState[Long] = _ - - override def open(conf: Configuration): Unit = { // setup timer state - val lastTimerDesc = new ValueStateDescriptor[Long]("lastTimer", classOf[Long]) - lastTimer = getRuntimeContext.getState(lastTimerDesc) - } - - override def processElement( - item: InEvent, - ctx: KeyedProcessFunction[String, InEvent, OutEvent]#Context, - out: Collector[OutEvent] - ): Unit = { - val time = extractTime(item) - if (useUnfolding) { - val (key, value) = extractKeyAndVal(item) - if (event.get(key).orNull == null || value != null) event(key) = (value, time) - } else { - allFieldsIndexesMap.keySet.foreach { key => - val newValue = Try(extractValue(item, key)) - newValue match { - case Success(nv) if nv != null || !event.contains(key) => event(key) = (nv.asInstanceOf[Value], time) - case _ => // do nothing - } - } - } - dropExpiredKeys(event, time) - val list = mutable.ListBuffer.tabulate[(InKey, AnyRef)](arity)(x => (keyCreator.create(s"empty_$x"), null)) - val indexesMap = if (defaultTimeout.isDefined) allFieldsIndexesMap else keysIndexesMap - event.foreach { - case (k, (v, _)) if indexesMap.contains(k) => list(indexesMap(k)) = (k, v.asInstanceOf[AnyRef]) - case _ => - } - //if (defaultTimeout.isEmpty) { - extraFieldNames.foreach { name => - val value = extractValue(item, name) - if (value != null) list(extraFieldsIndexesMap(name)) = (name, value.asInstanceOf[AnyRef]) - } - //} - val outEvent = eventCreator.create(list) - if (lastTimestamp.toMillis != time.toMillis && lastEvent != null) { - out.collect(lastEvent) - } - lastTimestamp = time - lastEvent = outEvent - ctx.timerService().registerProcessingTimeTimer(ctx.timerService.currentProcessingTime + 5000) - } - - private def dropExpiredKeys(event: mutable.Map[InKey, (Value, Time)], currentRowTime: Time): Unit = { - event.retain( - (k, v) => currentRowTime.toMillis - v._2.toMillis < fieldsKeysTimeoutsMs.getOrElse(k, defaultTimeout.getOrElse(0L)) - ) - } - - override def onTimer( - timestamp: Long, - ctx: KeyedProcessFunction[String, InEvent, OutEvent]#OnTimerContext, - out: Collector[OutEvent] - ): Unit = { - // check if this was the last timer we registered - if (timestamp == lastTimer.value) { - // it was, so no data was received afterwards. - // collect the last - out.collect(lastEvent) - } - } - -} - -object SparseRowsDataAccumulator { - - def apply[InEvent, InKey, Value, OutEvent: TypeInformation]( - streamSource: StreamSource[InEvent, InKey, Value], - patternFields: Set[InKey] - )( - implicit timeExtractor: TimeExtractor[InEvent], - extractKeyVal: InEvent => (InKey, Value), - extractAny: Extractor[InEvent, InKey, Value], - eventCreator: EventCreator[OutEvent, InKey], - keyCreator: KeyCreator[InKey] - ): SparseRowsDataAccumulator[InEvent, InKey, Value, OutEvent] = { - streamSource.conf.dataTransformation - .map({ - case ndu: NarrowDataUnfolding[InEvent, InKey, _] => - val sparseRowsConf = ndu - val fim = streamSource.fieldsIdxMap - val timeouts = patternFields - .map(k => (k, ndu.defaultTimeout.getOrElse(0L))) - .toMap[InKey, Long] ++ - ndu.fieldsTimeoutsMs - val extraFields = fim - .filterNot { - case (name, _) => name == sparseRowsConf.keyColumn || name == sparseRowsConf.defaultValueColumn - } - .keys - .toSeq - new SparseRowsDataAccumulator( - timeouts, - extraFields.map(streamSource.fieldToEKey), - useUnfolding = true, - defaultTimeout = ndu.defaultTimeout - )( - timeExtractor, - extractKeyVal, - extractAny, - eventCreator, - keyCreator - ) - case wdf: WideDataFilling[InEvent, InKey, _] => - val sparseRowsConf = wdf - val fim = streamSource.fieldsIdxMap - val toKey = streamSource.fieldToEKey - val timeouts = patternFields - .map(k => (k, wdf.defaultTimeout.getOrElse(0L))) - .toMap[InKey, Long] ++ - wdf.fieldsTimeoutsMs - val extraFields = - fim - .filterNot { - case (name, _) => sparseRowsConf.fieldsTimeoutsMs.contains(toKey(name)) - } - .keys - .toSeq - new SparseRowsDataAccumulator( - timeouts, - extraFields.map(streamSource.fieldToEKey), - useUnfolding = false, - defaultTimeout = wdf.defaultTimeout - )( - timeExtractor, - extractKeyVal, - extractAny, - eventCreator, - keyCreator - ) - case _ => - sys.error( - s"Invalid config type: expected NarrowDataUnfolding or WideDataFilling, got ${streamSource.conf.dataTransformation} instead" - ) - }) - .getOrElse(sys.error("No data transformation config specified")) - } -} diff --git a/flink/src/main/scala/ru/itclover/tsp/utils/ArrowOps.scala b/flink/src/main/scala/ru/itclover/tsp/utils/ArrowOps.scala deleted file mode 100644 index fe9b193b9..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/utils/ArrowOps.scala +++ /dev/null @@ -1,271 +0,0 @@ -package ru.itclover.tsp.utils - -import java.io.{ByteArrayInputStream, File, FileInputStream, FileOutputStream} - -import org.apache.arrow.memory.RootAllocator -import org.apache.arrow.vector.dictionary.DictionaryProvider -import org.apache.arrow.vector.ipc.{ - ArrowFileReader, - ArrowFileWriter, - ArrowReader, - ArrowStreamReader, - SeekableReadChannel -} -import org.apache.arrow.vector.{ - BaseValueVector, - BigIntVector, - BitVector, - FieldVector, - Float4Vector, - Float8Vector, - IntVector, - SmallIntVector, - VarCharVector, - VectorDefinitionSetter, - VectorSchemaRoot -} -import org.apache.arrow.vector.types.Types -import org.apache.arrow.vector.types.pojo.Schema -import org.apache.arrow.vector.util.Text -import org.apache.flink.types.Row - -import scala.collection.JavaConverters._ -import scala.collection.mutable - -object ArrowOps { - - /** - * Types mapping from Apache Arrow to Scala types - * @return - */ - def typesMap = Map( - Types.MinorType.BIGINT -> classOf[Long], - Types.MinorType.SMALLINT -> classOf[Short], - Types.MinorType.BIT -> classOf[Boolean], - Types.MinorType.INT -> classOf[Int], - Types.MinorType.VARCHAR -> classOf[String], - Types.MinorType.FLOAT4 -> classOf[Float], - Types.MinorType.FLOAT8 -> classOf[Double] - ) - - /** - * Method for retrieving typed value from vector - * Uses asInstanceOf for converting, so disabling the wart warning - * @param valueVector vector with raw value - * @return typed value - */ - @SuppressWarnings(Array("org.wartremover.warts.AsInstanceOf")) - def retrieveFieldValue(valueVector: FieldVector): BaseValueVector with FieldVector with VectorDefinitionSetter = { - - val valueInfo = typesMap(valueVector.getMinorType) - - valueInfo.getName match { - case "int" => valueVector.asInstanceOf[IntVector] - case "boolean" => valueVector.asInstanceOf[BitVector] - case "java.lang.String" => valueVector.asInstanceOf[VarCharVector] - case "float" => valueVector.asInstanceOf[Float4Vector] - case "double" => valueVector.asInstanceOf[Float8Vector] - case "long" => valueVector.asInstanceOf[BigIntVector] - case "short" => valueVector.asInstanceOf[SmallIntVector] - case _ => throw new IllegalArgumentException(s"No mapper for type ${valueInfo.getName}") - } - - } - - /** - * Method for retrieving schema and reader from input file - * @param input file with input data - * @param allocatorValue value for root allocator - * @return tuple with schema and reader - */ - def retrieveSchemaAndReader(input: File, allocatorValue: Int): (Schema, ArrowReader, RootAllocator) = { - - val allocator = new RootAllocator(allocatorValue) - val fileStream = new FileInputStream(input) - - val readChannel = new SeekableReadChannel(fileStream.getChannel) - - val reader = new ArrowFileReader(readChannel, allocator) - - (reader.getVectorSchemaRoot.getSchema, reader, allocator) - - } - - /** - * Method for retrieving schema and reader from bytes - * @param input bytes with input data - * @param allocatorValue value for root allocator - * @return tuple with schema and reader - */ - def retrieveSchemaAndReader(input: Array[Byte], allocatorValue: Int): (Schema, ArrowReader, RootAllocator) = { - - val allocator = new RootAllocator(allocatorValue) - val inputStream = new ByteArrayInputStream(input) - - val reader = new ArrowStreamReader(inputStream, allocator) - val schema = reader.getVectorSchemaRoot.getSchema - - (schema, reader, allocator) - - } - - /** - * Method for schema fields - * @param schema schema from Apache Arrow - * @return list of fields(string) - */ - def getSchemaFields(schema: Schema): List[String] = { - - schema.getFields.asScala - .map(_.getName) - .toList - - } - - /** - * Retrieve data in Apache Flink rows - * @param input arrow schema and reader - * @return flink rows - */ - def retrieveData(input: (Schema, ArrowReader, RootAllocator)): mutable.ListBuffer[Row] = { - - val (schema, reader, allocator) = input - val schemaFields = getSchemaFields(schema) - - val schemaRoot = reader.getVectorSchemaRoot - var rowCount = 0 - - var readCondition = reader.loadNextBatch() - val result: mutable.ListBuffer[Row] = mutable.ListBuffer.empty[Row] - val objectsList: mutable.ListBuffer[Any] = mutable.ListBuffer.empty[Any] - - while (readCondition) { - - rowCount = schemaRoot.getRowCount - - for (i <- 0 until rowCount) { - - for (field <- schemaFields) { - - val valueVector = schemaRoot.getVector(field) - objectsList += retrieveFieldValue(valueVector).getObject(i) - - } - - val row = new Row(objectsList.size) - for (i <- objectsList.indices) { - row.setField(i, objectsList(i)) - } - - result += row - objectsList.clear() - - } - - readCondition = reader.loadNextBatch() - - } - - reader.close() - allocator.close() - - result - - } - - /** - * Method for writing data to Apache Arrow file - * @param input file for data, schema for data, data, allocator - */ - def writeData(input: (File, Schema, mutable.ListBuffer[mutable.Map[String, Any]], RootAllocator)): Unit = { - - val (inputFile, schema, data, allocator) = input - - val outStream = new FileOutputStream(inputFile) - val rootSchema = VectorSchemaRoot.create(schema, allocator) - val provider = new DictionaryProvider.MapDictionaryProvider() - - val dataWriter = new ArrowFileWriter( - rootSchema, - provider, - outStream.getChannel - ) - - var counter = 0 - - dataWriter.start() - - for (item <- data) { - - val fields = rootSchema.getSchema.getFields.asScala - - for (field <- fields) { - - if (item.contains(field.getName)) { - - val data = item(field.getName) - val vector = rootSchema.getVector(field.getName) - - val valueInfo = typesMap(vector.getMinorType) - - valueInfo.getName match { - - case "int" => - val tempVector = vector.asInstanceOf[IntVector] - tempVector.setSafe(counter, data.asInstanceOf[Int]) - - case "boolean" => - var value: Int = 0 - if (!data.asInstanceOf[Boolean]) { - value = 1 - } - - val tempVector = vector.asInstanceOf[BitVector] - tempVector.setSafe(counter, value) - - case "java.lang.String" => - val tempVector = vector.asInstanceOf[VarCharVector] - tempVector.setSafe( - counter, - new Text(data.asInstanceOf[String]) - ) - - case "float" => - val tempVector = vector.asInstanceOf[Float4Vector] - tempVector.setSafe(counter, data.asInstanceOf[Float]) - - case "double" => - val tempVector = vector.asInstanceOf[Float8Vector] - tempVector.setSafe(counter, data.asInstanceOf[Double]) - - case "long" => - val tempVector = vector.asInstanceOf[BigIntVector] - tempVector.setSafe(counter, data.asInstanceOf[Long]) - - case "short" => { - val tempVector = vector.asInstanceOf[SmallIntVector] - tempVector.setSafe(counter, data.asInstanceOf[Short]) - } - - case _ => throw new IllegalArgumentException(s"No mapper for type ${valueInfo.getName}") - } - - counter += 1 - - dataWriter.writeBatch() - - } - - } - - } - - dataWriter.end() - dataWriter.close() - - outStream.flush() - outStream.close() - - } - -} diff --git a/flink/src/main/scala/ru/itclover/tsp/utils/DataStreamOps.scala b/flink/src/main/scala/ru/itclover/tsp/utils/DataStreamOps.scala deleted file mode 100644 index 6e55666ca..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/utils/DataStreamOps.scala +++ /dev/null @@ -1,28 +0,0 @@ -package ru.itclover.tsp.utils - -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.streaming.api.functions.timestamps.AscendingTimestampExtractor -import org.apache.flink.streaming.api.scala.DataStream -import ru.itclover.tsp.DebugTsViolationHandler - -object DataStreamOps { - - implicit class DataStreamOps[E: TypeInformation](val stream: DataStream[E]) { - -// def flatMapIf(cond: Boolean, mapper: => FlatMapFunction[E, E]): DataStream[E] = { -// if (cond) { stream.flatMap(mapper) } else { stream } -// } - -// def foreach(fn: E => Unit): DataStream[Unit] = stream.map[Unit](fn)(new UnitTypeInfo) // It seems to be unneeded - - def assignAscendingTimestamps_withoutWarns(extractor: E => Long): DataStream[E] = { - // Careful! Closure cleaner is disabled bcs it's private for Flink for some stupid reason // val cleanExtractor = stream.clean(extractor) - val extractorFunction = new AscendingTimestampExtractor[E] { - def extractAscendingTimestamp(element: E): Long = - extractor(element) - }.withViolationHandler(new DebugTsViolationHandler()) - stream.assignTimestampsAndWatermarks(extractorFunction) - } - - } -} diff --git a/flink/src/main/scala/ru/itclover/tsp/utils/ErrorsADT.scala b/flink/src/main/scala/ru/itclover/tsp/utils/ErrorsADT.scala deleted file mode 100644 index 24add85b6..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/utils/ErrorsADT.scala +++ /dev/null @@ -1,39 +0,0 @@ -package ru.itclover.tsp.utils - -object ErrorsADT { - - sealed trait Err extends Product with Serializable { - val error: String - val errorCode: Int - } - - /** - * Represents errors on configuration stage. It is guaranteed that no events is written in sink if error occurs. - */ - sealed trait ConfigErr extends Err - - case class InvalidRequest(error: String, errorCode: Int = 4010) extends ConfigErr - - case class InvalidPatternsCode(errors: Seq[String], errorCode: Int = 4020) extends ConfigErr { - override val error = errors.mkString("\n") - } - - case class SourceUnavailable(error: String, errorCode: Int = 4030) extends ConfigErr - - case class SinkUnavailable(error: String, errorCode: Int = 4040) extends ConfigErr - - case class GenericConfigError(ex: Exception, errorCode: Int = 4000) extends ConfigErr { - override val error = Option(ex.getMessage).getOrElse(ex.toString) - } - - /** - * Represents errors on run-time stage. Some data could be already written in the sink. - */ - sealed trait RuntimeErr extends Err - - case class GenericRuntimeErr(ex: Throwable, errorCode: Int = 5000) extends RuntimeErr { - override val error = Option(ex.getMessage).getOrElse(ex.toString) - } - - // case class RecoverableError() extends RuntimeError -} diff --git a/flink/src/main/scala/ru/itclover/tsp/utils/EventCounterTrigger.scala b/flink/src/main/scala/ru/itclover/tsp/utils/EventCounterTrigger.scala deleted file mode 100644 index 89bf9f99c..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/utils/EventCounterTrigger.scala +++ /dev/null @@ -1,35 +0,0 @@ -package ru.itclover.tsp.utils -import org.apache.flink.streaming.api.windowing.triggers.{Trigger, TriggerResult} -import org.apache.flink.streaming.api.windowing.windows.Window - -case class EventCounterTrigger[T, W <: Window](maxElements: Long) extends Trigger[T, W] { - var elements: Long = 0L - - override def onElement( - element: T, - timestamp: Long, - window: W, - ctx: Trigger.TriggerContext - ): TriggerResult = - if (elements < maxElements) { - elements += 1 - TriggerResult.CONTINUE - } else { - elements = 0 - TriggerResult.FIRE_AND_PURGE - } - override def onProcessingTime( - time: Long, - window: W, - ctx: Trigger.TriggerContext - ): TriggerResult = TriggerResult.CONTINUE - override def onEventTime( - time: Long, - window: W, - ctx: Trigger.TriggerContext - ): TriggerResult = TriggerResult.CONTINUE - override def clear( - window: W, - ctx: Trigger.TriggerContext - ): Unit = elements = 0 -} diff --git a/flink/src/main/scala/ru/itclover/tsp/utils/KeyCreator.scala b/flink/src/main/scala/ru/itclover/tsp/utils/KeyCreator.scala deleted file mode 100644 index cc8f5c77e..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/utils/KeyCreator.scala +++ /dev/null @@ -1,16 +0,0 @@ -package ru.itclover.tsp.utils -import scala.util.Try - -trait KeyCreator[Key] extends Serializable { - def create(keyName: String): Key -} - -object KeyCreatorInstances { - implicit val intKeyCreator: KeyCreator[Int] = new KeyCreator[Int] { - override def create(keyName: String): Int = Try(keyName.toInt).getOrElse(0) - } - - implicit val symbolKeyCreator: KeyCreator[Symbol] = new KeyCreator[Symbol] { - override def create(keyName: String): Symbol = Symbol(keyName) - } -} diff --git a/flink/src/main/scala/ru/itclover/tsp/utils/RowOps.scala b/flink/src/main/scala/ru/itclover/tsp/utils/RowOps.scala deleted file mode 100644 index 34eb07301..000000000 --- a/flink/src/main/scala/ru/itclover/tsp/utils/RowOps.scala +++ /dev/null @@ -1,54 +0,0 @@ -package ru.itclover.tsp.utils - -import java.time.Instant - -import org.apache.flink.types.Row -import ru.itclover.tsp.core.io.{Decoder, Extractor, TimeExtractor} -import ru.itclover.tsp.core.{Time => CoreTime} - -import scala.util.Try - -object RowOps { - implicit class RowOps(private val row: Row) extends AnyVal { - - def getFieldOrThrow(i: Int): AnyRef = - if (row.getArity > i) row.getField(i) - else throw new RuntimeException(s"Cannot extract $i from row ${row.mkString}") - - def mkString(sep: String): String = "Row(" + (0 until row.getArity).map(row.getField).mkString(sep) + ")" - - def mkString: String = mkString(", ") - } - - case class RowTsTimeExtractor(timeIndex: Int, tsMultiplier: Double, fieldId: Symbol) extends TimeExtractor[Row] { - - def apply(r: Row) = { - val millis = r.getField(timeIndex) match { - case d: java.lang.Double => (d * tsMultiplier).toLong - case f: java.lang.Float => (f * tsMultiplier).toLong - case n: java.lang.Number => (n.doubleValue() * tsMultiplier).toLong - case null => 0L // TODO: Where can nulls come from? - case x => sys.error(s"Cannot parse time `$x` from field $fieldId, should be number of millis since 1.1.1970") - } - CoreTime(toMillis = millis) - } - } - - case class RowIsoTimeExtractor(timeIndex: Int, fieldId: Symbol) extends TimeExtractor[Row] { - - def apply(r: Row) = { - val isoTime = r.getField(timeIndex).toString - if (isoTime == null || isoTime == "") - sys.error(s"Cannot parse time `$isoTime` from field $fieldId, should be in ISO 8601 format") - CoreTime(toMillis = Instant.parse(isoTime).toEpochMilli) - } - } - - case class RowSymbolExtractor(fieldIdxMap: Map[Symbol, Int]) extends Extractor[Row, Symbol, Any] { - def apply[T](r: Row, s: Symbol)(implicit d: Decoder[Any, T]): T = d(r.getField(fieldIdxMap(s))) - } - - case class RowIdxExtractor() extends Extractor[Row, Int, Any] { - def apply[T](r: Row, i: Int)(implicit d: Decoder[Any, T]): T = d(r.getFieldOrThrow(i)) - } -} diff --git a/flink/src/test/resources/arrow/aaa2 b/flink/src/test/resources/arrow/aaa2 deleted file mode 100644 index e8d2ff0c7..000000000 Binary files a/flink/src/test/resources/arrow/aaa2 and /dev/null differ diff --git a/flink/src/test/resources/arrow/df_billion b/flink/src/test/resources/arrow/df_billion deleted file mode 100644 index d15adaab2..000000000 Binary files a/flink/src/test/resources/arrow/df_billion and /dev/null differ diff --git a/flink/src/test/resources/log4j2-test.xml b/flink/src/test/resources/log4j2-test.xml deleted file mode 100644 index 2c9d75704..000000000 --- a/flink/src/test/resources/log4j2-test.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/flink/src/test/resources/parquet/df_billion.parquet b/flink/src/test/resources/parquet/df_billion.parquet deleted file mode 100644 index 9867bb685..000000000 Binary files a/flink/src/test/resources/parquet/df_billion.parquet and /dev/null differ diff --git a/flink/src/test/resources/parquet/df_billion_brotli.parquet b/flink/src/test/resources/parquet/df_billion_brotli.parquet deleted file mode 100644 index 4df3a7b48..000000000 Binary files a/flink/src/test/resources/parquet/df_billion_brotli.parquet and /dev/null differ diff --git a/flink/src/test/resources/parquet/df_billion_gzip.parquet b/flink/src/test/resources/parquet/df_billion_gzip.parquet deleted file mode 100644 index b0883291b..000000000 Binary files a/flink/src/test/resources/parquet/df_billion_gzip.parquet and /dev/null differ diff --git a/flink/src/test/resources/parquet/df_billion_snappy.parquet b/flink/src/test/resources/parquet/df_billion_snappy.parquet deleted file mode 100644 index 9867bb685..000000000 Binary files a/flink/src/test/resources/parquet/df_billion_snappy.parquet and /dev/null differ diff --git a/flink/src/test/scala/ru/itclover/tsp/services/FileServiceTest.scala b/flink/src/test/scala/ru/itclover/tsp/services/FileServiceTest.scala deleted file mode 100644 index c94633325..000000000 --- a/flink/src/test/scala/ru/itclover/tsp/services/FileServiceTest.scala +++ /dev/null @@ -1,26 +0,0 @@ -package ru.itclover.tsp.services - -import org.scalatest.{Matchers, WordSpec} - -import scala.util.Random - -class FileServiceTest extends WordSpec with Matchers { - - "FileService" should { - - "convert bytes to file" in { - - val randomBytes = Array.fill(20)((Random.nextInt(256) - 128).toByte) - val resultFile = FileService.convertBytes(randomBytes) - - assert(resultFile.exists()) - assert(resultFile.getName.contains("temp")) - assert(resultFile.length() > 0) - - resultFile.delete() - - } - - } - -} diff --git a/flink/src/test/scala/ru/itclover/tsp/utils/ArrowOpsTest.scala b/flink/src/test/scala/ru/itclover/tsp/utils/ArrowOpsTest.scala deleted file mode 100644 index 1dd99a739..000000000 --- a/flink/src/test/scala/ru/itclover/tsp/utils/ArrowOpsTest.scala +++ /dev/null @@ -1,97 +0,0 @@ -package ru.itclover.tsp.utils - -import java.io.File -import java.nio.file.{Files => JavaFiles} - -import org.apache.arrow.memory.RootAllocator -import org.apache.arrow.vector.ipc.{ArrowFileReader, ArrowStreamReader} -import org.apache.arrow.vector.types.pojo.{ArrowType, Field, Schema} -import org.scalatest.{Matchers, WordSpec} -import ru.itclover.tsp.services.FileService - -import scala.collection.JavaConverters._ -import scala.collection.mutable - -class ArrowOpsTest extends WordSpec with Matchers { - - val testFiles: List[File] = new File( - "flink/src/test/resources/arrow" - ).listFiles() - .filter(_.isFile) - .toList - - "ArrowOps" should { - - "retrieve schema and reader from file" in { - - testFiles.foreach(file => { - - val schemaAndReader = ArrowOps.retrieveSchemaAndReader(file, Integer.MAX_VALUE) - - schemaAndReader._1.getClass shouldBe classOf[Schema] - schemaAndReader._2.getClass shouldBe classOf[ArrowFileReader] - schemaAndReader._3.getClass shouldBe classOf[RootAllocator] - - }) - - } - -// "retrieve data" in { -// -// testFiles.foreach(file => { -// -// val schemaAndReader = ArrowOps.retrieveSchemaAndReader(file, Integer.MAX_VALUE) -// val rowData = ArrowOps.retrieveData(schemaAndReader) -// -// rowData.head.getArity shouldBe 3 -// -// }) -// -// } -// -// "write data" in { -// -// val tempPath = FileService.createTemporaryFile() -// val tempFile = tempPath.toFile -// -// val schemaFields = List( -// new Field( -// "a", -// false, -// new ArrowType.Int(32, true), -// null -// ), -// new Field( -// "b", -// false, -// new ArrowType.Utf8, -// null -// ) -// ) -// -// val schema = new Schema(schemaFields.asJava) -// val allocator = new RootAllocator(1000000) -// -// val data = mutable.ListBuffer( -// mutable.Map( -// "a" -> 4, -// "b" -> "test" -// ), -// mutable.Map( -// "a" -> 5, -// "b" -> "test1" -// ) -// ) -// -// ArrowOps.writeData((tempFile, schema, data, allocator)) -// -// val schemaAndReader = ArrowOps.retrieveSchemaAndReader(tempFile, Integer.MAX_VALUE) -// ArrowOps.retrieveData(schemaAndReader) -// -// tempFile.delete() -// -// } -// - } - -} diff --git a/flink/src/test/scala/ru/itclover/tsp/utils/ErrorsADTTest.scala b/flink/src/test/scala/ru/itclover/tsp/utils/ErrorsADTTest.scala deleted file mode 100644 index b79b6a41f..000000000 --- a/flink/src/test/scala/ru/itclover/tsp/utils/ErrorsADTTest.scala +++ /dev/null @@ -1,25 +0,0 @@ -package ru.itclover.tsp.utils -import org.scalatest.{FlatSpec, Matchers} -import ru.itclover.tsp.utils.ErrorsADT._ - -class ErrorsADTTest extends FlatSpec with Matchers { - "Request errors" should "inform the user correctly" in { - InvalidRequest("test").errorCode shouldBe 4010 - InvalidRequest("test", 4011).errorCode shouldBe 4011 - InvalidPatternsCode(Seq("one", "two", "three")).errorCode shouldBe 4020 - InvalidPatternsCode(Seq("one", "two", "three"), 4021).errorCode shouldBe 4021 - InvalidPatternsCode(Seq("one", "two", "three")).error shouldBe "one\ntwo\nthree" - SourceUnavailable("test").errorCode shouldBe 4030 - SourceUnavailable("test", 4031).errorCode shouldBe 4031 - SinkUnavailable("test").errorCode shouldBe 4040 - SinkUnavailable("test", 4041).errorCode shouldBe 4041 - GenericConfigError(new Exception()).errorCode shouldBe 4000 - GenericConfigError(new Exception(), 4001).errorCode shouldBe 4001 - GenericConfigError(new Exception()).error shouldBe "java.lang.Exception" - GenericConfigError(new Exception("message")).error shouldBe "message" - GenericRuntimeErr(new Exception()).errorCode shouldBe 5000 - GenericRuntimeErr(new Exception(), 5001).errorCode shouldBe 5001 - GenericRuntimeErr(new Exception()).error shouldBe "java.lang.Exception" - GenericRuntimeErr(new Exception("message")).error shouldBe "message" - } -} diff --git a/http/src/main/scala/ru/itclover/tsp/http/HttpService.scala b/http/src/main/scala/ru/itclover/tsp/http/HttpService.scala index f874d4565..e32fb8d95 100644 --- a/http/src/main/scala/ru/itclover/tsp/http/HttpService.scala +++ b/http/src/main/scala/ru/itclover/tsp/http/HttpService.scala @@ -9,8 +9,6 @@ import akka.stream.ActorMaterializer import cats.data.Reader import com.typesafe.config.ConfigFactory import com.typesafe.scalalogging.Logger -import org.apache.flink.runtime.client.JobExecutionException -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.spark.sql.SparkSession import ru.itclover.tsp.http.UtilsDirectives.{logRequest, logResponse} import ru.itclover.tsp.http.domain.output.FailureResponse @@ -26,7 +24,6 @@ import scala.util.Properties trait HttpService extends RoutesProtocols { implicit val system: ActorSystem implicit val materializer: ActorMaterializer - implicit val streamEnvironment: StreamExecutionEnvironment implicit val executionContext: ExecutionContextExecutor val blockingExecutorContext: ExecutionContextExecutor @@ -119,18 +116,6 @@ trait HttpService extends RoutesProtocols { ) ) - case ex: JobExecutionException => - val stackTrace = Exceptions.getStackTrace(ex) - val msg = if (ex.getCause != null) ex.getCause.getLocalizedMessage else ex.getMessage - val error = s"Uncaught error during job execution, cause - `${msg}`, \n\nstacktrace: `$stackTrace`" - log.error(error) - complete( - ( - InternalServerError, - FailureResponse(5002, "Job execution failure", if (!isHideExceptions) Seq(error) else Seq.empty) - ) - ) - case InvalidRequest(msg) => log.error(msg) complete((BadRequest, FailureResponse(4005, "Invalid request", Seq(msg)))) diff --git a/http/src/main/scala/ru/itclover/tsp/http/Launcher.scala b/http/src/main/scala/ru/itclover/tsp/http/Launcher.scala index 4b0d185d0..a7a980460 100644 --- a/http/src/main/scala/ru/itclover/tsp/http/Launcher.scala +++ b/http/src/main/scala/ru/itclover/tsp/http/Launcher.scala @@ -10,8 +10,6 @@ import cats.implicits._ import com.google.common.util.concurrent.ThreadFactoryBuilder import com.typesafe.config.ConfigFactory import com.typesafe.scalalogging.Logger -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.spark.sql.SparkSession import ru.itclover.tsp.spark.StreamSource @@ -74,39 +72,19 @@ object Launcher extends App with HttpService { ) ) - val streamEnvOrError = if (args.length > 0 && args(0) == "flink-cluster-test") { - val (host, port) = getClusterHostPort match { - case Right(hostAndPort) => hostAndPort - case Left(err) => throw new RuntimeException(err) - } - log.info(s"Starting TEST TSP on cluster Flink: $host:$port with monitoring in $monitoringUri") - Right(StreamExecutionEnvironment.createRemoteEnvironment(host, port, args(1))) - } else if (args.length != 1) { + val streamEnvOrError = if (args.length != 1) { Left( - "You need to provide one arg: `flink-xxx spark-xxx` where `xxx` can be `local` or `cluster` " + - "to specify Flink and Spark execution mode." + "You need to provide one arg: `spark-xxx` where `xxx` can be `local` or `cluster` " + + "to specify Spark execution mode." ) // TODO: More beautiful parsing - } else if (args(0) == "flink-local spark-local") { - createLocalEnv - } else if (args(0) == "flink-cluster spark-local") { - createClusterEnv - } else if (args(0) == "flink-local spark-cluster") { - useLocalSpark = false - createLocalEnv - } else if (args(0) == "flink-cluster spark-cluster") { + } else if (args(0) == "spark-local") { + } else if (args(0) == "spark-cluster") { useLocalSpark = false - createClusterEnv } else { Left(s"Unknown argument: `${args(0)}`.") } - implicit override val streamEnvironment = streamEnvOrError match { - case Right(env) => env - case Left(err) => throw new RuntimeException(err) - } - streamEnvironment.setParallelism(1) - streamEnvironment.setMaxParallelism(1) //(configs.getInt("flink.max-parallelism")) val spark = sparkSession @@ -185,65 +163,4 @@ object Launcher extends App with HttpService { .getOrCreate() } } - - /** - * Method for flink environment configuration - * @param env flink execution environment - */ - def configureEnv(env: StreamExecutionEnvironment): StreamExecutionEnvironment = { - -// env.enableCheckpointing(500) -// -// val flinkParameters = Try(env.getConfig.getGlobalJobParameters.toMap.asScala).getOrElse(Map.empty[String, String]) -// -// val config = env.getCheckpointConfig -// config.enableExternalizedCheckpoints(ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION) -// config.setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE) -// config.setMinPauseBetweenCheckpoints(250) -// config.setCheckpointTimeout(60000) -// config.setTolerableCheckpointFailureNumber(5) -// config.setMaxConcurrentCheckpoints(5) -// -// var savePointsPath = "" -// -// if(flinkParameters.contains("state.savepoints.dir")){ -// savePointsPath = flinkParameters("state.savepoints.dir") -// }else{ -// savePointsPath = getEnvVarOrConfig("FLINK_SAVEPOINTS_PATH", "flink.savepoints-dir") -// } -// -// val expectedStorages = Seq("s3", "hdfs", "file") -// -// if (savePointsPath.nonEmpty) { -// val storageIndex = savePointsPath.indexOf(":") -// val inputStorageType = savePointsPath.substring(0, storageIndex) -// -// if(!expectedStorages.contains(inputStorageType)){ -// throw new IllegalArgumentException(s"Unsupported type for checkpointing: ${inputStorageType}") -// } -// env.setStateBackend(new RocksDBStateBackend(savePointsPath)) -// } -// env.setRestartStrategy(RestartStrategies.noRestart) - env - - } - - def createClusterEnv: Either[String, StreamExecutionEnvironment] = getClusterHostPort.flatMap { - case (clusterHost, clusterPort) => - log.info(s"Starting TSP on cluster Flink: $clusterHost:$clusterPort with monitoring in $monitoringUri") - val rawJarPath = this.getClass.getProtectionDomain.getCodeSource.getLocation.getPath - val jarPath = URLDecoder.decode(rawJarPath, "UTF-8") - - Either.cond( - jarPath.endsWith(".jar"), - configureEnv(StreamExecutionEnvironment.createRemoteEnvironment(clusterHost, clusterPort, jarPath)), - s"Jar path is invalid: `$jarPath` (no jar extension)" - ) - } - - def createLocalEnv: Either[String, StreamExecutionEnvironment] = { - val config = new Configuration() - log.info(s"Starting local Flink with monitoring in $monitoringUri") - Right(configureEnv(StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(config))) - } } diff --git a/http/src/main/scala/ru/itclover/tsp/http/domain/output/Response.scala b/http/src/main/scala/ru/itclover/tsp/http/domain/output/Response.scala index b17aa6a7e..77ebd39e3 100644 --- a/http/src/main/scala/ru/itclover/tsp/http/domain/output/Response.scala +++ b/http/src/main/scala/ru/itclover/tsp/http/domain/output/Response.scala @@ -2,7 +2,6 @@ package ru.itclover.tsp.http.domain.output import akka.http.scaladsl.model.StatusCodes.ServerError import ru.itclover.tsp.http.utils.Exceptions -import ru.itclover.tsp.utils.ErrorsADT.{ConfigErr, RuntimeErr} import ru.itclover.tsp.spark.utils.ErrorsADT.{ConfigErr => SparkConfErr, RuntimeErr => SparkRTErr} sealed trait Response[T] extends Product with Serializable @@ -34,21 +33,6 @@ object FailureResponse { def apply(ex: Throwable): FailureResponse = apply(5000, ex) - def apply(err: ConfigErr): FailureResponse = { - val msg = makeConfigErrMsg(err.getClass.getName) - FailureResponse(err.errorCode, msg, Seq(err.error)) - } - - def apply(errs: Seq[ConfigErr]): FailureResponse = { - val msg = makeConfigErrMsg(errs.map(_.getClass.getName).mkString(", ")) - FailureResponse(4000, msg, errs.map(_.error)) - } - - def apply(err: RuntimeErr): FailureResponse = { - val msg = "Runtime error: " + err.getClass.getName - FailureResponse(err.errorCode, msg, Seq(err.error)) - } - def apply(err: SparkConfErr): FailureResponse = { val msg = makeConfigErrMsg(err.getClass.getName) FailureResponse(err.errorCode, msg, Seq(err.error)) diff --git a/http/src/main/scala/ru/itclover/tsp/http/protocols/PatternsValidatorProtocols.scala b/http/src/main/scala/ru/itclover/tsp/http/protocols/PatternsValidatorProtocols.scala index 82963fccc..9514e32ff 100644 --- a/http/src/main/scala/ru/itclover/tsp/http/protocols/PatternsValidatorProtocols.scala +++ b/http/src/main/scala/ru/itclover/tsp/http/protocols/PatternsValidatorProtocols.scala @@ -7,7 +7,7 @@ import spray.json.DefaultJsonProtocol case class ValidationResult(pattern: RawPattern, success: Boolean, context: String) trait PatternsValidatorProtocols extends SprayJsonSupport with DefaultJsonProtocol { - implicit val rawPattern = jsonFormat5(RawPattern.apply) + implicit val rawPattern = jsonFormat3(RawPattern.apply) implicit val patterns = jsonFormat2(PatternsValidatorConf.apply) implicit val patternResult = jsonFormat3(ValidationResult.apply) } diff --git a/http/src/main/scala/ru/itclover/tsp/http/protocols/RoutesProtocols.scala b/http/src/main/scala/ru/itclover/tsp/http/protocols/RoutesProtocols.scala index 0ec80aac8..7eaa0d891 100644 --- a/http/src/main/scala/ru/itclover/tsp/http/protocols/RoutesProtocols.scala +++ b/http/src/main/scala/ru/itclover/tsp/http/protocols/RoutesProtocols.scala @@ -5,8 +5,6 @@ import ru.itclover.tsp.core.RawPattern import ru.itclover.tsp.http.domain.input.{DSLPatternRequest, FindPatternsRequest} import ru.itclover.tsp.http.domain.output.SuccessfulResponse.ExecInfo import ru.itclover.tsp.http.domain.output.{FailureResponse, SuccessfulResponse} -import ru.itclover.tsp.io.input._ -import ru.itclover.tsp.io.output.{EventSchema, JDBCOutputConf, KafkaOutputConf, NewRowSchema, OutputConf} import spray.json._ import ru.itclover.tsp.spark import ru.itclover.tsp.spark.io.{ @@ -56,125 +54,8 @@ trait RoutesProtocols extends SprayJsonSupport with DefaultJsonProtocol { implicit def sResponseFmt[R: JsonFormat] = jsonFormat2(SuccessfulResponse.apply[R]) implicit val fResponseFmt = jsonFormat3(FailureResponse.apply) - implicit def nduFormat[Event, EKey: JsonFormat, EValue: JsonFormat] = - jsonFormat( - NarrowDataUnfolding[Event, EKey, EValue], - "keyColumn", - "defaultValueColumn", - "fieldsTimeoutsMs", - "valueColumnMapping", - "defaultTimeout" - ) - implicit def wdfFormat[Event, EKey: JsonFormat, EValue: JsonFormat] = - jsonFormat(WideDataFilling[Event, EKey, EValue], "fieldsTimeoutsMs", "defaultTimeout") - implicit def sdtFormat[Event, EKey: JsonFormat, EValue: JsonFormat] = - new RootJsonFormat[SourceDataTransformation[Event, EKey, EValue]] { - override def read(json: JsValue): SourceDataTransformation[Event, EKey, EValue] = json match { - case obj: JsObject => - val tp = obj.fields.getOrElse("type", sys.error("Source data transformation: missing type")) - val cfg = obj.fields.getOrElse("config", sys.error("Source data transformation: missing config")) - tp match { - case JsString("NarrowDataUnfolding") => nduFormat[Event, EKey, EValue].read(cfg) - case JsString("WideDataFilling") => wdfFormat[Event, EKey, EValue].read(cfg) - case _ => deserializationError(s"Source data transformation: unknown type $tp") - } - case _ => - deserializationError(s"Source data transformation must be an object, but got ${json.compactPrint} instead") - } - override def write(obj: SourceDataTransformation[Event, EKey, EValue]): JsValue = { - val c = obj.config match { - case ndu: NarrowDataUnfolding[Event, EKey, EValue] => nduFormat[Event, EKey, EValue].write(ndu) - case wdf: WideDataFilling[Event, EKey, EValue] => wdfFormat[Event, EKey, EValue].write(wdf) - case _ => deserializationError("Unknown source data transformation") - } - JsObject( - "type" -> obj.`type`.toJson, - "config" -> c - ) - } - } - - implicit val jdbcInpConfFmt = jsonFormat( - JDBCInputConf.apply, - "sourceId", - "jdbcUrl", - "query", - "driverName", - "datetimeField", - "eventsMaxGapMs", - "defaultEventsGapMs", - "chunkSizeMs", - "partitionFields", - "unitIdField", - "userName", - "password", - "dataTransformation", - "defaultToleranceFraction", - "parallelism", - "numParallelSources", - "patternsParallelism", - "timestampMultiplier" - ) - implicit val influxInpConfFmt = jsonFormat( - InfluxDBInputConf.apply, - "sourceId", - "dbName", - "url", - "query", - "eventsMaxGapMs", - "defaultEventsGapMs", - "chunkSizeMs", - "partitionFields", - "datetimeField", - "unitIdField", - "userName", - "password", - "timeoutSec", - "dataTransformation", - "defaultToleranceFraction", - "parallelism", - "numParallelSources", - "patternsParallelism", - "additionalTypeChecking" - ) - - implicit val kafkaInpConfFmt = jsonFormat14( - KafkaInputConf.apply - ) - - implicit val redisConfInputFmt = jsonFormat8( - RedisInputConf.apply - ) - - implicit val newRowSchemaFmt = jsonFormat( - NewRowSchema.apply, - "unitIdField", - "fromTsField", - "toTsField", - "appIdFieldVal", - "patternIdField", - "subunitIdField" - ) - - implicit object eventSchemaFmt extends JsonFormat[EventSchema] { - override def read(json: JsValue): EventSchema = Try(newRowSchemaFmt.read(json)) - .getOrElse(deserializationError("Cannot serialize EventSchema")) - - override def write(obj: EventSchema): JsValue = obj match { - case newRowSchema: NewRowSchema => newRowSchemaFmt.write(newRowSchema) - } - } - - // implicit val jdbcSinkSchemaFmt = jsonFormat(JDBCSegmentsSink.apply, "tableName", "rowSchema") - implicit val jdbcOutConfFmt = jsonFormat8(JDBCOutputConf.apply) - - implicit val kafkaOutConfFmt = jsonFormat5(KafkaOutputConf.apply) - - implicit val rawPatternFmt = jsonFormat5(RawPattern.apply) - - implicit def patternsRequestFmt[IN <: InputConf[_, _, _]: JsonFormat, OUT <: OutputConf[_]: JsonFormat] = - jsonFormat(FindPatternsRequest.apply[IN, OUT], "uuid", "source", "sink", "patterns") + implicit val rawPatternFmt = jsonFormat3(RawPattern.apply) // TODO: Remove type bounds for (In|Out)putConf? implicit def sparkPatternsRequestFmt[IN <: spark.io.InputConf[_, _, _]: JsonFormat, OUT <: spark.io.OutputConf[_]: JsonFormat] = @@ -222,7 +103,7 @@ trait RoutesProtocols extends SprayJsonSupport with DefaultJsonProtocol { } implicit val sparkRowSchemaFmt = jsonFormat( - spark.io.NewRowSchema.apply, + spark.io.RowSchema.apply, "unitIdField", "fromTsField", "toTsField", diff --git a/http/src/main/scala/ru/itclover/tsp/http/routes/JobsRoutes.scala b/http/src/main/scala/ru/itclover/tsp/http/routes/JobsRoutes.scala index 9754d6e54..d8e74b53c 100644 --- a/http/src/main/scala/ru/itclover/tsp/http/routes/JobsRoutes.scala +++ b/http/src/main/scala/ru/itclover/tsp/http/routes/JobsRoutes.scala @@ -1,10 +1,9 @@ package ru.itclover.tsp.http.routes import java.util.concurrent.TimeUnit - import akka.actor.ActorSystem import akka.http.scaladsl.model.StatusCodes.{BadRequest, InternalServerError} -import akka.http.scaladsl.model.Uri +import akka.http.scaladsl.model.{StatusCodes, Uri} import akka.http.scaladsl.server.Directives._ import akka.http.scaladsl.server.Route import akka.http.scaladsl.unmarshalling.FromRequestUnmarshaller @@ -12,11 +11,6 @@ import akka.stream.ActorMaterializer import cats.data.Reader import cats.implicits._ import com.typesafe.scalalogging.Logger -import org.apache.flink.api.common.JobExecutionResult -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.streaming.api.TimeCharacteristic -import org.apache.flink.streaming.api.scala.{StreamExecutionEnvironment, _} -import org.apache.flink.types.Row import ru.itclover.tsp._ import ru.itclover.tsp.core.{Incident, RawPattern} import ru.itclover.tsp.core.io.{AnyDecodersInstances, BasicDecoders} @@ -28,22 +22,12 @@ import ru.itclover.tsp.http.domain.output.SuccessfulResponse.ExecInfo import ru.itclover.tsp.http.domain.output._ import ru.itclover.tsp.http.protocols.RoutesProtocols import ru.itclover.tsp.http.services.streaming.FlinkMonitoringService -import ru.itclover.tsp.io.input.{InfluxDBInputConf, InputConf, JDBCInputConf} -import ru.itclover.tsp.io.output.{JDBCOutputConf, KafkaOutputConf, OutputConf} -import ru.itclover.tsp.mappers._ import ru.itclover.tsp.spark import org.apache.spark.sql.{Row => SparkRow} import scala.concurrent.{ExecutionContextExecutor, Future} -import ru.itclover.tsp.io.input.KafkaInputConf import ru.itclover.tsp.spark.utils.{DataWriterWrapper, ErrorsADT} -import ru.itclover.tsp.utils.ErrorsADT.{ConfigErr, Err, GenericRuntimeErr, RuntimeErr} -import ru.itclover.tsp.spark.utils.ErrorsADT.{ - ConfigErr => SparkConfErr, - Err => SparkErr, - GenericRuntimeErr => SparkGenRTErr, - RuntimeErr => SparkRTErr -} +import ru.itclover.tsp.spark.utils.ErrorsADT.{ConfigErr => SparkConfErr, Err => SparkErr, GenericRuntimeErr => SparkGenRTErr, RuntimeErr => SparkRTErr} import scala.reflect.ClassTag import scala.reflect.runtime.universe.TypeTag @@ -53,7 +37,6 @@ import scala.reflect.runtime.universe.TypeTag trait JobsRoutes extends RoutesProtocols { implicit val executionContext: ExecutionContextExecutor val blockingExecutionContext: ExecutionContextExecutor - implicit val streamEnv: StreamExecutionEnvironment implicit val actorSystem: ActorSystem implicit val materializer: ActorMaterializer implicit val decoders = AnyDecodersInstances @@ -67,72 +50,45 @@ trait JobsRoutes extends RoutesProtocols { val route: Route = parameter('run_async.as[Boolean] ? true) { isAsync => path("streamJob" / """from-(\w+)""".r / """to-(\w+)""".r./) { case (from, to) => - val um = (from, to) match { - case ("jdbc", "jdbc") => as[FindPatternsRequest[JDBCInputConf, JDBCOutputConf]] - case ("influxdb", "jdbc") => as[FindPatternsRequest[InfluxDBInputConf, JDBCOutputConf]] - case ("kafka", "jdbc") => as[FindPatternsRequest[KafkaInputConf, JDBCOutputConf]] - case ("jdbc", "kafka") => as[FindPatternsRequest[JDBCInputConf, KafkaOutputConf]] - case ("influxdb", "kafka") => as[FindPatternsRequest[InfluxDBInputConf, KafkaOutputConf]] - case ("kafka", "kafka") => as[FindPatternsRequest[KafkaInputConf, KafkaOutputConf]] - case _ => null // Not implemented, will crash with a 500 - } - entity( - um.asInstanceOf[FromRequestUnmarshaller[ - FindPatternsRequest[InputConf[RowWithIdx, Symbol, Any], OutputConf[Row]] - ]] - ) { request: FindPatternsRequest[InputConf[RowWithIdx, Symbol, Any], OutputConf[Row]] => - import request._ - val fields = PatternFieldExtractor.extract(patterns) - - val srcOrError: Either[Err, StreamSource[RowWithIdx, Symbol, Any]] = from match { - case "jdbc" => JdbcSource.create(inputConf.asInstanceOf[JDBCInputConf], fields) - case "influxdb" => InfluxDBSource.create(inputConf.asInstanceOf[InfluxDBInputConf], fields) - case "kafka" => KafkaSource.create(inputConf.asInstanceOf[KafkaInputConf], fields) - //case _ => Left(ConfigErr) - } - - val resultOrErr = for { - source <- srcOrError - _ <- createStream(patterns, /*fields,*/ inputConf, outConf, source) - result <- runStream(uuid, isAsync) - } yield result - - matchResultToResponse(resultOrErr, uuid) - } + redirect("sparkJob/from-$from/to-$to/", StatusCodes.PermanentRedirect) } ~ path("sparkJob" / """from-(\w+)""".r / """to-(\w+)""".r./) { case (from, to) => val um = (from, to) match { - case ("jdbc", "jdbc") => as[FindPatternsRequest[spark.io.JDBCInputConf, spark.io.JDBCOutputConf]] - case ("kafka", "jdbc") => as[FindPatternsRequest[spark.io.KafkaInputConf, spark.io.JDBCOutputConf]] - case ("jdbc", "kafka") => as[FindPatternsRequest[spark.io.JDBCInputConf, spark.io.KafkaOutputConf]] + case ("jdbc", "jdbc") => as[FindPatternsRequest[spark.io.JDBCInputConf, spark.io.JDBCOutputConf]] + case ("kafka", "jdbc") => as[FindPatternsRequest[spark.io.KafkaInputConf, spark.io.JDBCOutputConf]] + case ("jdbc", "kafka") => as[FindPatternsRequest[spark.io.JDBCInputConf, spark.io.KafkaOutputConf]] case ("kafka", "kafka") => as[FindPatternsRequest[spark.io.KafkaInputConf, spark.io.KafkaOutputConf]] - case _ => null // Not implemented, will crash with a 500 + case _ => null } - entity( - um.asInstanceOf[FromRequestUnmarshaller[ - FindPatternsRequest[spark.io.InputConf[spark.utils.RowWithIdx, Symbol, Any], spark.io.OutputConf[SparkRow]] - ]] - ) { request => - import request._ - val fields = PatternFieldExtractor.extract(patterns) - - // val resultOrErr: Either[Err, Option[Unit]] = for { - // source <- spark.JdbcSource.create(inputConf, fields) - // stream <- createSparkStream(patterns, fields, inputConf, outConf, source) - // result <- runSparkStream(stream, isAsync) - // } yield result - - val source: Either[SparkConfErr, spark.StreamSource[spark.utils.RowWithIdx, Symbol, Any]] = from match { - case "jdbc" => spark.JdbcSource.create(inputConf.asInstanceOf[spark.io.JDBCInputConf], fields) - case "kafka" => spark.KafkaSource.create(inputConf.asInstanceOf[spark.io.KafkaInputConf], fields) + if (um != null) { + entity( + um.asInstanceOf[FromRequestUnmarshaller[ + FindPatternsRequest[spark.io.InputConf[spark.utils.RowWithIdx, Symbol, Any], spark.io.OutputConf[SparkRow]] + ]] + ) { request => + import request._ + val fields = PatternFieldExtractor.extract(patterns) + + // val resultOrErr: Either[Err, Option[Unit]] = for { + // source <- spark.JdbcSource.create(inputConf, fields) + // stream <- createSparkStream(patterns, fields, inputConf, outConf, source) + // result <- runSparkStream(stream, isAsync) + // } yield result + + val source: Either[SparkConfErr, spark.StreamSource[spark.utils.RowWithIdx, Symbol, Any]] = from match { + case "jdbc" => spark.JdbcSource.create(inputConf.asInstanceOf[spark.io.JDBCInputConf], fields) + case "kafka" => spark.KafkaSource.create(inputConf.asInstanceOf[spark.io.KafkaInputConf], fields) + } + val stream: Either[SparkErr, DataWriterWrapper[SparkRow]] = + source.flatMap(createSparkStream(uuid, patterns, fields, inputConf, outConf, _)) + val result: Either[SparkErr, Option[Long]] = stream.flatMap(runSparkStream(_, isAsync)) + val resultOrErr = result + + matchSparkResultToResponse(resultOrErr, uuid) } - val stream: Either[SparkErr, DataWriterWrapper[SparkRow]] = - source.flatMap(createSparkStream(uuid, patterns, fields, inputConf, outConf, _)) - val result: Either[SparkErr, Option[Long]] = stream.flatMap(runSparkStream(_, isAsync)) - val resultOrErr = result - - matchSparkResultToResponse(resultOrErr, uuid) + } else { + complete(404 -> s"The $from -> $to Spark job is not supported") } } } @@ -140,35 +96,6 @@ trait JobsRoutes extends RoutesProtocols { // TODO: Restore EKey type parameter type EKey = Symbol - def createStream[E: TypeInformation, EItem]( - patterns: Seq[RawPattern], - inputConf: InputConf[E, EKey, EItem], - outConf: OutputConf[Row], - source: StreamSource[E, EKey, EItem] - )(implicit decoders: BasicDecoders[EItem]) = { - streamEnv.setStreamTimeCharacteristic(TimeCharacteristic.EventTime) - - log.debug("createStream started") - - val searcher = PatternsSearchJob(source, decoders) - val strOrErr = searcher.patternsSearchStream( - patterns, - outConf, - PatternsToRowMapper(inputConf.sourceId, outConf.rowSchema) - ) - strOrErr.map { - case (parsedPatterns, stream) => - // .. patternV2.format - val strPatterns = parsedPatterns.map { - case ((_, meta), _) => - /*p.format(source.emptyEvent) +*/ - s" ;; Meta=$meta" - } - log.debug(s"Parsed patterns:\n${strPatterns.mkString(";\n")}") - stream - } - } - def createSparkStream[E: ClassTag: TypeTag, EItem]( uuid: String, patterns: Seq[RawPattern], @@ -200,20 +127,6 @@ trait JobsRoutes extends RoutesProtocols { } } - def runStream(uuid: String, isAsync: Boolean): Either[RuntimeErr, Option[JobExecutionResult]] = { - log.debug("runStream started") - - val res = if (isAsync) { // Just detach job thread in case of async run - Future { streamEnv.execute(uuid) }(blockingExecutionContext) - Right(None) - } else { // Wait for the execution finish - Either.catchNonFatal(Some(streamEnv.execute(uuid))).leftMap(GenericRuntimeErr(_)) - } - - log.debug("runStream finished") - res - } - def runSparkStream(stream: DataWriterWrapper[SparkRow], isAsync: Boolean): Either[SparkErr, Option[Long]] = { log.debug("runStream started") @@ -240,30 +153,6 @@ trait JobsRoutes extends RoutesProtocols { res } - def matchResultToResponse(result: Either[Err, Option[JobExecutionResult]], uuid: String): Route = { - - log.debug("matchResultToResponse started") - - val res = result match { - case Left(err: ConfigErr) => complete((BadRequest, FailureResponse(err))) - case Left(err: RuntimeErr) => - log.error("Error in processing", err.asInstanceOf[GenericRuntimeErr].ex) - complete((InternalServerError, FailureResponse(err))) - // Async job - response with message about successful start - case Right(None) => complete(SuccessfulResponse(uuid, Seq(s"Job `$uuid` has started."))) - // Sync job - response with message about successful ending - case Right(Some(execResult)) => { - // todo query read and written rows (onComplete(monitoring.queryJobInfo(request.uuid))) - val execTime = execResult.getNetRuntime(TimeUnit.SECONDS) - complete(SuccessfulResponse(ExecInfo(execTime.toDouble, Map.empty))) - } - } - log.debug("matchResultToResponse finished") - - res - - } - def matchSparkResultToResponse(result: Either[SparkErr, Option[Long]], uuid: String): Route = { log.debug("matchResultToResponse started") @@ -293,8 +182,7 @@ object JobsRoutes { private val log = Logger[JobsRoutes] def fromExecutionContext(monitoringUrl: Uri, blocking: ExecutionContextExecutor)( - implicit strEnv: StreamExecutionEnvironment, - as: ActorSystem, + implicit as: ActorSystem, am: ActorMaterializer ): Reader[ExecutionContextExecutor, Route] = { @@ -304,7 +192,6 @@ object JobsRoutes { new JobsRoutes { val blockingExecutionContext = blocking implicit val executionContext: ExecutionContextExecutor = execContext - implicit val streamEnv: StreamExecutionEnvironment = strEnv implicit val actorSystem = as implicit val materializer = am override val monitoringUri = monitoringUrl diff --git a/http/src/main/scala/ru/itclover/tsp/http/routes/MonitoringRoutes.scala b/http/src/main/scala/ru/itclover/tsp/http/routes/MonitoringRoutes.scala index e2b43c6c4..018835aff 100644 --- a/http/src/main/scala/ru/itclover/tsp/http/routes/MonitoringRoutes.scala +++ b/http/src/main/scala/ru/itclover/tsp/http/routes/MonitoringRoutes.scala @@ -144,8 +144,7 @@ trait MonitoringRoutes extends RoutesProtocols with MonitoringServiceProtocols { Map( "tsp" -> BuildInfo.version, "scala" -> BuildInfo.scalaVersion, - "spark" -> BuildInfo.sparkVersion, - "flink" -> BuildInfo.flinkVersion + "spark" -> BuildInfo.sparkVersion ) ) ) diff --git a/http/src/main/scala/ru/itclover/tsp/http/services/kafka/Serdes.scala b/http/src/main/scala/ru/itclover/tsp/http/services/kafka/Serdes.scala deleted file mode 100644 index e2505b313..000000000 --- a/http/src/main/scala/ru/itclover/tsp/http/services/kafka/Serdes.scala +++ /dev/null @@ -1,32 +0,0 @@ -package ru.itclover.tsp.http.kafka - -import java.io.{ByteArrayInputStream} -import org.apache.flink.api.common.serialization.{AbstractDeserializationSchema} - -import org.apache.arrow.memory.RootAllocator -import org.apache.arrow.vector.ipc.{ArrowStreamReader} - -object Serdes { - - type BArr = Array[Byte] - - class StringDeserializer extends AbstractDeserializationSchema[String] { - override def deserialize(bytes: BArr): String = bytes.toString - } - - class BytesDeserializer extends AbstractDeserializationSchema[BArr] { - override def deserialize(bytes: BArr): BArr = bytes - } - - class ArrowDeserializer extends AbstractDeserializationSchema[ArrowStreamReader] { - - override def deserialize(bytes: BArr): ArrowStreamReader = { - val alloc = new RootAllocator(Integer.MAX_VALUE) - val stream = new ByteArrayInputStream(bytes) - val reader = new ArrowStreamReader(stream, alloc) - - reader - } - } - -} diff --git a/http/src/test/scala/ru/itclover/tsp/http/HttpServiceTest.scala b/http/src/test/scala/ru/itclover/tsp/http/HttpServiceTest.scala index d20db0b47..1a366f95b 100644 --- a/http/src/test/scala/ru/itclover/tsp/http/HttpServiceTest.scala +++ b/http/src/test/scala/ru/itclover/tsp/http/HttpServiceTest.scala @@ -6,15 +6,11 @@ import akka.http.scaladsl.testkit.ScalatestRouteTest import akka.http.scaladsl.unmarshalling.Unmarshal import akka.stream.ActorMaterializer import com.google.common.util.concurrent.ThreadFactoryBuilder -import org.apache.flink.api.common.JobID -import org.apache.flink.runtime.client.JobExecutionException -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.spark.sql.SparkSession import org.scalatest.Inspectors._ import org.scalatest.{FlatSpec, Matchers} import ru.itclover.tsp.http.domain.output.FailureResponse import ru.itclover.tsp.http.protocols.RoutesProtocols -import ru.itclover.tsp.utils.ErrorsADT.{GenericConfigError, GenericRuntimeErr} import ru.itclover.tsp.http.utils.Exceptions.InvalidRequest import ru.yandex.clickhouse.except.ClickHouseException @@ -29,8 +25,6 @@ class HttpServiceTest extends FlatSpec with Matchers with ScalatestRouteTest wit implicit val system: ActorSystem = ActorSystem("TSP-system-test") implicit val materializer: ActorMaterializer = ActorMaterializer() implicit val executionContext: ExecutionContextExecutor = system.dispatcher - implicit override val streamEnvironment: StreamExecutionEnvironment = - StreamExecutionEnvironment.createLocalEnvironment() val spark = SparkSession .builder() @@ -81,22 +75,6 @@ class HttpServiceTest extends FlatSpec with Matchers with ScalatestRouteTest wit resp.message shouldBe "Job execution failure" resp.errors.isEmpty shouldBe service.isHideExceptions } - Get() ~> service.exceptionsHandler(new JobExecutionException(new JobID(), "test", new Exception())) ~> check { - response.status shouldBe StatusCodes.InternalServerError - noException should be thrownBy Unmarshal(response.entity).to[FailureResponse] - val resp = Await.result(Unmarshal(response.entity).to[FailureResponse], Duration.Inf) - resp.errorCode shouldBe 5002 - resp.message shouldBe "Job execution failure" - resp.errors.isEmpty shouldBe service.isHideExceptions - } - Get() ~> service.exceptionsHandler(new JobExecutionException(new JobID(), "test", null)) ~> check { - response.status shouldBe StatusCodes.InternalServerError - noException should be thrownBy Unmarshal(response.entity).to[FailureResponse] - val resp = Await.result(Unmarshal(response.entity).to[FailureResponse], Duration.Inf) - resp.errorCode shouldBe 5002 - resp.message shouldBe "Job execution failure" - resp.errors.isEmpty shouldBe service.isHideExceptions - } Get() ~> service.exceptionsHandler(new ClickHouseException(54, null, "127.0.0.1", 8123)) ~> check { response.status shouldBe StatusCodes.InternalServerError noException should be thrownBy Unmarshal(response.entity).to[FailureResponse] @@ -135,8 +113,5 @@ class HttpServiceTest extends FlatSpec with Matchers with ScalatestRouteTest wit "FailureResponse objects" should "construct" in { FailureResponse(new Exception()).errorCode shouldBe 5000 FailureResponse(5011, new Exception()).errorCode shouldBe 5011 - FailureResponse(GenericRuntimeErr(new Exception(), 5012)).errorCode shouldBe 5012 - FailureResponse(GenericConfigError(new Exception(), 4013)).errorCode shouldBe 4013 - FailureResponse(Seq(GenericConfigError(new Exception(), 4014), GenericConfigError(new Exception(), 4015))).errorCode shouldBe 4000 } } diff --git a/http/src/test/scala/ru/itclover/tsp/http/RoutesProtocolsFormatTest.scala b/http/src/test/scala/ru/itclover/tsp/http/RoutesProtocolsFormatTest.scala index 96c3198c1..9d9d6f845 100644 --- a/http/src/test/scala/ru/itclover/tsp/http/RoutesProtocolsFormatTest.scala +++ b/http/src/test/scala/ru/itclover/tsp/http/RoutesProtocolsFormatTest.scala @@ -1,7 +1,6 @@ package ru.itclover.tsp.http import org.scalatest.{FlatSpec, Matchers} import ru.itclover.tsp.http.protocols.RoutesProtocols -import ru.itclover.tsp.io.input.{NarrowDataUnfolding, WideDataFilling} import spray.json.{JsBoolean, JsNumber, JsString, JsValue} // In test cases, 'should' expressions are non-unit. Suppressing wartremover warnings about it @@ -36,9 +35,6 @@ class RoutesProtocolsFormatTest extends FlatSpec with Matchers with RoutesProtoc } "SDT formats" should "work" in { - sdtFormat[Any, Symbol, Symbol].write(NarrowDataUnfolding('key, 'value, Map.empty[Symbol, Long])) shouldBe a[JsValue] - sdtFormat[Any, Symbol, Symbol].write(WideDataFilling[Any, Symbol, Symbol](Map.empty[Symbol, Long], Some(0L))) shouldBe a[ - JsValue - ] + // TODO: Spark SDT } } diff --git a/integration/correctness/src/test/scala/ru/itclover/tsp/RulesTest.scala b/integration/correctness/src/test/scala/ru/itclover/tsp/RulesTest.scala deleted file mode 100644 index 24de37524..000000000 --- a/integration/correctness/src/test/scala/ru/itclover/tsp/RulesTest.scala +++ /dev/null @@ -1,197 +0,0 @@ -/* -package ru.itclover.tsp - -import java.time.Instant -import org.scalatest.{Matchers, WordSpec} -import ru.itclover.tsp.aggregators.AggregatorPhases.{Derivation, ToSegments} -import ru.itclover.tsp.phases.NumericPhases._ -import ru.itclover.tsp.core.{Pattern, PatternResult, Window} -import ru.itclover.tsp.core.PatternResult.{Failure, Success} -import ru.itclover.tsp.core.Pattern.Functions._ -import ru.itclover.tsp.phases.Phases.{Decreasing, Increasing} -import ru.itclover.tsp.http.utils.{Timer => TimerGenerator, _} -import ru.itclover.tsp.phases.BooleanPhases.Assert -import scala.concurrent.duration._ -import scala.util.Random - - -case class Row(time: Instant, speed: Double, pump: Double, wagonId: Int = 0) - -class RulesTest extends WordSpec with Matchers { - - import ru.itclover.tsp.core.Time._ - import Predef.{any2stringadd => _, assert => _, _} - - implicit val random: Random = new java.util.Random(345l) - - implicit val symbolNumberExtractorEvent = new SymbolNumberExtractor[Row] { - override def extract(event: Row, symbol: Symbol) = { - symbol match { - case 'speed => event.speed - case 'pump => event.pump - case _ => sys.error(s"No field $symbol in $event") - } - } - } - implicit val timeExtractor: TimeExtractor[Row] = new TimeExtractor[Row] { - override def apply(v1: Row) = v1.time - } - - def fakeMapper[Event, PhaseOut](p: Pattern[Event, _, PhaseOut]) = FakeMapper[Event, PhaseOut]() - - - def run[Event, Out](rule: Pattern[Event, _, Out], events: Seq[Event]): Vector[PatternResult.TerminalResult[Out]] = { - val mapResults = fakeMapper(rule) - events - .foldLeft(PatternMapper(rule, mapResults)) { case (machine, event) => machine(event) } - .result - } - - def runWithSegmentation[Event, Out](rule: Pattern[Event, _, Out], events: Seq[Event]) - (implicit te: TimeExtractor[Event]) = { - val mapResults = segmentMapper(rule)(te) - events - .foldLeft(PatternMapper(rule, mapResults)) { case (machine, event) => machine(event) } - .result - } - - type Phase[Row] = Pattern[Row, _, _] - - "Combine And & Assert parsers" should { - "work correctly" in { - import ru.itclover.tsp.phases.NumericPhases._ - val phase: Phase[Row] = Assert('speed.asDouble > 10.0) and Assert('speed.asDouble < 20.0) - - val rows = ( - for (time <- TimerGenerator(from = Instant.now()); - speed <- Constant(30.0).timed(1.seconds) - .after(Change(from = 30.0, to = 0.0, howLong = 10.seconds)) - .after(Constant(0.0)) - ) yield Row(time, speed.toInt, 0) - ).run(seconds = 10) - - val results = run(phase, rows) - - println(s"Results = $results") - - assert(results.nonEmpty) - - val (success, failures) = results partition { - case Success(_) => true - case Failure(_) => false - } - - success.length should be > 1 - failures.length should be > 1 - } - } - - "dsl" should { - - "works" in { - import ru.itclover.tsp.core.Time._ - - import Predef.{any2stringadd => _, assert => _, _} - - implicit val random: Random = new java.util.Random(345l) - - val window: Window = 5.seconds - - type Phase[Row] = Pattern[Row, _, _] - - val phase: Phase[Row] = avg((e: Row) => e.speed, 2.seconds) > 100.0 - - val rows = ( - for (time <- TimerGenerator(from = Instant.now()); - pump <- RandomInRange(1, 100).map(_.toDouble).timed(40.second) - .after(Constant(0)); - speed <- Constant(261.0).timed(1.seconds) - .after(Change(from = 260.0, to = 0.0, howLong = 10.seconds)) - .after(Constant(0.0)) - ) yield Row(time, speed.toInt, pump.toInt) - ).run(seconds = 10) - - val results = run(phase, rows) - - assert(results.nonEmpty) - } - - } - - - "Result segmentation" should { - - implicit val random: Random = new java.util.Random(345l) - - implicit val symbolNumberExtractorEvent = new SymbolNumberExtractor[Row] { - override def extract(event: Row, symbol: Symbol) = { - symbol match { - case 'speed => event.speed - case 'pump => event.pump - case _ => sys.error(s"No field $symbol in $event") - } - } - } - - implicit val timeExtractor = new TimeExtractor[Row] { - override def apply(v1: Row) = v1.time - } - - "work on not segmented output" in { - val phase: Phase[Row] = Assert('speed.asDouble > 35.0) - val rows = ( - for (time <- TimerGenerator(from = Instant.now()); - speed <- Constant(50.0).timed(1.seconds) - .after(Change(from = 50.0, to = 30.0, howLong = 20.seconds)) - .after(Constant(0.0)) - ) yield Row(time, speed.toInt, 0) - ).run(seconds = 20) - val (successes, failures) = runWithSegmentation(phase, rows).partition(_.isInstanceOf[Success[Segment]]) - - failures should not be empty - successes should not be empty - successes.length should equal(1) - - val segmentLengthOpt = successes.head match { - case Success(Segment(from, to)) => Some(to.toMillis - from.toMillis) - case _ => None - } - segmentLengthOpt should not be empty - segmentLengthOpt.get should be > 12000L - segmentLengthOpt.get should be < 20000L - } - - "Segment Increasing" in { - val phase: Phase[Row] = ToSegments(Increasing(_.speed, 35.0, 50.0)) - val rows = ( - for (time <- TimerGenerator(from = Instant.now()); - speed <- Constant(30.0).timed(2.seconds) - .after(Change(from = 30.0, to = 35.0, howLong = 5.seconds)) - .after(Constant(35.0).timed(1.seconds)) - .after(Change(from = 35.0, to = 50.0, howLong = 13.seconds)) - .after(Constant(51.0).timed(1.seconds)) - ) yield Row(time, speed.toInt, 0) - ).run(seconds = 15) - - println(rows.map(_.speed)) - - val (successes, failures) = runWithSegmentation(phase, rows).partition(_.isInstanceOf[Success[_]]) - - println(successes) - - failures.length should be > 0 - successes should not be empty - successes.length should equal(1) - - val segmentLengthOpt = successes.head match { - case Success(Segment(from, to)) => Some(to.toMillis - from.toMillis) - case _ => None - } - segmentLengthOpt should not be empty - segmentLengthOpt.get should be > 3000L - segmentLengthOpt.get should be < 10000L - } - } - -} - */ diff --git a/integration/correctness/src/test/scala/ru/itclover/tsp/http/AccumsTest.scala b/integration/correctness/src/test/scala/ru/itclover/tsp/http/AccumsTest.scala deleted file mode 100644 index ed4aab2fb..000000000 --- a/integration/correctness/src/test/scala/ru/itclover/tsp/http/AccumsTest.scala +++ /dev/null @@ -1,282 +0,0 @@ -//TODO: may fail by java.lang.OutOfMemoryError: Metaspace -package ru.itclover.tsp.http - -import java.util.concurrent.{SynchronousQueue, ThreadPoolExecutor, TimeUnit} -import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest} -import com.dimafeng.testcontainers._ -import com.google.common.util.concurrent.ThreadFactoryBuilder -import com.typesafe.scalalogging.Logger -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.spark.sql.SparkSession -import org.scalatest.FlatSpec -import org.testcontainers.containers.wait.strategy.Wait -import ru.itclover.tsp.core.RawPattern -import ru.itclover.tsp.http.domain.input.FindPatternsRequest -import ru.itclover.tsp.http.domain.output.SuccessfulResponse.FinishedJobResponse -import ru.itclover.tsp.http.utils.{JDBCContainer, SqlMatchers} -import ru.itclover.tsp.io.input.JDBCInputConf -import ru.itclover.tsp.io.output.{JDBCOutputConf, NewRowSchema} -import ru.itclover.tsp.utils.Files - -import scala.concurrent.duration.DurationInt -import scala.concurrent.{ExecutionContext, ExecutionContextExecutor} -import scala.util.Success - -// In test cases, 'should' expressions are non-unit. Suppressing wartremover warnings about it -// Also, some test cases indirectly use Any type. -@SuppressWarnings(Array("org.wartremover.warts.NonUnitStatements", "org.wartremover.warts.Any")) -class AccumsTest extends FlatSpec with SqlMatchers with ScalatestRouteTest with HttpService with ForAllTestContainer { - - implicit override val executionContext: ExecutionContextExecutor = scala.concurrent.ExecutionContext.global - implicit override val streamEnvironment: StreamExecutionEnvironment = - StreamExecutionEnvironment.createLocalEnvironment() - streamEnvironment.setParallelism(4) // To prevent run out of network buffers on large number of CPUs (e.g. 32) - streamEnvironment.setMaxParallelism(30000) // For proper keyBy partitioning - - val spark = SparkSession - .builder() - .master("local") - .appName("TSP Spark test") - .config("spark.io.compression.codec", "snappy") - .getOrCreate() - - // to run blocking tasks. - val blockingExecutorContext: ExecutionContextExecutor = - ExecutionContext.fromExecutor( - new ThreadPoolExecutor( - 0, // corePoolSize - Int.MaxValue, // maxPoolSize - 1000L, //keepAliveTime - TimeUnit.MILLISECONDS, //timeUnit - new SynchronousQueue[Runnable](), //workQueue - new ThreadFactoryBuilder().setNameFormat("blocking-thread").setDaemon(true).build() - ) - ) - - private val log = Logger("AccumsTest") - - implicit def defaultTimeout( /*implicit system: ActorSystem*/ ): RouteTestTimeout = RouteTestTimeout(300.seconds) - - val port = 8137 - implicit override val container: JDBCContainer = new JDBCContainer( - "yandex/clickhouse-server:latest", - port -> 8123 :: 9089 -> 9000 :: Nil, - "ru.yandex.clickhouse.ClickHouseDriver", - s"jdbc:clickhouse://localhost:$port/default", - waitStrategy = Some(Wait.forHttp("/").forStatusCode(200).forStatusCode(400)) - ) - - val windowLength = 1000 - val windowMin = 10 - - // format: off - val realWorkloadQuery: String = """SELECT * FROM ( - | SELECT toFloat64(number * 1 + 100000000) as ts, toString(rand() % 2) as t1, toFloat32(rand() % 300) as lt300Sens, toUInt8(rand() % 10) as lt10Sens, 1000 + (rand() % 5000) as gt1000Sens FROM numbers(1000) - | union all -- 499 1 - | SELECT toFloat64(number * 1 + 100001000) as ts, toString(1) as t1, toFloat32(1 + (rand() % 299)) as lt300Sens, toUInt8(8 + (rand() % 2)) as lt10Sens, 1000 + (rand() % 5000) as gt1000Sens FROM numbers(1000) - | union all -- 988 1 - | SELECT toFloat64(number * 1 + 100002000) as ts, toString(1) as t1, toFloat32(0) as lt300Sens, toUInt8(1) as lt10Sens, 1000 + (rand() % 5000) as gt1000Sens FROM numbers(1000) - | union all -- 466 1 - | SELECT toFloat64(number * 1 + 100003000) as ts, toString(2) as t1, toFloat32(0) as lt300Sens, toUInt8(0) as lt10Sens, 5990 + (rand() % 10) as gt1000Sens FROM numbers(1000) - | union all -- 0 - | SELECT toFloat64(number * 1 + 100004000) as ts, toString(rand() % 2) as t1, toFloat32(rand() % 300) as lt300Sens, toUInt8(rand() % 10) as lt10Sens, 1000 + (rand() % 5000) as gt1000Sens FROM numbers(1000) - | union all -- 466 2 - | SELECT toFloat64(number * 1 + 100005000) as ts, toString(1) as t1, toFloat32(0) as lt300Sens, toUInt8(0) as lt10Sens, 5990 + (rand() % 10) as gt1000Sens FROM numbers(1000) - |) ORDER BY ts""".stripMargin - // format: on - - val (countWindowMaxTimeSec, countWindowPattern) = 150.0 -> List( - RawPattern(4990, s"lt10Sens >= 8 for $windowMin min >= ${windowMin * 60 - 30} times") - ) - - val (timeWindowMaxTimeSec, timeWindowPattern) = 250.0 -> List( - RawPattern(499, s"lt10Sens >= 8 for $windowMin min > ${windowMin - 1} min") - ) - - val (nestedTimeWindowMaxTimeSec, nestedTimeWindowPattern) = 175.0 -> List( - RawPattern(4991, s"(avg(lt10Sens as float64, 30 sec) >= 8.0) for $windowMin min > ${windowMin - 1} min") - ) - - val (timeWindowCountMaxTimeSec, timeWindowCountPattern) = 60.0 -> List( - RawPattern(988, s"lt10Sens = 1 for $windowMin min > ${windowMin * 60 - 1} times") - ) - - val (timedMaxTimeSec, timedPattern) = 75.0 -> List( - RawPattern(466, s"gt1000Sens >= 5990 for $windowMin min") - ) - - val (reducerMaxTimeSec, reducerPattern) = 100.0 -> List( - RawPattern(467, "avgOf(1.0, 0.0) < 200") - ) - - val inputConf = JDBCInputConf( - sourceId = 123, - jdbcUrl = container.jdbcUrl, - query = realWorkloadQuery, - driverName = container.driverName, - datetimeField = 'ts, - unitIdField = Some('unit), - eventsMaxGapMs = Some(2000L), - defaultEventsGapMs = Some(2000L), - chunkSizeMs = Some(900000L), - partitionFields = Seq('t1) - ) - - val sinkSchema = - NewRowSchema('series_storage, 'from, 'to, ('app, 1), 'id, 'subunit) - - val outputConf = JDBCOutputConf( - "Test.SM_basic_patterns", - sinkSchema, - s"jdbc:clickhouse://localhost:$port/default", - "ru.yandex.clickhouse.ClickHouseDriver" - ) - - override def afterStart(): Unit = { - super.afterStart() - - Files - .readResource("/sql/test-db-schema.sql") - .mkString - .split(";") - .foreach(container.executeUpdate) - - Files - .readResource("/sql/sink-schema.sql") - .mkString - .split(";") - .foreach(container.executeUpdate) - } - - override def afterAll(): Unit = { - super.afterAll() - container.stop() - } - - "Count window (count)" should "compute in time" in { - Post( - "/streamJob/from-jdbc/to-jdbc/?run_async=0", - FindPatternsRequest("1", inputConf, outputConf, countWindowPattern) - ) ~> route ~> check { - - status shouldEqual StatusCodes.OK - val resp = unmarshal[FinishedJobResponse](responseEntity) - resp shouldBe a[Success[_]] - val execTimeS = resp.map(_.response.execTimeSec).getOrElse(Double.MaxValue) - log.info(s"Test job completed for $execTimeS sec.") - - // Correctness - checkByQuery( - List(List(0.0)), - "SELECT count(*) FROM Test.SM_basic_patterns WHERE id = 4990 AND toUnixTimestamp(to) - toUnixTimestamp(from) > 900" - ) - // Performance - execTimeS should be <= countWindowMaxTimeSec - } - } - - "Time window (truthMillis)" should "compute in time" in { - Post("/streamJob/from-jdbc/to-jdbc/?run_async=0", FindPatternsRequest("1", inputConf, outputConf, timeWindowPattern)) ~> route ~> check { - - status shouldEqual StatusCodes.OK - val resp = unmarshal[FinishedJobResponse](responseEntity) - resp shouldBe a[Success[_]] - val execTimeS = resp.map(_.response.execTimeSec).getOrElse(Double.MaxValue) - log.info(s"Test job completed for $execTimeS sec.") - - // Correctness - checkByQuery( - List(List(0.0)), - "SELECT count(*) FROM Test.SM_basic_patterns WHERE id = 499 AND toUnixTimestamp(to) - toUnixTimestamp(from) > 990" - ) - // Performance - execTimeS should be <= timeWindowMaxTimeSec - } - } - - "Nested time window (truthMillis)" should "compute in time" in { - Post( - "/streamJob/from-jdbc/to-jdbc/?run_async=0", - FindPatternsRequest("1", inputConf, outputConf, nestedTimeWindowPattern) - ) ~> route ~> check { - - status shouldEqual StatusCodes.OK - val resp = unmarshal[FinishedJobResponse](responseEntity) - resp shouldBe a[Success[_]] - val execTimeS = resp.map(_.response.execTimeSec).getOrElse(Double.MaxValue) - log.info(s"Test job completed for $execTimeS sec.") - - // Correctness - checkByQuery( - List(List(0.0)), - "SELECT count(*) FROM Test.SM_basic_patterns WHERE id = 4991 AND toUnixTimestamp(to) - toUnixTimestamp(from) > 990" - ) - // Performance - execTimeS should be <= nestedTimeWindowMaxTimeSec - } - } - - "Time window count (truthMillisCount)" should "compute in time" in { - Post( - "/streamJob/from-jdbc/to-jdbc/?run_async=0", - FindPatternsRequest("2", inputConf, outputConf, timeWindowCountPattern) - ) ~> route ~> check { - - status shouldEqual StatusCodes.OK - val resp = unmarshal[FinishedJobResponse](responseEntity) - resp shouldBe a[Success[_]] - val execTimeS = resp.map(_.response.execTimeSec).getOrElse(Double.MaxValue) - log.info(s"Test job completed for $execTimeS sec.") - - // Correctness - checkByQuery( - List(List(0.0)), - "SELECT count(*) FROM Test.SM_basic_patterns WHERE id = 988 AND toUnixTimestamp(to) - toUnixTimestamp(from) > 990" - ) - // Performance - execTimeS should be <= timeWindowCountMaxTimeSec - } - } - - "Timed window (.timed)" should "compute in time" in { - Post("/streamJob/from-jdbc/to-jdbc/?run_async=0", FindPatternsRequest("3", inputConf, outputConf, timedPattern)) ~> route ~> check { - - status shouldEqual StatusCodes.OK - val resp = unmarshal[FinishedJobResponse](responseEntity) - resp shouldBe a[Success[_]] - val execTimeS = resp.map(_.response.execTimeSec).getOrElse(Double.MaxValue) - log.info(s"Test job completed for $execTimeS sec.") - - // Correctness - checkByQuery( - List(List(0.0)), - "SELECT count(*) FROM Test.SM_basic_patterns WHERE id = 466 AND toUnixTimestamp(to) - toUnixTimestamp(from) > 990" - ) - // Performance - execTimeS should be <= timedMaxTimeSec - } - - } - - /* - "Reducer (.avgOf)" should "compute in time" in { - Post("/streamJob/from-jdbc/to-jdbc/?run_async=0", FindPatternsRequest("3", inputConf, outputConf, reducerPattern)) ~> route ~> check { - - status shouldEqual StatusCodes.OK - val resp = unmarshal[FinishedJobResponse](responseEntity) - resp shouldBe a[Success[_]] - val execTimeS = resp.get.response.execTimeSec - log.info(s"Test job completed for $execTimeS sec.") - - // Correctness - checkByQuery( - List(List(521.5)), - "SELECT count(*) FROM Test.SM_basic_patterns WHERE id = 467" - ) - // Performance - execTimeS should be <= reducerMaxTimeSec - } - - }**/ -} diff --git a/integration/correctness/src/test/scala/ru/itclover/tsp/http/BasicInfluxToJdbcTest.scala b/integration/correctness/src/test/scala/ru/itclover/tsp/http/BasicInfluxToJdbcTest.scala deleted file mode 100644 index c9f572783..000000000 --- a/integration/correctness/src/test/scala/ru/itclover/tsp/http/BasicInfluxToJdbcTest.scala +++ /dev/null @@ -1,215 +0,0 @@ -package ru.itclover.tsp.http - -import java.util.concurrent.{SynchronousQueue, ThreadPoolExecutor, TimeUnit} -import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest} -import com.dimafeng.testcontainers._ -import com.google.common.util.concurrent.ThreadFactoryBuilder -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.spark.sql.SparkSession -import org.scalatest.FlatSpec -import org.testcontainers.containers.wait.strategy.Wait -import ru.itclover.tsp.core.RawPattern -import ru.itclover.tsp.http.domain.input.FindPatternsRequest -import ru.itclover.tsp.http.utils.{InfluxDBContainer, JDBCContainer, SqlMatchers} -import ru.itclover.tsp.io.input.{InfluxDBInputConf, WideDataFilling} -import ru.itclover.tsp.io.output.{JDBCOutputConf, NewRowSchema} -import ru.itclover.tsp.utils.Files - -import scala.concurrent.duration.DurationInt -import scala.concurrent.{ExecutionContext, ExecutionContextExecutor} - -// In test cases, 'should' expressions are non-unit. Suppressing wartremover warnings about it -// Also, IO configurations use Any. -@SuppressWarnings(Array("org.wartremover.warts.NonUnitStatements", "org.wartremover.warts.Any")) -class BasicInfluxToJdbcTest - extends FlatSpec - with SqlMatchers - with ScalatestRouteTest - with HttpService - with ForAllTestContainer { - - implicit override val executionContext: ExecutionContextExecutor = scala.concurrent.ExecutionContext.global - implicit override val streamEnvironment: StreamExecutionEnvironment = - StreamExecutionEnvironment.createLocalEnvironment() - streamEnvironment.setParallelism(4) // To prevent run out of network buffers on large number of CPUs (e.g. 32) - streamEnvironment.setMaxParallelism(30000) // For proper keyBy partitioning - - val spark = SparkSession - .builder() - .master("local") - .appName("TSP Spark test") - .config("spark.io.compression.codec", "snappy") - .getOrCreate() - - // to run blocking tasks. - val blockingExecutorContext: ExecutionContextExecutor = - ExecutionContext.fromExecutor( - new ThreadPoolExecutor( - 0, // corePoolSize - Int.MaxValue, // maxPoolSize - 1000L, //keepAliveTime - TimeUnit.MILLISECONDS, //timeUnit - new SynchronousQueue[Runnable](), //workQueue - new ThreadFactoryBuilder().setNameFormat("blocking-thread").setDaemon(true).build() - ) - ) - - implicit def defaultTimeout = RouteTestTimeout(300.seconds) - - val influxPort = 8138 - - val influxContainer = - new InfluxDBContainer( - "influxdb:1.5", - influxPort -> 8086 :: Nil, - s"http://localhost:$influxPort", - "Test", - "default", - waitStrategy = Some(Wait.forHttp("/").forStatusCode(200).forStatusCode(404)) - ) - - val jdbcPort = 8157 - implicit val jdbcContainer = new JDBCContainer( - "yandex/clickhouse-server:latest", - jdbcPort -> 8123 :: 9072 -> 9000 :: Nil, - "ru.yandex.clickhouse.ClickHouseDriver", - s"jdbc:clickhouse://localhost:$jdbcPort/default", - waitStrategy = Some(Wait.forHttp("/").forStatusCode(200).forStatusCode(400)) - ) - - override val container = MultipleContainers(LazyContainer(jdbcContainer), LazyContainer(influxContainer)) - - val inputConf = InfluxDBInputConf( - sourceId = 123, - url = influxContainer.url, - query = "select * from SM_basic_wide", - dbName = influxContainer.dbName, - eventsMaxGapMs = Some(60000L), - defaultEventsGapMs = Some(1000L), - chunkSizeMs = Some(900000L), - unitIdField = Some('mechanism_id), - partitionFields = Seq('series_id, 'mechanism_id), - userName = Some("default"), - additionalTypeChecking = Some(false) - ) - val typeCastingInputConf = inputConf.copy(query = """select * from SM_typeCasting_wide""") - - val fillingInputConf = inputConf.copy( - query = """select *, speed as "speed64" from SM_sparse_wide""", - dataTransformation = Some(WideDataFilling(Map('speed -> 2000L, 'pos -> 2000L), None)) - ) - - val rowSchema = NewRowSchema('series_storage, 'from, 'to, ('app, 1), 'id, 'subunit) - - val outputConf = JDBCOutputConf( - "Test.SM_basic_patterns", - rowSchema, - s"jdbc:clickhouse://localhost:$jdbcPort/default", - "ru.yandex.clickhouse.ClickHouseDriver" - ) - - val basicAssertions = Seq( - RawPattern(1, "speed < 15"), - RawPattern(3, "speed > 10.0", Some(Map("test" -> "test")), Some(540), Some(Seq('speed))) - ) - val typesCasting = Seq(RawPattern(10, "speed = 15"), RawPattern(11, "speed64 < 15.0")) - val filling = Seq(RawPattern(20, "speed = 20 and pos = 15")) - - override def afterStart(): Unit = { - super.afterStart() - - Files - .readResource("/sql/test-db-schema.sql") - .mkString - .split(";") - .foreach(jdbcContainer.executeUpdate) - - Files - .readResource("/sql/infl-test-db-schema.sql") - .mkString - .split(";") - .foreach(influxContainer.executeQuery) - - Files - .readResource("/sql/wide/infl-source-inserts.influx") - .mkString - .split(";") - .foreach(influxContainer.executeUpdate) - - Files - .readResource("/sql/sink-schema.sql") - .mkString - .split(";") - .foreach(jdbcContainer.executeUpdate) - } - - override def afterAll(): Unit = { - super.afterAll() - container.stop() - } - - "Basic assertions and forwarded fields" should "work for wide dense table" in { - - Post( - "/streamJob/from-influxdb/to-jdbc/?run_async=0", - FindPatternsRequest("1", inputConf, outputConf, basicAssertions) - ) ~> - route ~> check { - status shouldEqual StatusCodes.OK - - // for 65001 - checkByQuery( - List(List(2.0)), - "SELECT toUnixTimestamp(to) - toUnixTimestamp(from) FROM Test.SM_basic_patterns WHERE id = 1 " - ) - - checkByQuery( - List(List(1.0), List(1.0)), - "SELECT toUnixTimestamp(to) - toUnixTimestamp(from) FROM Test.SM_basic_patterns WHERE id = 3 " - ) - - } - } - - "Types casting" should "work for wide dense table" in { - Post( - "/streamJob/from-influxdb/to-jdbc/?run_async=0", - FindPatternsRequest("2", typeCastingInputConf, outputConf, typesCasting) - ) ~> - route ~> check { - status shouldEqual StatusCodes.OK - - // for 65001 - checkByQuery( - List(List(0.0)), - "SELECT toUnixTimestamp(to) - toUnixTimestamp(from) FROM Test.SM_basic_patterns WHERE id = 10 " - ) - - // for 65001 and 65002 - checkByQuery( - List(List(2.0), List(0.0)), - "SELECT toUnixTimestamp(to) - toUnixTimestamp(from) FROM Test.SM_basic_patterns WHERE id = 11 " - ) - } - } - - /* - // TODO: Fix json format for arbitrary - "Data filling" should "work for wide sparse table" in { - - Post( - "/streamJob/from-influxdb/to-jdbc/?run_async=0", - FindPatternsRequest("3", fillingInputConf, outputConf, filling) - ) ~> - route ~> check { - status shouldEqual StatusCodes.OK - - checkByQuery( - List(List(0.0)), - "SELECT toUnixTimestamp(to) - toUnixTimestamp(from) FROM Test.SM_basic_patterns WHERE id = 20 " - ) - } - } - **/ -} diff --git a/integration/correctness/src/test/scala/ru/itclover/tsp/http/BasicJdbcTest.scala b/integration/correctness/src/test/scala/ru/itclover/tsp/http/BasicJdbcTest.scala deleted file mode 100644 index b5fda9d51..000000000 --- a/integration/correctness/src/test/scala/ru/itclover/tsp/http/BasicJdbcTest.scala +++ /dev/null @@ -1,176 +0,0 @@ -package ru.itclover.tsp.http - -import java.util.concurrent.{SynchronousQueue, ThreadPoolExecutor, TimeUnit} -import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest} -import com.dimafeng.testcontainers._ -import com.google.common.util.concurrent.ThreadFactoryBuilder -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.spark.sql.SparkSession -import org.scalatest.FlatSpec -import org.testcontainers.containers.wait.strategy.Wait -import ru.itclover.tsp.core.RawPattern -import ru.itclover.tsp.http.domain.input.FindPatternsRequest -import ru.itclover.tsp.http.utils.{JDBCContainer, SqlMatchers} -import ru.itclover.tsp.io.input.JDBCInputConf -import ru.itclover.tsp.io.output.{JDBCOutputConf, NewRowSchema} -import ru.itclover.tsp.utils.Files - -import scala.concurrent.duration.DurationInt -import scala.concurrent.{ExecutionContext, ExecutionContextExecutor} - -// In test cases, 'should' expressions are non-unit. Suppressing wartremover warnings about it -// Also, IO configurations use Any. -@SuppressWarnings(Array("org.wartremover.warts.NonUnitStatements", "org.wartremover.warts.Any")) -class BasicJdbcTest extends FlatSpec with SqlMatchers with ScalatestRouteTest with HttpService with ForAllTestContainer { - - implicit override val executionContext: ExecutionContextExecutor = scala.concurrent.ExecutionContext.global - implicit override val streamEnvironment: StreamExecutionEnvironment = { - StreamExecutionEnvironment.createLocalEnvironment() - } - streamEnvironment.setParallelism(4) // To prevent run out of network buffers on large number of CPUs (e.g. 32) - streamEnvironment.setMaxParallelism(30000) // For proper keyBy partitioning - - val spark = SparkSession - .builder() - .master("local") - .appName("TSP Spark test") - .config("spark.io.compression.codec", "snappy") - .getOrCreate() - - // to run blocking tasks. - val blockingExecutorContext: ExecutionContextExecutor = - ExecutionContext.fromExecutor( - new ThreadPoolExecutor( - 0, // corePoolSize - Int.MaxValue, // maxPoolSize - 1000L, //keepAliveTime - TimeUnit.MILLISECONDS, //timeUnit - new SynchronousQueue[Runnable](), //workQueue - new ThreadFactoryBuilder().setNameFormat("blocking-thread").setDaemon(true).build() - ) - ) - - implicit def defaultTimeout = RouteTestTimeout(300.seconds) - - val port = 8148 - implicit override val container = new JDBCContainer( - "yandex/clickhouse-server:latest", - port -> 8123 :: 9087 -> 9000 :: Nil, - "ru.yandex.clickhouse.ClickHouseDriver", - s"jdbc:clickhouse://localhost:$port/default", - waitStrategy = Some(Wait.forHttp("/")) - ) - - val inputConf = JDBCInputConf( - sourceId = 123, - jdbcUrl = container.jdbcUrl, - query = """select *, speed as "speed(1)(2)" from Test.SM_basic_wide""", // speed(1)(2) fancy colnames test - driverName = container.driverName, - datetimeField = 'datetime, - unitIdField = Some('mechanism_id), - eventsMaxGapMs = Some(60000L), - defaultEventsGapMs = Some(1000L), - chunkSizeMs = Some(900000L), - partitionFields = Seq('series_id, 'mechanism_id) - ) - - val typeCastingInputConf = inputConf.copy(query = "select * from Test.SM_typeCasting_wide limit 1000") - - val rowSchema = NewRowSchema('series_storage, 'from, 'to, ('app, 1), 'id, 'subunit) - - val outputConf = JDBCOutputConf( - "Test.SM_basic_patterns", - rowSchema, - s"jdbc:clickhouse://localhost:$port/default", - "ru.yandex.clickhouse.ClickHouseDriver" - ) - - val basicAssertions = Seq( - RawPattern(1, "speed < 15"), - RawPattern(2, """"speed(1)(2)" > 10"""), - RawPattern(3, "speed > 10.0", Some(Map("test" -> "test")), Some(540), Some(Seq('speed))) - ) - val typesCasting = Seq(RawPattern(10, "speed = 15"), RawPattern(11, "speed64 < 15.0")) - val errors = Seq(RawPattern(20, "speed = QWE 15"), RawPattern(21, "speed64 < 15.0")) - - override def afterStart(): Unit = { - super.afterStart() - - Files - .readResource("/sql/test-db-schema.sql") - .mkString - .split(";") - .foreach(container.executeUpdate) - - Files - .readResource("/sql/wide/source-schema.sql") - .mkString - .split(";") - .foreach(container.executeUpdate) - - Files - .readResource("/sql/wide/source-inserts.sql") - .mkString - .split(";") - .foreach(container.executeUpdate) - - Files - .readResource("/sql/sink-schema.sql") - .mkString - .split(";") - .foreach(container.executeUpdate) - } - - override def afterAll(): Unit = { - super.afterAll() - container.stop() - } - - "Basic assertions and forwarded fields" should "work for wide dense table" in { - - Post("/streamJob/from-jdbc/to-jdbc/?run_async=0", FindPatternsRequest("1", inputConf, outputConf, basicAssertions)) ~> - route ~> check { - status shouldEqual StatusCodes.OK - - // for 65001 - checkByQuery( - List(List(2.0)), - "SELECT toUnixTimestamp(to) - toUnixTimestamp(from) FROM Test.SM_basic_patterns WHERE id = 1 " - ) - - // for 65001 and 65002 - checkByQuery( - List(List(1.0), List(1.0)), - "SELECT toUnixTimestamp(to) - toUnixTimestamp(from) FROM Test.SM_basic_patterns WHERE id = 2 " - //"visitParamExtractString(context, 'mechanism_id') = '65001'" - ) - - // for 65001 - checkByQuery( - List(List(1.0)), - "SELECT toUnixTimestamp(to) - toUnixTimestamp(from) FROM Test.SM_basic_patterns WHERE id = 3 " - ) - } - } - - "Types casting" should "work for wide dense table" in { - Post( - "/streamJob/from-jdbc/to-jdbc/?run_async=0", - FindPatternsRequest("1", typeCastingInputConf, outputConf, typesCasting) - ) ~> - route ~> check { - status shouldEqual StatusCodes.OK - // For 65001 - checkByQuery( - List(List(0.0)), - "SELECT toUnixTimestamp(to) - toUnixTimestamp(from) FROM Test.SM_basic_patterns WHERE id = 10" - ) - // For 65001 and 65002 - checkByQuery( - List(List(2.0), List(0.0)), - "SELECT toUnixTimestamp(to) - toUnixTimestamp(from) FROM Test.SM_basic_patterns WHERE id = 11" - ) - } - } -} diff --git a/integration/correctness/src/test/scala/ru/itclover/tsp/http/BasicJdbcToKafkaTest.scala b/integration/correctness/src/test/scala/ru/itclover/tsp/http/BasicJdbcToKafkaTest.scala deleted file mode 100644 index 2903d39fe..000000000 --- a/integration/correctness/src/test/scala/ru/itclover/tsp/http/BasicJdbcToKafkaTest.scala +++ /dev/null @@ -1,155 +0,0 @@ -package ru.itclover.tsp.http - -import java.util.concurrent.{SynchronousQueue, ThreadPoolExecutor, TimeUnit} -import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest} -import com.dimafeng.testcontainers._ -import com.google.common.util.concurrent.ThreadFactoryBuilder -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.spark.sql.SparkSession -import org.scalatest.FlatSpec -import org.testcontainers.containers.wait.strategy.Wait -import ru.itclover.tsp.core.RawPattern -import ru.itclover.tsp.http.domain.input.FindPatternsRequest -import ru.itclover.tsp.http.utils.{JDBCContainer, SqlMatchers} -import ru.itclover.tsp.io.input.JDBCInputConf -import ru.itclover.tsp.io.output.{KafkaOutputConf, NewRowSchema} -import ru.itclover.tsp.utils.Files - -import scala.concurrent.duration.DurationInt -import scala.concurrent.{ExecutionContext, ExecutionContextExecutor} - -@SuppressWarnings(Array("org.wartremover.warts.Any")) -class BasicJdbcToKafkaTest - extends FlatSpec - with SqlMatchers - with ScalatestRouteTest - with HttpService - with ForAllTestContainer { - - implicit override val executionContext: ExecutionContextExecutor = scala.concurrent.ExecutionContext.global - implicit override val streamEnvironment: StreamExecutionEnvironment = - StreamExecutionEnvironment.createLocalEnvironment() - streamEnvironment.setParallelism(4) // To prevent run out of network buffers on large number of CPUs (e.g. 32) - streamEnvironment.setMaxParallelism(30000) // For proper keyBy partitioning - - val spark = SparkSession - .builder() - .master("local") - .appName("TSP Spark test") - .config("spark.io.compression.codec", "snappy") - .getOrCreate() - - // to run blocking tasks. - val blockingExecutorContext: ExecutionContextExecutor = - ExecutionContext.fromExecutor( - new ThreadPoolExecutor( - 0, // corePoolSize - Int.MaxValue, // maxPoolSize - 1000L, //keepAliveTime - TimeUnit.MILLISECONDS, //timeUnit - new SynchronousQueue[Runnable](), //workQueue - new ThreadFactoryBuilder().setNameFormat("blocking-thread").setDaemon(true).build() - ) - ) - - implicit def defaultTimeout = RouteTestTimeout(300.seconds) - - val port = 8170 - - val clickhouseContainer = new JDBCContainer( - "yandex/clickhouse-server:latest", - port -> 8123 :: 9080 -> 9000 :: Nil, - "ru.yandex.clickhouse.ClickHouseDriver", - s"jdbc:clickhouse://localhost:$port/default", - waitStrategy = Some(Wait.forHttp("/").forStatusCode(200).forStatusCode(400)) - ) - - val inputConf = JDBCInputConf( - sourceId = 123, - jdbcUrl = clickhouseContainer.jdbcUrl, - query = """select *, speed as "speed(1)(2)" from Test.SM_basic_wide""", // speed(1)(2) fancy colnames test - driverName = clickhouseContainer.driverName, - datetimeField = 'datetime, - unitIdField = Some('unit), - eventsMaxGapMs = Some(60000L), - defaultEventsGapMs = Some(1000L), - chunkSizeMs = Some(900000L), - partitionFields = Seq('series_id, 'mechanism_id) - ) - - val kafkaPort = 8092 - - val kafkaContainer = KafkaContainer() - - implicit override val container = MultipleContainers(clickhouseContainer, kafkaContainer) - - val typeCastingInputConf = inputConf.copy(query = "select * from Test.SM_typeCasting_wide limit 1000") - - val rowSchema = NewRowSchema('series_storage, 'from, 'to, ('app, 1), 'id, 'subunit) - - val outputConf = KafkaOutputConf( - "127.0.0.1:9092", - "SM_basic_patterns", - Some("json"), - rowSchema - ) - - val basicAssertions = Seq( - RawPattern(1, "speed < 15"), - RawPattern(2, """"speed(1)(2)" > 10"""), - RawPattern(3, "speed > 10.0", Some(Map("test" -> "test")), Some(540), Some(Seq('speed))) - ) - val typesCasting = Seq(RawPattern(10, "speed = 15"), RawPattern(11, "speed64 < 15.0")) - val errors = Seq(RawPattern(20, "speed = QWE 15"), RawPattern(21, "speed64 < 15.0")) - - override def afterStart(): Unit = { - super.afterStart() - - Files - .readResource("/sql/test-db-schema.sql") - .mkString - .split(";") - .foreach(clickhouseContainer.executeUpdate) - - Files - .readResource("/sql/wide/source-schema.sql") - .mkString - .split(";") - .foreach(clickhouseContainer.executeUpdate) - - Files - .readResource("/sql/wide/source-inserts.sql") - .mkString - .split(";") - .foreach(clickhouseContainer.executeUpdate) - - Files - .readResource("/sql/sink-schema.sql") - .mkString - .split(";") - .foreach(clickhouseContainer.executeUpdate) - } - - override def afterAll(): Unit = { - super.afterAll() - container.stop() - } - - "Basic assertions and forwarded fields" should "work for wide dense table" in { - - Post("/streamJob/from-jdbc/to-kafka/?run_async=0", FindPatternsRequest("1", inputConf, outputConf, basicAssertions)) ~> - route ~> check { - //status shouldEqual StatusCodes.OK - } - } - - "Types casting" should "work for wide dense table" in { - Post( - "/streamJob/from-jdbc/to-kafka/?run_async=0", - FindPatternsRequest("1", typeCastingInputConf, outputConf, typesCasting) - ) ~> - route ~> check { - //status shouldEqual StatusCodes.OK - } - } -} diff --git a/integration/correctness/src/test/scala/ru/itclover/tsp/http/BasicKafkaTest.scala b/integration/correctness/src/test/scala/ru/itclover/tsp/http/BasicKafkaTest.scala deleted file mode 100644 index 8f5625653..000000000 --- a/integration/correctness/src/test/scala/ru/itclover/tsp/http/BasicKafkaTest.scala +++ /dev/null @@ -1,195 +0,0 @@ -//package ru.itclover.tsp.http -// -//import java.util.Properties -//import java.util.concurrent.{SynchronousQueue, ThreadPoolExecutor, TimeUnit} -//import akka.http.scaladsl.model.StatusCodes -//import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest} -//import com.dimafeng.testcontainers._ -//import com.google.common.util.concurrent.ThreadFactoryBuilder -//import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -//import org.apache.kafka.clients.consumer.{ConsumerRecord, KafkaConsumer} -//import org.apache.kafka.clients.producer._ -//import org.apache.kafka.common.serialization.StringDeserializer -//import org.apache.spark.sql.SparkSession -//import org.scalatest.FlatSpec -//import ru.itclover.tsp.core.RawPattern -//import ru.itclover.tsp.http.domain.input.FindPatternsRequest -//import ru.itclover.tsp.http.utils.SqlMatchers -//import ru.itclover.tsp.io.input.KafkaInputConf -//import ru.itclover.tsp.io.output.{KafkaOutputConf, NewRowSchema} -// -//import scala.collection.mutable.ArrayBuffer -//import scala.concurrent._ -//import scala.concurrent.duration.DurationInt -//import scala.util.{Failure, Success} -// -//// In test cases, 'should' expressions are non-unit. Suppressing wartremover warnings about it -//@SuppressWarnings(Array("org.wartremover.warts.NonUnitStatements")) -//class BasicKafkaTest -// extends FlatSpec -// with SqlMatchers -// with ScalatestRouteTest -// with HttpService -// with ForAllTestContainer { -// -// implicit override val executionContext: ExecutionContextExecutor = scala.concurrent.ExecutionContext.global -// implicit override val streamEnvironment: StreamExecutionEnvironment = -// StreamExecutionEnvironment.createLocalEnvironment() -// streamEnvironment.setParallelism(4) // To prevent run out of network buffers on large number of CPUs (e.g. 32) -// streamEnvironment.setMaxParallelism(30000) // For proper keyBy partitioning -// -// val spark = SparkSession.builder() -// .master("local") -// .appName("TSP Spark test") -// .config("spark.io.compression.codec", "snappy") -// .getOrCreate() -// -// // to run blocking tasks. -// val blockingExecutorContext: ExecutionContextExecutor = -// ExecutionContext.fromExecutor( -// new ThreadPoolExecutor( -// 0, // corePoolSize -// Int.MaxValue, // maxPoolSize -// 1000L, //keepAliveTime -// TimeUnit.MILLISECONDS, //timeUnit -// new SynchronousQueue[Runnable](), //workQueue -// new ThreadFactoryBuilder().setNameFormat("blocking-thread").setDaemon(true).build() -// ) -// ) -// -// implicit def defaultTimeout = RouteTestTimeout(300.seconds) -// -// val container: KafkaContainer = KafkaContainer() -// def servers = container.kafkaContainer.getBootstrapServers.replaceAll("PLAINTEXT://", "") -// -// private val inputTopic = "input_topic" -// private val outputTopic = "output_topic" -// -// private val partitionFields = Seq('series_id, 'mechanism_id) -// val rowSchema = NewRowSchema('series_storage, 'from, 'to, ('app, 1), 'id, 'subunit) -// -// private def kafkaInputConf() = KafkaInputConf( -// brokers = servers, -// topic = inputTopic, -// datetimeField = 'timestamp, -// partitionFields = partitionFields, -// fieldsTypes = Map("timestamp" -> "int64", "series_id" -> "string", "mechanism_id" -> "string", "speed" -> "float64"), -// serializer = Some("json"), -// eventsMaxGapMs = Some(100000L) -// ) -// -// private def kafkaOutputConf() = KafkaOutputConf( -// servers, -// outputTopic, -// Some("json"), -// rowSchema -// ) -// -// val messages = Seq( -// """{"timestamp": 1410127755,"series_id": "series1","mechanism_id": "65001","speed": 0.0}""", -// """{"timestamp": 1410127756,"series_id": "series1","mechanism_id": "65001","speed": 5.0}""", -// """{"timestamp": 1410127757,"series_id": "series1","mechanism_id": "65001","speed": 10.0}""", -// """{"timestamp": 1410127758,"series_id": "series1","mechanism_id": "65001","speed": 15.0}""", -// """{"timestamp": 1410127759,"series_id": "series1","mechanism_id": "65001","speed": 20.0}""", -// """{"timestamp": 1410127761,"series_id": "series1","mechanism_id": "65002","speed": 20.0}""", -// """{"timestamp": 1410127762,"series_id": "series1","mechanism_id": "65002","speed": 20.0}""" -// ) -// -// val basicAssertions = Seq( -// RawPattern(1, "speed < 15"), -// RawPattern(2, """"speed" > 10"""), -// RawPattern(3, "speed > 10.0", Some(Map("test" -> "test")), Some(540), Some(Seq('speed))) -// ) -// val windowPattern = Seq(RawPattern(20, "speed < 20 for 1 sec")) -// -// lazy val producer: KafkaProducer[String, String] = { -// val props = new Properties() -// props.put("bootstrap.servers", servers) -// -// props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") -// props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") -// -// new KafkaProducer[String, String](props) -// } -// -// def sendToKafka(topic: String, key: String, value: String): Future[RecordMetadata] = { -// def producerCallback(promise: Promise[RecordMetadata]): Callback = -// (metadata: RecordMetadata, exception: Exception) => { -// val result = if (exception == null) Success(metadata) else Failure(exception) -// promise.complete(result) -// } -// -// val promise = Promise[RecordMetadata]() -// producer.send(new ProducerRecord(topic, key, value), producerCallback(promise)) -// promise.future -// } -// -// lazy val kafkaConsumer: KafkaConsumer[String, String] = { -// val properties = new Properties() -// properties.put("bootstrap.servers", servers) -// properties.put("group.id", "consumer-tutorial") -// properties.put("key.deserializer", classOf[StringDeserializer]) -// properties.put("value.deserializer", classOf[StringDeserializer]) -// properties.put("auto.offset.reset", "earliest") -// -// new KafkaConsumer[String, String](properties) -// } -// -// def readFromKafkaForDuration( -// duration: scala.concurrent.duration.Duration, -// topics: String* -// ): Seq[ConsumerRecord[String, String]] = { -// import scala.collection.JavaConverters._ -// kafkaConsumer.subscribe(topics.asJava) -// val currentTime = System.currentTimeMillis() -// var results = ArrayBuffer.empty[ConsumerRecord[String, String]] -// while (System.currentTimeMillis() - currentTime < duration.toMillis) { -// results ++= kafkaConsumer.poll(java.time.Duration.ofSeconds(1)).asScala -// } -// results -// } -// -// override def afterStart(): Unit = { -// super.afterStart() -// -// container.start() -// def sendToInputTopic(value: String) = sendToKafka(inputTopic, "0", value) -// -// val sentMessages = Future.sequence(messages.map(sendToInputTopic)) -// -// Await.result(sentMessages, 10.seconds) -// } -// -// override def afterAll(): Unit = { -// super.afterAll() -// container.stop() -// } -// "Basic assertions and forwarded fields" should "work for wide dense table" in { -// -// Post( -// "/streamJob/from-kafka/to-kafka/?run_async=1", -// FindPatternsRequest("1", kafkaInputConf(), kafkaOutputConf(), basicAssertions) -// ) ~> -// route ~> check { -// status shouldEqual StatusCodes.OK -// } -// -// val result = readFromKafkaForDuration(10.seconds, outputTopic) -// result.size shouldBe 11 -// } -// -// "Simple window pattern" should "work for wide dense table" in { -// -// Post( -// "/streamJob/from-kafka/to-kafka/?run_async=1", -// FindPatternsRequest("2", kafkaInputConf(), kafkaOutputConf(), windowPattern) -// ) ~> -// route ~> check { -// status shouldEqual StatusCodes.OK -// } -// -// val result = readFromKafkaForDuration(20.seconds, outputTopic) -// result.size shouldBe 1 -// } -// -//} diff --git a/integration/correctness/src/test/scala/ru/itclover/tsp/http/MonitoringMockTest.scala b/integration/correctness/src/test/scala/ru/itclover/tsp/http/MonitoringMockTest.scala deleted file mode 100644 index dc104bf0f..000000000 --- a/integration/correctness/src/test/scala/ru/itclover/tsp/http/MonitoringMockTest.scala +++ /dev/null @@ -1,109 +0,0 @@ -package ru.itclover.tsp.http -import akka.actor.ActorSystem -import akka.http.scaladsl.model.{StatusCodes, Uri} -import akka.http.scaladsl.server.Directives -import akka.http.scaladsl.testkit.ScalatestRouteTest -import akka.stream.ActorMaterializer -import org.apache.spark.sql.SparkSession -import org.scalatest.concurrent.ScalaFutures -import org.scalatest.{AsyncFlatSpec, BeforeAndAfter, Matchers} -import ru.itclover.tsp.http.routes.MonitoringRoutes -import ru.itclover.tsp.http.services.streaming.FlinkMonitoringService -import ru.itclover.tsp.http.utils.MockServer - -import scala.concurrent._ - -// In test cases, 'should' expressions are non-unit. Suppressing wartremover warnings about it -@SuppressWarnings(Array("org.wartremover.warts.NonUnitStatements")) -class MonitoringMockTest - extends AsyncFlatSpec - with ScalatestRouteTest - with Matchers - with Directives - with BeforeAndAfter - with ScalaFutures { - - val port = 9034 - - // We cannot initialise `t` here yet - @SuppressWarnings(Array("org.wartremover.warts.Null", "org.wartremover.warts.Var")) - private var t: Thread = _ - - before { - t = new Thread(() => MockServer.startServer("127.0.0.1", port)) - t.start() - Thread.sleep(1000) - } - - after { - t.join(1000) - } - - "Monitoring service" should "work with mocked Flink service" in { - val monitoringService = FlinkMonitoringService(s"http://127.0.0.1:$port") - monitoringService.queryJobsOverview.map { res => - assert(res.jobs.length == 2) - } - - monitoringService.queryJobByName("job1").map { res => - assert(res.isDefined) - } - monitoringService.queryJobByName("job2").map { res => - assert(res.isDefined) - } - monitoringService.queryJobByName("job3").map { res => - assert(res.isEmpty) - } - - monitoringService.queryJobExceptions("one").map { res => - assert(res.isEmpty) - } - monitoringService.queryJobInfo("job1").map { res => - assert(res.map(x => x.jid).getOrElse("error") == "1") - } - - monitoringService.sendStopQuery("job1").map { res => - assert(res.isDefined) - } - monitoringService.sendStopQuery("job2").map { res => - assert(res.isDefined) - } - monitoringService.sendStopQuery("job3").map { res => - assert(res.isEmpty) - } - } - - "Monitoring routes" should "work" in { - val monitoringRoutes = new MonitoringRoutes { - implicit override val actors: ActorSystem = ActorSystem("TSP-monitoring-test") - implicit override val materializer: ActorMaterializer = ActorMaterializer()(actors) - implicit override val executionContext: ExecutionContextExecutor = system.dispatcher - override val uri: Uri = s"http://127.0.0.1:$port" - override val spark = SparkSession - .builder() - .master("local") - .appName("TSP Spark test") - .config("spark.io.compression.codec", "snappy") - .getOrCreate() - } - Get("/metainfo/getVersion") ~> monitoringRoutes.route ~> check { - response.status shouldBe StatusCodes.OK - } - - Get("/jobs/overview") ~> monitoringRoutes.route ~> check { - response.status shouldBe StatusCodes.OK - } - - Get("/job/3/status") ~> monitoringRoutes.route ~> check { - response.status shouldBe StatusCodes.BadRequest - } - - Get("/job/3/exceptions") ~> monitoringRoutes.route ~> check { - response.status shouldBe StatusCodes.BadRequest - } - - Get("/job/3/stop") ~> monitoringRoutes.route ~> check { - response.status shouldBe StatusCodes.BadRequest - } - } -} diff --git a/integration/correctness/src/test/scala/ru/itclover/tsp/http/NarrowTableTest.scala b/integration/correctness/src/test/scala/ru/itclover/tsp/http/NarrowTableTest.scala deleted file mode 100644 index b56ed246e..000000000 --- a/integration/correctness/src/test/scala/ru/itclover/tsp/http/NarrowTableTest.scala +++ /dev/null @@ -1,141 +0,0 @@ -package ru.itclover.tsp.http - -import java.util.concurrent.{SynchronousQueue, ThreadPoolExecutor, TimeUnit} -import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest} -import com.dimafeng.testcontainers.ForAllTestContainer -import com.google.common.util.concurrent.ThreadFactoryBuilder -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.spark.sql.SparkSession -import org.scalatest.FlatSpec -import org.testcontainers.containers.wait.strategy.Wait -import ru.itclover.tsp.RowWithIdx -import ru.itclover.tsp.core.RawPattern -import ru.itclover.tsp.http.domain.input.FindPatternsRequest -import ru.itclover.tsp.http.utils.{JDBCContainer, SqlMatchers} -import ru.itclover.tsp.io.input.{JDBCInputConf, NarrowDataUnfolding} -import ru.itclover.tsp.io.output.{JDBCOutputConf, NewRowSchema} -import ru.itclover.tsp.utils.Files - -import scala.concurrent.duration.DurationInt -import scala.concurrent.{ExecutionContext, ExecutionContextExecutor} - -// In test cases, 'should' expressions are non-unit. Suppressing wartremover warnings about it -// Also, some test cases indirectly use Any type. -@SuppressWarnings(Array("org.wartremover.warts.NonUnitStatements", "org.wartremover.warts.Any")) -class NarrowTableTest - extends FlatSpec - with SqlMatchers - with ScalatestRouteTest - with HttpService - with ForAllTestContainer { - - implicit override val executionContext: ExecutionContextExecutor = scala.concurrent.ExecutionContext.global - implicit override val streamEnvironment: StreamExecutionEnvironment = - StreamExecutionEnvironment.createLocalEnvironment() - streamEnvironment.setMaxParallelism(30000) // For proper keyBy partitioning - - val spark = SparkSession - .builder() - .master("local") - .appName("TSP Spark test") - .config("spark.io.compression.codec", "snappy") - .getOrCreate() - - // to run blocking tasks. - val blockingExecutorContext: ExecutionContextExecutor = - ExecutionContext.fromExecutor( - new ThreadPoolExecutor( - 0, // corePoolSize - Int.MaxValue, // maxPoolSize - 1000L, //keepAliveTime - TimeUnit.MILLISECONDS, //timeUnit - new SynchronousQueue[Runnable](), //workQueue - new ThreadFactoryBuilder().setNameFormat("blocking-thread").setDaemon(true).build() - ) - ) - - implicit def defaultTimeout = RouteTestTimeout(300.seconds) - - val port = 8151 - implicit override val container = new JDBCContainer( - "yandex/clickhouse-server:latest", - port -> 8123 :: 9088 -> 9000 :: Nil, - "ru.yandex.clickhouse.ClickHouseDriver", - s"jdbc:clickhouse://localhost:$port/default", - // 400 for the native CH port - waitStrategy = Some(Wait.forHttp("/").forStatusCode(200).forStatusCode(400)) - ) - - val transformation = NarrowDataUnfolding[RowWithIdx, Symbol, Any]( - 'key, - 'value, - Map('speed1 -> 1000, 'speed2 -> 1000) - ) - - val inputConf = JDBCInputConf( - sourceId = 123, - jdbcUrl = container.jdbcUrl, - query = """select * from Test.SM_basic_narrow""", // speed(1)(2) fancy colnames test - driverName = container.driverName, - datetimeField = 'datetime, - eventsMaxGapMs = Some(60000L), - defaultEventsGapMs = Some(1000L), - chunkSizeMs = Some(900000L), - partitionFields = Seq('series_id, 'mechanism_id), - dataTransformation = Some(transformation) - ) - - val rowSchema = NewRowSchema('series_storage, 'from, 'to, ('app, 1), 'id, 'subunit) - - val outputConf = JDBCOutputConf( - "Test.SM_basic_patterns", - rowSchema, - s"jdbc:clickhouse://localhost:$port/default", - "ru.yandex.clickhouse.ClickHouseDriver" - ) - - val basicAssertions = Seq( - RawPattern(1, "speed1 < 15"), - RawPattern(2, """"speed2" > 10""") - ) - - override def afterStart(): Unit = { - super.afterStart() - - Files - .readResource("/sql/test-db-schema.sql") - .mkString - .split(";") - .foreach(container.executeUpdate) - - Files - .readResource("/sql/narrow/source-schema.sql") - .mkString - .split(";") - .foreach(container.executeUpdate) - - Files - .readResource("/sql/narrow/source-inserts.sql") - .mkString - .split(";") - .foreach(container.executeUpdate) - - Files - .readResource("/sql/sink-schema.sql") - .mkString - .split(";") - .foreach(container.executeUpdate) - } - - override def afterAll(): Unit = { - super.afterAll() - container.stop() - } - - "Basic assertions and forwarded fields" should "work for wide dense table" in { - Post("/streamJob/from-jdbc/to-jdbc/?run_async=0", FindPatternsRequest("1", inputConf, outputConf, basicAssertions)) ~> - route ~> check { - //status shouldEqual StatusCodes.OK - } - } -} diff --git a/integration/correctness/src/test/scala/ru/itclover/tsp/http/NonExistentBaseTest.scala b/integration/correctness/src/test/scala/ru/itclover/tsp/http/NonExistentBaseTest.scala deleted file mode 100644 index edbc0a95e..000000000 --- a/integration/correctness/src/test/scala/ru/itclover/tsp/http/NonExistentBaseTest.scala +++ /dev/null @@ -1,71 +0,0 @@ -package ru.itclover.tsp.http - -import java.util.concurrent.{SynchronousQueue, ThreadPoolExecutor, TimeUnit} -import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.testkit.ScalatestRouteTest -import com.google.common.util.concurrent.ThreadFactoryBuilder -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.spark.sql.SparkSession -import org.scalatest.FlatSpec -import ru.itclover.tsp.http.domain.input.FindPatternsRequest -import ru.itclover.tsp.http.utils.SqlMatchers -import ru.itclover.tsp.io.input.JDBCInputConf -import ru.itclover.tsp.io.output.{JDBCOutputConf, NewRowSchema} - -import scala.concurrent.{ExecutionContext, ExecutionContextExecutor} - -class NonExistentBaseTest extends FlatSpec with SqlMatchers with ScalatestRouteTest with HttpService { - implicit override val executionContext: ExecutionContextExecutor = scala.concurrent.ExecutionContext.global - implicit override val streamEnvironment: StreamExecutionEnvironment = - StreamExecutionEnvironment.createLocalEnvironment() - - val spark = SparkSession - .builder() - .master("local") - .appName("TSP Spark test") - .config("spark.io.compression.codec", "snappy") - .getOrCreate() - - // to run blocking tasks. - val blockingExecutorContext: ExecutionContextExecutor = - ExecutionContext.fromExecutor( - new ThreadPoolExecutor( - 0, // corePoolSize - Int.MaxValue, // maxPoolSize - 1000L, //keepAliveTime - TimeUnit.MILLISECONDS, //timeUnit - new SynchronousQueue[Runnable](), //workQueue - new ThreadFactoryBuilder().setNameFormat("blocking-thread").setDaemon(true).build() - ) - ) - - val dummyPort = 6000 - - val inputConf = JDBCInputConf( - sourceId = 123, - jdbcUrl = s"jdbc:clickhouse://localhost:$dummyPort/default", - query = """select *, speed as "speed(1)(2)" from Test.SM_basic_wide""", // speed(1)(2) fancy colnames test - driverName = "ru.yandex.clickhouse.ClickHouseDriver", - datetimeField = 'datetime, - eventsMaxGapMs = Some(60000L), - defaultEventsGapMs = Some(1000L), - chunkSizeMs = Some(900000L), - partitionFields = Seq('series_id, 'mechanism_id) - ) - - val rowSchema = NewRowSchema('series_storage, 'from, 'to, ('app, 1), 'id, 'subunit) - - val outputConf = JDBCOutputConf( - "Test.SM_basic_patterns", - rowSchema, - s"jdbc:clickhouse://localhost:$dummyPort/default", - "ru.yandex.clickhouse.ClickHouseDriver" - ) - - "Non-existent database" should "give error upon execution" in { - Post("/streamJob/from-jdbc/to-jdbc/?run_async=0", FindPatternsRequest("1", inputConf, outputConf, Seq())) ~> - route ~> check { - status shouldEqual StatusCodes.BadRequest - } - } -} diff --git a/integration/correctness/src/test/scala/ru/itclover/tsp/http/RealDataHITest.scala b/integration/correctness/src/test/scala/ru/itclover/tsp/http/RealDataHITest.scala deleted file mode 100644 index 47145bc89..000000000 --- a/integration/correctness/src/test/scala/ru/itclover/tsp/http/RealDataHITest.scala +++ /dev/null @@ -1,156 +0,0 @@ -package ru.itclover.tsp.http - -import java.util.concurrent.{SynchronousQueue, ThreadPoolExecutor, TimeUnit} -import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest} -import com.dimafeng.testcontainers._ -import com.google.common.util.concurrent.ThreadFactoryBuilder -import com.typesafe.scalalogging.Logger -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.spark.sql.SparkSession -import org.scalatest.FlatSpec -import org.testcontainers.containers.wait.strategy.Wait -import ru.itclover.tsp.core.RawPattern -import ru.itclover.tsp.http.domain.input.FindPatternsRequest -import ru.itclover.tsp.http.domain.output.SuccessfulResponse.FinishedJobResponse -import ru.itclover.tsp.http.utils.{JDBCContainer, SqlMatchers} -import ru.itclover.tsp.io.input.JDBCInputConf -import ru.itclover.tsp.io.output.{JDBCOutputConf, NewRowSchema} -import ru.itclover.tsp.utils.Files - -import scala.concurrent.duration.DurationInt -import scala.concurrent.{ExecutionContext, ExecutionContextExecutor} -import scala.util.Success - -// In test cases, 'should' expressions are non-unit. Suppressing wartremover warnings about it -// Also, some test cases indirectly use Any type. -@SuppressWarnings(Array("org.wartremover.warts.NonUnitStatements", "org.wartremover.warts.Any")) -class RealDataHITest - extends FlatSpec - with SqlMatchers - with ScalatestRouteTest - with HttpService - with ForAllTestContainer { - - implicit def defaultTimeout = RouteTestTimeout(300.seconds) - implicit override val executionContext: ExecutionContextExecutor = scala.concurrent.ExecutionContext.global - implicit override val streamEnvironment: StreamExecutionEnvironment = - StreamExecutionEnvironment.createLocalEnvironment() - streamEnvironment.setParallelism(4) // To prevent run out of network buffers on large number of CPUs (e.g. 32) - streamEnvironment.setMaxParallelism(30000) // For proper keyBy partitioning - - val spark = SparkSession - .builder() - .master("local") - .appName("TSP Spark test") - .config("spark.io.compression.codec", "snappy") - .getOrCreate() - - // to run blocking tasks. - val blockingExecutorContext: ExecutionContextExecutor = - ExecutionContext.fromExecutor( - new ThreadPoolExecutor( - 0, // corePoolSize - Int.MaxValue, // maxPoolSize - 1000L, //keepAliveTime - TimeUnit.MILLISECONDS, //timeUnit - new SynchronousQueue[Runnable](), //workQueue - new ThreadFactoryBuilder().setNameFormat("blocking-thread").setDaemon(true).build() - ) - ) - - private val log = Logger("RealDataTest") - - val port = 8136 - - implicit override val container = new JDBCContainer( - "yandex/clickhouse-server:latest", - port -> 8123 :: 9083 -> 9000 :: Nil, - "ru.yandex.clickhouse.ClickHouseDriver", - s"jdbc:clickhouse://localhost:$port/default", - waitStrategy = Some(Wait.forHttp("/")) - ) - - val inputConf = JDBCInputConf( - sourceId = 123, - jdbcUrl = container.jdbcUrl, - query = "select * from Test.Bigdata_HI", - driverName = container.driverName, - datetimeField = 'dt, - eventsMaxGapMs = Some(60000L), - defaultEventsGapMs = Some(10000L), - chunkSizeMs = Some(900000L), - partitionFields = Seq('stock_num), - unitIdField = Some('stock_num), - patternsParallelism = Some(1) - ) - - val sinkSchema = NewRowSchema('series_storage, 'from, 'to, ('app, 1), 'id, 'subunit) - - val outputConf = JDBCOutputConf( - "Test.SM_basic_patterns", - sinkSchema, - s"jdbc:clickhouse://localhost:$port/default", - "ru.yandex.clickhouse.ClickHouseDriver" - ) - - val (timeRangeSec, assertions) = (1 to 80) -> Seq( - RawPattern(6, "HI__wagon_id__6 < 0.5"), - RawPattern(4, "HI__wagon_id__4 < 0.5") - ) - - override def afterStart(): Unit = { - super.afterStart() - - Files - .readResource("/sql/test-db-schema.sql") - .mkString - .split(";") - .foreach(container.executeUpdate) - - Files - .readResource("/sql/wide/bigdata-schema.sql") - .mkString - .split(";") - .foreach(container.executeUpdate) - - val csvData = Files - .readResource("/sql/wide/source_bigdata.csv") - .drop(1) - .mkString("\n") - - container.executeUpdate(s"INSERT INTO Test.Bigdata_HI FORMAT CSV\n${csvData}") - - Files - .readResource("/sql/sink-schema.sql") - .mkString - .split(";") - .foreach(container.executeUpdate) - } - - override def afterAll(): Unit = { - super.afterAll() - container.stop() - } - - "Basic assertions" should "work for wide dense table" in { - - Post("/streamJob/from-jdbc/to-jdbc/?run_async=0", FindPatternsRequest("1", inputConf, outputConf, assertions)) ~> - route ~> check { - status shouldEqual StatusCodes.OK - val resp = unmarshal[FinishedJobResponse](responseEntity) - resp shouldBe a[Success[_]] - val execTimeS = resp.map(_.response.execTimeSec).getOrElse(Double.MaxValue) - log.info(s"Test job completed for $execTimeS sec.") - - // Correctness TODO: check the actual values - checkByQuery(List(List(686.0)), "SELECT count(*) FROM Test.SM_basic_patterns WHERE id = 6") // was 1275 - checkByQuery(List(List(1078.0)), "SELECT count(*) FROM Test.SM_basic_patterns WHERE id = 4") // was 1832 - - // Performance - val fromT = timeRangeSec.headOption.map(_.toDouble).getOrElse(Double.NaN) - val toT = timeRangeSec.lastOption.map(_.toDouble).getOrElse(Double.NaN) - execTimeS should ((be >= fromT).and(be <= toT)) - } - } -} diff --git a/integration/correctness/src/test/scala/ru/itclover/tsp/http/RedisTest.scala b/integration/correctness/src/test/scala/ru/itclover/tsp/http/RedisTest.scala deleted file mode 100644 index 0f4fe8a63..000000000 --- a/integration/correctness/src/test/scala/ru/itclover/tsp/http/RedisTest.scala +++ /dev/null @@ -1,119 +0,0 @@ -//package ru.itclover.tsp.http -// -//import java.util.concurrent.{SynchronousQueue, ThreadPoolExecutor, TimeUnit} -// -//import akka.http.scaladsl.model.StatusCodes -//import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest} -//import com.dimafeng.testcontainers.{ForAllTestContainer, GenericContainer} -//import com.google.common.util.concurrent.ThreadFactoryBuilder -//import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -//import org.redisson.client.codec.ByteArrayCodec -//import org.scalatest.{FlatSpec, Matchers} -//import org.testcontainers.containers.wait.strategy.Wait -//import org.testcontainers.shaded.com.fasterxml.jackson.databind.ObjectMapper -//import ru.itclover.tsp.core.RawPattern -//import ru.itclover.tsp.http.domain.input.FindPatternsRequest -//import ru.itclover.tsp.io.input.RedisInputConf -// -//import scala.concurrent.duration.DurationInt -//import scala.concurrent.{ExecutionContext, ExecutionContextExecutor} -//import ru.itclover.tsp.io.output.{RedisOutputConf, RowSchema} -//import ru.itclover.tsp.services.RedisService -// -//import scala.collection.JavaConverters._ -// -//class RedisTest extends FlatSpec with ScalatestRouteTest with HttpService with ForAllTestContainer with Matchers { -// -// implicit def defaultTimeout = RouteTestTimeout(300.seconds) -// -// override implicit val streamEnvironment: StreamExecutionEnvironment = -// StreamExecutionEnvironment.createLocalEnvironment() -// -// override implicit val executionContext: ExecutionContextExecutor = ExecutionContext.global -// -// override val blockingExecutorContext: ExecutionContextExecutor = ExecutionContext.fromExecutor( -// new ThreadPoolExecutor( -// 0, // corePoolSize -// Int.MaxValue, // maxPoolSize -// 1000L, //keepAliveTime -// TimeUnit.MILLISECONDS, //timeUnit -// new SynchronousQueue[Runnable](), //workQueue -// new ThreadFactoryBuilder().setNameFormat("blocking-thread").setDaemon(true).build() -// ) -// ) -// -// val redisPort = 6383 -// -// override val container: GenericContainer = new GenericContainer( -// "redis:latest", -// waitStrategy = Some(Wait.forLogMessage(".*Ready to accept connections.*\\n", 1)) -// ) -// -// container.container.setPortBindings(List(s"$redisPort:6379").asJava) -// -// val redisURL = s"redis://@${container.containerIpAddress}:$redisPort/" -// -// val inputConf = RedisInputConf( -// url = redisURL, -// datetimeField = 'dt, -// partitionFields = Seq('stock_num), -// fieldsTypes = Map( -// "dt" -> "float64", -// "stock_num" -> "string", -// "test_int" -> "int8", -// "test_string" -> "string" -// ), -// key = "test_key", -// serializer = "json" -// ) -// -// val sinkSchema = RowSchema('series_storage, 'from, 'to, ('app, 1), 'id, 'timestamp, 'context, inputConf.partitionFields) -// -// val outputConf = RedisOutputConf( -// url = redisURL, -// key = "test_key", -// serializer = "json", -// rowSchema = sinkSchema -// ) -// -// val (timeRangeSec, assertions) = (1 to 80) -> Seq( -// RawPattern("6", "test_int > 20") -// ) -// -// override def afterStart(): Unit = { -// super.afterStart() -// Thread.sleep(8000) -// -// val redisInfo = RedisService.clientInstance(inputConf, inputConf.serializer) -// val client = redisInfo._1 -// -// val testData = Map[String, Any]( -// "dt" -> 1500000000.0, -// "stock_num" -> "0017", -// "test_int" -> 87, -// "test_string" -> "test" -// ) -// -// val mapper = new ObjectMapper() -// val jsonString = mapper.writeValueAsString(testData) -// -// val bucket = client.getBucket[Array[Byte]](inputConf.key, ByteArrayCodec.INSTANCE) -// bucket.set(jsonString.getBytes("UTF-8")) -// -// } -// -// override def afterAll(): Unit = { -// super.afterAll() -// container.stop() -// } -// -// "Redis test assertions" should "work for redis source" in { -// -// Post("/streamJob/from-redis/to-redis/?run_async=0", FindPatternsRequest("1", inputConf, outputConf, assertions)) ~> -// route ~> check { -// status shouldBe StatusCodes.OK -// } -// -// } -// -//} diff --git a/integration/correctness/src/test/scala/ru/itclover/tsp/http/SimpleCasesTest.scala b/integration/correctness/src/test/scala/ru/itclover/tsp/http/SimpleCasesTest.scala index f25abb6a1..cb16f8a7d 100644 --- a/integration/correctness/src/test/scala/ru/itclover/tsp/http/SimpleCasesTest.scala +++ b/integration/correctness/src/test/scala/ru/itclover/tsp/http/SimpleCasesTest.scala @@ -8,9 +8,9 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder import java.util.{Properties, UUID} import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord} -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.spark.sql.SparkSession import org.scalatest.{Assertion, FlatSpec} +import ru.itclover.tsp.utils.Files import scala.util.Failure // import org.scalatest.concurrent.Waiters._ @@ -19,16 +19,13 @@ import org.testcontainers.containers.wait.strategy.Wait import ru.itclover.tsp.core.RawPattern import ru.itclover.tsp.http.domain.input.FindPatternsRequest import ru.itclover.tsp.http.protocols.RoutesProtocols -import ru.itclover.tsp.http.utils.{InfluxDBContainer, JDBCContainer, SqlMatchers} -import ru.itclover.tsp.io.input.{InfluxDBInputConf, JDBCInputConf, KafkaInputConf, NarrowDataUnfolding, WideDataFilling} -import ru.itclover.tsp.io.output.{JDBCOutputConf, NewRowSchema} -import ru.itclover.tsp.utils.Files +import ru.itclover.tsp.http.utils.{JDBCContainer, SqlMatchers} import ru.itclover.tsp.spark.io.{ JDBCInputConf => SparkJDBCInputConf, JDBCOutputConf => SparkJDBCOutputConf, KafkaInputConf => SparkKafkaInputConf, KafkaOutputConf => SparkKafkaOutputConf, - NewRowSchema => SparkRowSchema + RowSchema => SparkRowSchema } import ru.itclover.tsp.spark.io.{NarrowDataUnfolding => SparkNDU, WideDataFilling => SparkWDF} import spray.json._ @@ -51,10 +48,6 @@ class SimpleCasesTest with ForAllTestContainer with RoutesProtocols { implicit override val executionContext: ExecutionContextExecutor = scala.concurrent.ExecutionContext.global - implicit override val streamEnvironment: StreamExecutionEnvironment = - StreamExecutionEnvironment.createLocalEnvironment() - streamEnvironment.setParallelism(1) - streamEnvironment.setMaxParallelism(30000) // For proper keyBy partitioning val spark = SparkSession .builder() @@ -89,21 +82,10 @@ class SimpleCasesTest waitStrategy = Some(Wait.forHttp("/")) ) - implicit val influxContainer = - new InfluxDBContainer( - "influxdb:1.7", - influxPort -> 8086 :: Nil, - s"http://localhost:$influxPort", - "Test", - "default", - waitStrategy = Some(Wait.forHttp("/").forStatusCode(200).forStatusCode(404)) - ) - val kafkaContainer = KafkaContainer() override val container = MultipleContainers( clickhouseContainer, - influxContainer, kafkaContainer ) @@ -197,92 +179,6 @@ class SimpleCasesTest val incidentsIvolgaTimestamps: List[List[Double]] = ivolgaIncidentsTimestamps.toList - val influxInputConf = InfluxDBInputConf( - sourceId = 300, - url = influxContainer.url, - query = "SELECT * FROM \"2te116u_tmy_test_simple_rules\" ORDER BY time", - userName = Some("default"), - password = Some("default"), - dbName = influxContainer.dbName, - eventsMaxGapMs = Some(60000L), - defaultEventsGapMs = Some(1000L), - chunkSizeMs = Some(900000L), - unitIdField = Some('loco_num), - partitionFields = Seq('loco_num, 'section, 'upload_id), - additionalTypeChecking = Some(false) - ) - - val wideInputConf = JDBCInputConf( - sourceId = 100, - jdbcUrl = clickhouseContainer.jdbcUrl, - query = "SELECT * FROM `2te116u_tmy_test_simple_rules` ORDER BY ts", - driverName = clickhouseContainer.driverName, - datetimeField = 'ts, - eventsMaxGapMs = Some(60000L), - defaultEventsGapMs = Some(1000L), - chunkSizeMs = Some(900000L), - unitIdField = Some('loco_num), - partitionFields = Seq('loco_num, 'section, 'upload_id) - ) - - val narrowInputConf = wideInputConf.copy( - sourceId = 200, - query = "SELECT * FROM math_test ORDER BY dt", - datetimeField = 'dt, - dataTransformation = Some(NarrowDataUnfolding('sensor_id, 'value_float, Map.empty, Some(Map.empty), Some(1000))) - ) - - val narrowInputIvolgaConf = wideInputConf.copy( - sourceId = 400, - query = "SELECT * FROM ivolga_test_narrow ORDER BY dt", - datetimeField = 'dt, - unitIdField = Some('stock_num), - partitionFields = Seq('stock_num, 'upload_id), - dataTransformation = Some( - NarrowDataUnfolding( - 'sensor_id, - 'value_float, - Map.empty, - Some(Map('value_str -> List('SOC_2_UKV1_UOVS))), - Some(15000L) - ) - ) - ) - - val wideInputIvolgaConf = wideInputConf.copy( - sourceId = 500, - query = "SELECT * FROM `ivolga_test_wide` ORDER BY ts", - unitIdField = Some('stock_num), - partitionFields = Seq('stock_num, 'upload_id), - dataTransformation = Some(WideDataFilling(Map.empty, defaultTimeout = Some(15000L))) - ) - - val wideRowSchema = NewRowSchema( - unitIdField = 'series_storage, - fromTsField = 'from, - toTsField = 'to, - appIdFieldVal = ('app, 1), - patternIdField = 'id, - subunitIdField = 'subunit - ) - lazy val wideKafkaInputConf = KafkaInputConf( - sourceId = 600, - brokers = kafkaBrokerUrl, - topic = "2te116u_tmy_test_simple_rules", - datetimeField = 'dt, - unitIdField = Some('loco_num), - partitionFields = Seq('loco_num, 'section, 'upload_id), - fieldsTypes = Map( - "dt" -> "float64", - "upload_id" -> "string", - "loco_num" -> "string", - "section" -> "string", - "POilDieselOut" -> "float64", - "SpeedThrustMin" -> "float64", - "PowerPolling" -> "float64" - ) - ) - val wideSparkInputConf = SparkJDBCInputConf( sourceId = 100, jdbcUrl = clickhouseContainer.jdbcUrl, @@ -360,27 +256,9 @@ class SimpleCasesTest dataTransformation = Some(SparkNDU('sensor_id, 'value_float, Map.empty, Some(Map.empty), Some(1000))) ) - val narrowRowSchema = wideRowSchema.copy( - appIdFieldVal = ('app, 2) - ) - - val influxRowSchema = wideRowSchema.copy( - appIdFieldVal = ('app, 3) - ) - - val narrowIvolgaRowSchema = wideRowSchema.copy( - appIdFieldVal = ('app, 4) - ) - - val wideIvolgaRowSchema = wideRowSchema.copy( - appIdFieldVal = ('app, 5) - ) - val chConnection = s"jdbc:clickhouse://localhost:$port/default" val chDriver = "ru.yandex.clickhouse.ClickHouseDriver" - val wideKafkaRowSchema = - NewRowSchema('series_storage, 'from, 'to, ('app, 4), 'id, 'subunit) val wideSparkRowSchema = SparkRowSchema('series_storage, 'from, 'to, ('app, 1), 'id, 'subunit) @@ -401,40 +279,6 @@ class SimpleCasesTest appIdFieldVal = ('app, 6) ) - val wideOutputConf = JDBCOutputConf( - tableName = "events_wide_test", - rowSchema = wideRowSchema, - jdbcUrl = chConnection, - driverName = chDriver - ) - - val narrowOutputConf = wideOutputConf.copy( - tableName = "events_narrow_test", - rowSchema = narrowRowSchema - ) - - val influxOutputConf = wideOutputConf.copy( - tableName = "events_influx_test", - rowSchema = influxRowSchema - ) - - val narrowOutputIvolgaConf = wideOutputConf.copy( - tableName = "events_narrow_ivolga_test", - rowSchema = narrowIvolgaRowSchema - ) - - val wideOutputIvolgaConf = wideOutputConf.copy( - tableName = "events_wide_ivolga_test", - rowSchema = wideIvolgaRowSchema - ) - - val wideKafkaOutputConf = JDBCOutputConf( - "events_wide_kafka_test", - wideKafkaRowSchema, - s"jdbc:clickhouse://localhost:$port/default", - "ru.yandex.clickhouse.ClickHouseDriver" - ) - val wideSparkOutputConf = SparkJDBCOutputConf( "events_wide_spark_test", wideSparkRowSchema, @@ -493,18 +337,6 @@ class SimpleCasesTest }) - Files - .readResource("/sql/infl-test-db-schema.sql") - .mkString - .split(";") - .foreach(influxContainer.executeQuery) - - Files - .readResource("/sql/test/cases-narrow-new.influx") - .mkString - .split(";") - .foreach(influxContainer.executeUpdate) - val insertInfo = Seq( ("math_test", "/sql/test/cases-narrow-new.csv"), ("ivolga_test_narrow", "/sql/test/cases-narrow-ivolga.csv"), @@ -584,7 +416,6 @@ class SimpleCasesTest override def afterAll(): Unit = { super.afterAll() clickhouseContainer.stop() - influxContainer.stop() container.stop() } @@ -603,7 +434,6 @@ class SimpleCasesTest "Data" should "load properly" in { checkByQuery(List(List(53.0)), "SELECT COUNT(*) FROM `2te116u_tmy_test_simple_rules`") checkByQuery(List(List(159.0)), "SELECT COUNT(*) FROM math_test") - checkInfluxByQuery(List(List(53.0, 53.0, 53.0)), "SELECT COUNT(*) FROM \"2te116u_tmy_test_simple_rules\"") } // "Cases 1-17, 43-53" should "work in wide table" in { diff --git a/flink/src/main/scala/ru/itclover/tsp/utils/CollectionsOps.scala b/integration/correctness/src/test/scala/ru/itclover/tsp/http/utils/CollectionsOps.scala similarity index 99% rename from flink/src/main/scala/ru/itclover/tsp/utils/CollectionsOps.scala rename to integration/correctness/src/test/scala/ru/itclover/tsp/http/utils/CollectionsOps.scala index cab36ccc9..a69136f89 100644 --- a/flink/src/main/scala/ru/itclover/tsp/utils/CollectionsOps.scala +++ b/integration/correctness/src/test/scala/ru/itclover/tsp/http/utils/CollectionsOps.scala @@ -61,4 +61,4 @@ object CollectionsOps { deqList } } -} +} \ No newline at end of file diff --git a/flink/src/main/scala/ru/itclover/tsp/utils/Files.scala b/integration/correctness/src/test/scala/ru/itclover/tsp/http/utils/Files.scala similarity index 99% rename from flink/src/main/scala/ru/itclover/tsp/utils/Files.scala rename to integration/correctness/src/test/scala/ru/itclover/tsp/http/utils/Files.scala index 5f0e2d213..0e4c07c9c 100644 --- a/flink/src/main/scala/ru/itclover/tsp/utils/Files.scala +++ b/integration/correctness/src/test/scala/ru/itclover/tsp/http/utils/Files.scala @@ -32,4 +32,4 @@ object Files { def rmFile(path: String): Boolean = new File(path).delete() -} +} \ No newline at end of file diff --git a/integration/correctness/src/test/scala/ru/itclover/tsp/http/utils/InfluxDBContainer.scala b/integration/correctness/src/test/scala/ru/itclover/tsp/http/utils/InfluxDBContainer.scala deleted file mode 100644 index 8ed648ace..000000000 --- a/integration/correctness/src/test/scala/ru/itclover/tsp/http/utils/InfluxDBContainer.scala +++ /dev/null @@ -1,66 +0,0 @@ -package ru.itclover.tsp.http.utils - -import com.dimafeng.testcontainers.SingleContainer -import org.influxdb.InfluxDB -import org.influxdb.dto.{Query, QueryResult} -import org.testcontainers.containers.wait.strategy.WaitStrategy -import org.testcontainers.containers.{BindMode, GenericContainer => OTCGenericContainer} -import ru.itclover.tsp.services.InfluxDBService - -import scala.collection.JavaConverters._ -import scala.language.existentials -import scala.util.{Failure, Success} - -// Default arguments for constructing a container are useful -@SuppressWarnings(Array("org.wartremover.warts.DefaultArguments")) -class InfluxDBContainer( - imageName: String, - val portsBindings: List[(Int, Int)] = List.empty, - val url: String, - val dbName: String, - val userName: String, - val password: String = "", - env: Map[String, String] = Map(), - command: Seq[String] = Seq(), - classpathResourceMapping: Seq[(String, String, BindMode)] = Seq(), - waitStrategy: Option[WaitStrategy] = None -) extends SingleContainer[OTCGenericContainer[_]] { - - type OTCContainer = OTCGenericContainer[T] forSome { type T <: OTCGenericContainer[T] } - implicit override val container: OTCContainer = new OTCGenericContainer(imageName) - - if (portsBindings.nonEmpty) { - val bindings = portsBindings.map { case (out, in) => s"${out.toString}:${in.toString}" } - container.setPortBindings(bindings.asJava) - } - env.foreach(Function.tupled(container.withEnv)) - if (command.nonEmpty) { - val _ = container.withCommand(command: _*) - } - classpathResourceMapping.foreach(Function.tupled(container.withClasspathResourceMapping)) - waitStrategy.foreach(container.waitingFor) - - // We cannot initialise `db` here yet - @SuppressWarnings(Array("org.wartremover.warts.Null", "org.wartremover.warts.Var")) - var db: InfluxDB = _ - - // This must throw an exception upon error - @SuppressWarnings(Array("org.wartremover.warts.Throw")) - override def start(): Unit = { - super.start() - val conf = InfluxDBService.InfluxConf(url, dbName, Some(userName), Some(password), 30L, false) - db = InfluxDBService.connectDb(conf) match { - case Success(database) => database - case Failure(exception) => throw exception - } - } - - override def stop(): Unit = { - super.stop() - db.close() - } - - def executeQuery(sql: String): QueryResult = db.query(new Query(sql, dbName)) - - def executeUpdate(sql: String): Unit = db.write(sql) -} diff --git a/integration/correctness/src/test/scala/ru/itclover/tsp/http/utils/SqlMatchers.scala b/integration/correctness/src/test/scala/ru/itclover/tsp/http/utils/SqlMatchers.scala index 6660928f8..c8ec15bb1 100644 --- a/integration/correctness/src/test/scala/ru/itclover/tsp/http/utils/SqlMatchers.scala +++ b/integration/correctness/src/test/scala/ru/itclover/tsp/http/utils/SqlMatchers.scala @@ -51,39 +51,5 @@ trait SqlMatchers extends Matchers { } } - // Here, default argument for `epsilon` is useful. - @SuppressWarnings(Array("org.wartremover.warts.DefaultArguments")) - def checkInfluxByQuery(expectedValues: Seq[Seq[Double]], query: String, epsilon: Double = 0.0001)( - implicit container: InfluxDBContainer - ): Assertion = { - val resultSet = container.executeQuery(query) - val results: List[Seq[Double]] = resultSet.getResults - .get(0) - .getSeries - .asScala - .map( - _.getValues.asScala.map(_.asScala.drop(1).map(x => Try(x.toString.toDouble).getOrElse(Double.NaN)).toList).toList - ) - .foldLeft(List.empty[Seq[Double]])(_ ++ _) - logger.info( - s"Expected Values: [${toStringRepresentation(expectedValues)}], " + - s"actual values: [${toStringRepresentation(results)}]" - ) - implicit val customEqualityList: Equality[List[Double]] = (a: scala.List[Double], b: Any) => { - a.size == b.asInstanceOf[Iterable[Double]].size && a.zip(b.asInstanceOf[Iterable[Double]]).forall { - case (x, y) => Math.abs(x - y) < epsilon - } - } - val unfound = expectedValues.filter(x => !results.exists(y => customEqualityList.areEqual(x.toList, y))) - val unexpected = results.filter(x => !expectedValues.exists(y => customEqualityList.areEqual(x.toList, y))) - withClue( - s"Expected but not found:\n [${toStringRepresentation(unfound)}]\n; found\n [${toStringRepresentation(unexpected)}]\n instead" - ) { - // results should ===(expectedValues) - unfound shouldBe empty - unexpected shouldBe empty - } - } - def toStringRepresentation(data: Seq[Seq[Double]]): String = data.map(_.mkString(", ")).mkString(";\n") } diff --git a/integration/performance/src/test/scala/ru/itclover/tsp/http/AggregatorsPerfTest.scala b/integration/performance/src/test/scala/ru/itclover/tsp/http/AggregatorsPerfTest.scala index 5034d115e..16cc90f72 100644 --- a/integration/performance/src/test/scala/ru/itclover/tsp/http/AggregatorsPerfTest.scala +++ b/integration/performance/src/test/scala/ru/itclover/tsp/http/AggregatorsPerfTest.scala @@ -5,8 +5,7 @@ import org.apache.spark.sql.SparkSession import org.scalatest.FlatSpec import ru.itclover.tsp.core.RawPattern import ru.itclover.tsp.http.utils.{HttpServiceMatchers, JDBCContainer} -import ru.itclover.tsp.io.input.JDBCInputConf -import ru.itclover.tsp.io.output.{JDBCOutputConf, NewRowSchema} +import ru.itclover.tsp.spark.io.{JDBCInputConf, JDBCOutputConf, RowSchema} import ru.itclover.tsp.utils.Files class AggregatorsPerfTest extends FlatSpec with HttpServiceMatchers with ForAllTestContainer { @@ -53,7 +52,7 @@ class AggregatorsPerfTest extends FlatSpec with HttpServiceMatchers with ForAllT partitionFields = Seq('t1) ) - val sinkSchema = NewRowSchema('series_storage, 'from, 'to, ('app, 1), 'id, 'subunit) + val sinkSchema = RowSchema('series_storage, 'from, 'to, ('app, 1), 'id, 'subunit) val outputConf = JDBCOutputConf( "Test.SM_basic_patterns", diff --git a/integration/performance/src/test/scala/ru/itclover/tsp/http/DebugRealDataTest.scala b/integration/performance/src/test/scala/ru/itclover/tsp/http/DebugRealDataTest.scala index 559725324..5aafd12e5 100644 --- a/integration/performance/src/test/scala/ru/itclover/tsp/http/DebugRealDataTest.scala +++ b/integration/performance/src/test/scala/ru/itclover/tsp/http/DebugRealDataTest.scala @@ -9,8 +9,7 @@ import org.testcontainers.containers.wait.strategy.Wait import ru.itclover.tsp.http.domain.input.FindPatternsRequest import ru.itclover.tsp.http.domain.output.SuccessfulResponse.FinishedJobResponse import ru.itclover.tsp.http.utils.{HttpServiceMatchers, JDBCContainer} -import ru.itclover.tsp.io.input.JDBCInputConf -import ru.itclover.tsp.io.output.JDBCOutputConf +import ru.itclover.tsp.spark.io.{JDBCInputConf, JDBCOutputConf} import ru.itclover.tsp.utils.Files import spray.json.JsonParser import spray.json.ParserInput.StringBasedParserInput diff --git a/integration/performance/src/test/scala/ru/itclover/tsp/http/RealDataPerfTest.scala b/integration/performance/src/test/scala/ru/itclover/tsp/http/RealDataPerfTest.scala index c21469504..77a77520a 100644 --- a/integration/performance/src/test/scala/ru/itclover/tsp/http/RealDataPerfTest.scala +++ b/integration/performance/src/test/scala/ru/itclover/tsp/http/RealDataPerfTest.scala @@ -10,8 +10,7 @@ import ru.itclover.tsp.core.RawPattern import ru.itclover.tsp.http.domain.input.FindPatternsRequest import ru.itclover.tsp.http.domain.output.SuccessfulResponse.FinishedJobResponse import ru.itclover.tsp.http.utils.{HttpServiceMatchers, JDBCContainer} -import ru.itclover.tsp.io.input.JDBCInputConf -import ru.itclover.tsp.io.output.{JDBCOutputConf, NewRowSchema} +import ru.itclover.tsp.spark.io.{JDBCInputConf, JDBCOutputConf, RowSchema} import ru.itclover.tsp.utils.Files import scala.util.Success @@ -53,7 +52,7 @@ class RealDataPerfTest extends FlatSpec with HttpServiceMatchers with ForAllTest partitionFields = Seq('stock_num) ) - val sinkSchema = NewRowSchema('series_storage, 'from, 'to, ('app, 1), 'id, 'subunit) + val sinkSchema = RowSchema('series_storage, 'from, 'to, ('app, 1), 'id, 'subunit) val outputConf = JDBCOutputConf( "Test.SM_basic_patterns", diff --git a/integration/performance/src/test/scala/ru/itclover/tsp/http/utils/HttpServiceMatchers.scala b/integration/performance/src/test/scala/ru/itclover/tsp/http/utils/HttpServiceMatchers.scala index ecd697b07..6d9abeeba 100644 --- a/integration/performance/src/test/scala/ru/itclover/tsp/http/utils/HttpServiceMatchers.scala +++ b/integration/performance/src/test/scala/ru/itclover/tsp/http/utils/HttpServiceMatchers.scala @@ -6,7 +6,6 @@ import akka.http.scaladsl.model.StatusCodes import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest} import com.google.common.util.concurrent.ThreadFactoryBuilder import com.typesafe.scalalogging.Logger -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.scalatest.{Matchers, Suite} import ru.itclover.tsp.http.HttpService import ru.itclover.tsp.http.domain.output.SuccessfulResponse.FinishedJobResponse @@ -24,8 +23,6 @@ import scala.util.Success trait HttpServiceMatchers extends ScalatestRouteTest with Matchers with HttpService { self: Suite => implicit override val executionContext: ExecutionContextExecutor = scala.concurrent.ExecutionContext.global - implicit override val streamEnvironment = StreamExecutionEnvironment.createLocalEnvironment() - streamEnvironment.setMaxParallelism(30000) // For proper keyBy partitioning // to run blocking tasks. val blockingExecutorContext: ExecutionContextExecutor = diff --git a/integration/performance/src/test/scala/ru/itclover/tsp/http/utils/InfluxDBContainer.scala b/integration/performance/src/test/scala/ru/itclover/tsp/http/utils/InfluxDBContainer.scala deleted file mode 100644 index 2e43c69cf..000000000 --- a/integration/performance/src/test/scala/ru/itclover/tsp/http/utils/InfluxDBContainer.scala +++ /dev/null @@ -1,66 +0,0 @@ -package ru.itclover.tsp.http.utils - -import com.dimafeng.testcontainers.SingleContainer -import org.influxdb.InfluxDB -import org.influxdb.dto.{Query, QueryResult} -import org.testcontainers.containers.wait.strategy.WaitStrategy -import org.testcontainers.containers.{BindMode, GenericContainer => OTCGenericContainer} -import ru.itclover.tsp.services.InfluxDBService - -import scala.collection.JavaConverters._ -import scala.language.existentials -import scala.util.{Failure, Success} - -// Default arguments for constructing a container are useful -@SuppressWarnings(Array("org.wartremover.warts.DefaultArguments")) -class InfluxDBContainer( - imageName: String, - val portsBindings: List[(Int, Int)] = List.empty, - val url: String, - val dbName: String, - val userName: String, - val password: String = "", - env: Map[String, String] = Map(), - command: Seq[String] = Seq(), - classpathResourceMapping: Seq[(String, String, BindMode)] = Seq(), - waitStrategy: Option[WaitStrategy] = None - ) extends SingleContainer[OTCGenericContainer[_]] { - - type OTCContainer = OTCGenericContainer[T] forSome { type T <: OTCGenericContainer[T] } - implicit override val container: OTCContainer = new OTCGenericContainer(imageName) - - if (portsBindings.nonEmpty) { - val bindings = portsBindings.map { case (out, in) => s"${out.toString}:${in.toString}" } - val _ = container.setPortBindings(bindings.asJava) - } - env.foreach(Function.tupled(container.withEnv)) - if (command.nonEmpty) { - val _ = container.withCommand(command: _*) - } - classpathResourceMapping.foreach(Function.tupled(container.withClasspathResourceMapping)) - waitStrategy.foreach(container.waitingFor) - - // We cannot initialise `db` here yet - @SuppressWarnings(Array("org.wartremover.warts.Null", "org.wartremover.warts.Var")) - var db: InfluxDB = _ - - // This must throw an exception upon error - @SuppressWarnings(Array("org.wartremover.warts.Throw")) - override def start(): Unit = { - super.start() - val conf = InfluxDBService.InfluxConf(url, dbName, Some(userName), Some(password), 30L, false) - db = InfluxDBService.connectDb(conf) match { - case Success(database) => database - case Failure(exception) => throw exception - } - } - - override def stop(): Unit = { - super.stop() - db.close() - } - - def executeQuery(sql: String): QueryResult = db.query(new Query(sql, dbName)) - - def executeUpdate(sql: String): Unit = db.write(sql) -} diff --git a/project/Library.scala b/project/Library.scala index a79d8bc6d..67298d8a5 100644 --- a/project/Library.scala +++ b/project/Library.scala @@ -10,7 +10,6 @@ object Version { val clickhouse = "0.3.0" val chNative = "2.4.2" - val flink = "1.10.0" val akka = "2.5.25" val akkaHttp = "10.1.9" @@ -88,35 +87,6 @@ object Library { val postgre: Seq[ModuleID] = Seq("org.postgresql" % "postgresql" % Version.postgres) val dbDrivers: Seq[ModuleID] = influx ++ clickhouse ++ postgre - val flinkCore: Seq[ModuleID] = Seq("org.apache.flink" %% "flink-scala" % Version.flink) - - val flink: Seq[ModuleID] = flinkCore ++ Seq( - "org.apache.flink" % "flink-runtime-web_2.12" % Version.flink exclude ( - "org.slf4j", "slf4j-log4j12" - ), - "org.apache.flink" %% "flink-streaming-scala" % Version.flink exclude ( - "org.slf4j", "slf4j-log4j12" - ), - "org.apache.flink" % "flink-connector-kafka_2.12" % Version.flink exclude ( - "org.slf4j", "slf4j-log4j12" - ), - "org.apache.flink" % "flink-jdbc_2.12" % Version.flink exclude ( - "org.slf4j", "slf4j-log4j12" - ), - "org.apache.flink" % "flink-metrics-dropwizard" % Version.flink exclude ( - "org.slf4j", "slf4j-log4j12" - ), - "org.apache.flink" %% "flink-metrics-prometheus" % Version.flink exclude ( - "org.slf4j", "slf4j-log4j12" - ), - "org.apache.flink" % "flink-avro" % Version.flink exclude ( - "org.slf4j", "slf4j-log4j12" - ), - "org.apache.flink" %% "flink-statebackend-rocksdb" % Version.flink exclude ( - "org.slf4j", "slf4j-log4j12" - ) - ) - val akka: Seq[ModuleID] = Seq( "com.typesafe.akka" %% "akka-slf4j" % Version.akka, "com.typesafe.akka" %% "akka-stream" % Version.akka, diff --git a/spark/src/main/scala/ru/itclover/tsp/spark/PatternsSearchJob.scala b/spark/src/main/scala/ru/itclover/tsp/spark/PatternsSearchJob.scala index 189fa6eb0..22faff4f3 100644 --- a/spark/src/main/scala/ru/itclover/tsp/spark/PatternsSearchJob.scala +++ b/spark/src/main/scala/ru/itclover/tsp/spark/PatternsSearchJob.scala @@ -17,7 +17,7 @@ import ru.itclover.tsp.core.optimizations.Optimizer import ru.itclover.tsp.core.{Incident, RawPattern, _} import ru.itclover.tsp.dsl.{ASTPatternGenerator, AnyState, PatternMetadata} import ru.itclover.tsp.spark.utils._ -import ru.itclover.tsp.spark.io.{InputConf, JDBCInputConf, JDBCOutputConf, KafkaInputConf, KafkaOutputConf, NewRowSchema, OutputConf} +import ru.itclover.tsp.spark.io.{InputConf, JDBCInputConf, JDBCOutputConf, KafkaInputConf, KafkaOutputConf, RowSchema, OutputConf} import ru.itclover.tsp.spark.transformers.SparseRowsDataAccumulator import ru.itclover.tsp.spark.utils.ErrorsADT.{ConfigErr, InvalidPatternsCode} import ru.itclover.tsp.spark.utils.DataWriterWrapperImplicits._ @@ -107,16 +107,11 @@ case class PatternsSearchJob[In: ClassTag: TypeTag, InKey, InItem]( val mappers: Seq[PatternProcessor[In, Optimizer.S[Segment], Incident]] = patterns.map { case ((pattern, meta), rawP) => - val allForwardFields = forwardedFields ++ rawP.forwardedFields - .getOrElse(Seq()) - .map(id => (id, source.fieldToEKey(id))) val toIncidents = ToIncidentsMapper( rawP.id, - allForwardFields.map { case (id, k) => id.toString.tail -> k }, source.fieldToEKey(source.conf.unitIdField.get), rawP.subunit.getOrElse(0), - rawP.payload.getOrElse(Map()).toSeq, if (meta.sumWindowsMs > 0L) meta.sumWindowsMs else source.conf.defaultEventsGapMs.getOrElse(2000L), source.conf.partitionFields.map(source.fieldToEKey) ) @@ -175,18 +170,6 @@ case class PatternsSearchJob[In: ClassTag: TypeTag, InKey, InItem]( } } -// def applyTransformation(stream: Dataset[In]): Dataset[In] = source.conf.dataTransformation match { -// case Some(_) => -// import source.{extractor, timeExtractor} -// Dataset -// .keyBy(source.partitioner) -// .process( -// SparseRowsDataAccumulator[In, InKey, InItem, In](source.asInstanceOf[StreamSource[In, InKey, InItem]], fields) -// ) -// .setParallelism(1) // SparseRowsDataAccumulator cannot work in parallel -// case _ => stream -// } - def queryListener: StreamingQueryListener = new StreamingQueryListener { override def onQueryStarted(event: StreamingQueryListener.QueryStartedEvent): Unit = { //println(s"${event.id} started") @@ -194,7 +177,7 @@ case class PatternsSearchJob[In: ClassTag: TypeTag, InKey, InItem]( override def onQueryProgress(event: StreamingQueryListener.QueryProgressEvent): Unit = { val rows = event.progress.sources.map(_.numInputRows).sum - println(s"$rows rows processed") + //println(s"$rows rows processed") } override def onQueryTerminated(event: StreamingQueryListener.QueryTerminatedEvent): Unit = { @@ -409,7 +392,7 @@ object PatternsSearchJob { } } - def rowSchemaToSchema(rowSchema: NewRowSchema): StructType = StructType( + def rowSchemaToSchema(rowSchema: RowSchema): StructType = StructType( rowSchema.fieldDatatypes.zip(rowSchema.fieldsNames).map { case (t, n) => new StructField(n.name, t) } ) } diff --git a/spark/src/main/scala/ru/itclover/tsp/spark/io/OutputConf.scala b/spark/src/main/scala/ru/itclover/tsp/spark/io/OutputConf.scala index 6e7078547..9d33d31c9 100644 --- a/spark/src/main/scala/ru/itclover/tsp/spark/io/OutputConf.scala +++ b/spark/src/main/scala/ru/itclover/tsp/spark/io/OutputConf.scala @@ -7,7 +7,7 @@ trait OutputConf[Event] { def parallelism: Option[Int] - def rowSchema: NewRowSchema + def rowSchema: RowSchema } /** @@ -21,24 +21,24 @@ trait OutputConf[Event] { * @param parallelism num of parallel task to write data */ case class JDBCOutputConf( - tableName: String, - rowSchema: NewRowSchema, - jdbcUrl: String, - driverName: String, - password: Option[String] = None, - batchInterval: Option[Int] = None, - userName: Option[String] = None, - parallelism: Option[Int] = Some(1) + tableName: String, + rowSchema: RowSchema, + jdbcUrl: String, + driverName: String, + password: Option[String] = None, + batchInterval: Option[Int] = None, + userName: Option[String] = None, + parallelism: Option[Int] = Some(1) ) extends OutputConf[Row] { override def forwardedFieldsIds = Seq.empty } case class KafkaOutputConf( - broker: String, - topic: String, - //serializer: Option[String] = Some("json"), - rowSchema: NewRowSchema, - parallelism: Option[Int] = Some(1) + broker: String, + topic: String, + //serializer: Option[String] = Some("json"), + rowSchema: RowSchema, + parallelism: Option[Int] = Some(1) ) extends OutputConf[Row] { override def forwardedFieldsIds = Seq.empty diff --git a/spark/src/main/scala/ru/itclover/tsp/spark/io/RowSchema.scala b/spark/src/main/scala/ru/itclover/tsp/spark/io/RowSchema.scala index c018736cf..42cc8dab6 100644 --- a/spark/src/main/scala/ru/itclover/tsp/spark/io/RowSchema.scala +++ b/spark/src/main/scala/ru/itclover/tsp/spark/io/RowSchema.scala @@ -5,7 +5,7 @@ import org.apache.spark.sql.types.{DataType, DataTypes} import scala.collection.mutable -case class NewRowSchema( +case class RowSchema( unitIdField: Symbol, fromTsField: Symbol, toTsField: Symbol, diff --git a/spark/src/main/scala/ru/itclover/tsp/spark/utils/IncidentAggregator.scala b/spark/src/main/scala/ru/itclover/tsp/spark/utils/IncidentAggregator.scala index f947ae9a7..8412876d1 100644 --- a/spark/src/main/scala/ru/itclover/tsp/spark/utils/IncidentAggregator.scala +++ b/spark/src/main/scala/ru/itclover/tsp/spark/utils/IncidentAggregator.scala @@ -38,10 +38,8 @@ class IncidentAggregator extends UserDefinedAggregateFunction { buffer.update(1, res.patternId) buffer.update(2, res.maxWindowMs) buffer.update(3, res.segment) - buffer.update(4, res.forwardedFields) - buffer.update(5, res.patternUnit) - buffer.update(6, res.patternSubunit) - buffer.update(7, res.patternPayload) + buffer.update(4, res.patternUnit) + buffer.update(5, res.patternSubunit) } override def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = { @@ -57,10 +55,8 @@ class IncidentAggregator extends UserDefinedAggregateFunction { buffer1.update(1, res.patternId) buffer1.update(2, res.maxWindowMs) buffer1.update(3, res.segment) - buffer1.update(4, res.forwardedFields) - buffer1.update(5, res.patternUnit) - buffer1.update(6, res.patternSubunit) - buffer1.update(7, res.patternPayload) + buffer1.update(4, res.patternUnit) + buffer1.update(5, res.patternSubunit) } override def evaluate(buffer: Row): Any = Incident( @@ -68,9 +64,7 @@ class IncidentAggregator extends UserDefinedAggregateFunction { buffer.getInt(1), buffer.getLong(2), Segment(Time(buffer.getAs[Row](3).getAs[Row](0).getLong(0)), Time(buffer.getAs[Row](3).getAs[Row](1).getLong(0))), - buffer.getAs[Seq[(String, String)]](4), + buffer.getInt(4), buffer.getInt(5), - buffer.getInt(6), - buffer.getAs[Seq[(String, String)]](7) ) } diff --git a/spark/src/main/scala/ru/itclover/tsp/spark/utils/PatternsToRowMapper.scala b/spark/src/main/scala/ru/itclover/tsp/spark/utils/PatternsToRowMapper.scala index 77f4b0877..dcc3f25e8 100644 --- a/spark/src/main/scala/ru/itclover/tsp/spark/utils/PatternsToRowMapper.scala +++ b/spark/src/main/scala/ru/itclover/tsp/spark/utils/PatternsToRowMapper.scala @@ -6,14 +6,14 @@ import java.time.{Instant, LocalDateTime, ZoneId, ZonedDateTime} import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.expressions.GenericRow import ru.itclover.tsp.core.Incident -import ru.itclover.tsp.spark.io.NewRowSchema +import ru.itclover.tsp.spark.io.RowSchema import scala.util.Try /** * Packer of found incident into [[org.apache.spark.sql.Row]] */ -case class PatternsToRowMapper[Event, EKey](sourceId: Int, schema: NewRowSchema) { +case class PatternsToRowMapper[Event, EKey](sourceId: Int, schema: RowSchema) { def map(incident: Incident): Row = { val values = new Array[Any](schema.fieldsCount) diff --git a/spark/src/main/scala/ru/itclover/tsp/spark/utils/ToIncidentsMapper.scala b/spark/src/main/scala/ru/itclover/tsp/spark/utils/ToIncidentsMapper.scala index c797a7c33..53531562d 100644 --- a/spark/src/main/scala/ru/itclover/tsp/spark/utils/ToIncidentsMapper.scala +++ b/spark/src/main/scala/ru/itclover/tsp/spark/utils/ToIncidentsMapper.scala @@ -7,20 +7,15 @@ import scala.util.Try final case class ToIncidentsMapper[E, EKey, EItem]( patternId: Int, - forwardedFields: Seq[(String, EKey)], unitIdField: EKey, subunit: Int, - payload: Seq[(String, String)], sessionWindowMs: Long, partitionFields: Seq[EKey] )(implicit extractor: Extractor[E, EKey, EItem], decoder: Decoder[EItem, Any]) { def apply(event: E): Segment => Incident = { val incidentId = s"P#$patternId;" + partitionFields.map(f => f -> extractor[Any](event, f)).mkString - val extractedFields = forwardedFields.map { - case (name, k) => name -> Try(extractor[Any](event, k).toString).getOrElse("null") - } val unit = Try(extractor[Any](event, unitIdField).toString.toInt).getOrElse(Int.MinValue) - segment => Incident(incidentId, patternId, sessionWindowMs, segment, extractedFields, unit, subunit, payload) + segment => Incident(incidentId, patternId, sessionWindowMs, segment, unit, subunit) } }