chronon/.circleci/config.yml at 2ffd32de09019d357af391f4cd7639e2e1efdc77 · abbywh/chronon · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
version: 2.1

base_defaults: &base_defaults
    resource_class: xlarge
    working_directory: /chronon

executors:
    docker_baseimg_executor:
        resource_class: xlarge
        working_directory: /chronon
        docker:
            - image: houpy0829/chronon-ci:base--f87f50dc520f7a73894ae024eb78bd305d5b08e2
    docker_baseimg_executor_xxlarge:
      resource_class: 2xlarge
      working_directory: /chronon
      docker:
        - image: houpy0829/chronon-ci:base--f87f50dc520f7a73894ae024eb78bd305d5b08e2

jobs:
    "Pull Docker Image":
        <<: *base_defaults
        docker:
            - image: docker:17.05.0-ce-git
        steps:
            - setup_remote_docker:
                docker_layer_caching: true
            - checkout
            - run:
                name: Pull existing docker image
                command: |
                    set +o pipefail
                    docker pull houpy0829/chronon-ci:base--f87f50dc520f7a73894ae024eb78bd305d5b08e2 || true

    "Scala 12 -- Spark 3 Tests":
        executor: docker_baseimg_executor_xxlarge
        steps:
            - checkout
            - run:
                  name: Run Spark 3.1.1 tests
                  shell: /bin/bash -leuxo pipefail
                  command: |
                      conda activate chronon_py
                      # Increase if we see OOM.
                      export SBT_OPTS="-XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=4G -Xmx4G -Xms2G"
                      sbt "++ 2.12.12 test"
            - store_test_results:
                  path: /chronon/spark/target/test-reports
            - store_test_results:
                  path: /chronon/aggregator/target/test-reports
            - run:
                  name: Compress spark-warehouse
                  command: |
                    cd /tmp/ && tar -czvf spark-warehouse.tar.gz chronon/spark-warehouse
                  when: on_fail
            - store_artifacts:
                  path: /tmp/spark-warehouse.tar.gz
                  destination: spark_warehouse.tar.gz
                  when: on_fail

    "Scala 13 -- Tests":
        executor: docker_baseimg_executor
        steps:
            - checkout
            - run:
                  name: Run Scala 13 tests
                  shell: /bin/bash -leuxo pipefail
                  command: |
                      conda activate chronon_py
                      # Increase if we see OOM.
                      export SBT_OPTS="-XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=8G -Xmx8G -Xms4G"
                      sbt "++ 2.13.6 test"
            - store_test_results:
                  path: /chronon/spark/target/test-reports
            - store_test_results:
                  path: /chronon/aggregator/target/test-reports
            - run:
                  name: Compress spark-warehouse
                  command: |
                    cd /tmp/ && tar -czvf spark-warehouse.tar.gz chronon/spark-warehouse
                  when: on_fail
            - store_artifacts:
                  path: /tmp/spark-warehouse.tar.gz
                  destination: spark_warehouse.tar.gz
                  when: on_fail

    "Scala 11 -- Compile":
      executor: docker_baseimg_executor
      steps:
        - checkout
        - run:
            name: Compile Scala 11
            shell: /bin/bash -leuxo pipefail
            command: |
              conda activate chronon_py
              # Increase if we see OOM.
              export SBT_OPTS="-XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=4G -Xmx4G -Xms2G"
              sbt "++ 2.11.12 compile"

    "Chronon Python Lint":
        executor: docker_baseimg_executor
        steps:
            - checkout
            - run:
                  name: Run Chronon Python lint
                  shell: /bin/bash -leuxo pipefail
                  command: |
                      conda activate chronon_py
                      cd /chronon/api/py/ai/chronon
                      pip install importlib-metadata==4.11.4 #Install importlib-metadata < 5
                      flake8 --extend-ignore=W605,Q000,F631,E203

    "Chronon Python Tests":
        executor: docker_baseimg_executor
        steps:
            - checkout
            - run:
                  name: Run Chronon Python tests
                  shell: /bin/bash -leuxo pipefail
                  command: |
                    conda activate chronon_py
                    pushd /chronon/api/
                    thrift --gen py -out /chronon/api/py/ai/chronon\
                       /chronon/api/thrift/api.thrift                 # Generate thrift files
                    cd /chronon/                                      # Go to Python module
                    pip install -r api/py/requirements/dev.txt        # Install latest requirements
                    tox                                               # Run tests
                    popd
            - store_artifacts:
                  path: /chronon/api/py/htmlcov

    "Scalafmt Check":
        executor: docker_baseimg_executor
        steps:
            - checkout
            - run:
                  name: Run ScalafmtCheck
                  shell: /bin/bash -leuxo pipefail
                  command: |
                      conda activate chronon_py
                      sbt +scalafmtCheck
    # run these separately as we need a isolated JVM to not have Spark session settings interfere with other runs
    # long term goal is to refactor the current testing spark session builder and avoid adding new single test to CI
    "Scala 13 -- Iceberg Format Tests":
      executor: docker_baseimg_executor
      steps:
        - checkout
        - run:
            name: Run Scala 13 tests for Iceberg format
            environment:
              format_test: iceberg
            shell: /bin/bash -leuxo pipefail
            command: |
              conda activate chronon_py
              # Increase if we see OOM.
              export SBT_OPTS="-XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=4G -Xmx4G -Xms2G"
              sbt ';project spark_embedded; ++ 2.13.6; testOnly ai.chronon.spark.test.TableUtilsFormatTest'
        - store_test_results:
            path: /chronon/spark/target/test-reports
        - store_test_results:
            path: /chronon/aggregator/target/test-reports
        - run:
            name: Compress spark-warehouse
            command: |
              cd /tmp/ && tar -czvf spark-warehouse.tar.gz chronon/spark-warehouse
            when: on_fail
        - store_artifacts:
            path: /tmp/spark-warehouse.tar.gz
            destination: spark_warehouse.tar.gz
            when: on_fail
    "Scala 13 -- Iceberg Table Utils Tests":
      executor: docker_baseimg_executor
      steps:
        - checkout
        - run:
            name: Run Scala 13 tests for Iceberg Table Utils
            environment:
              format_test: iceberg
            shell: /bin/bash -leuxo pipefail
            command: |
              conda activate chronon_py
              # Increase if we see OOM.
              export SBT_OPTS="-XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=4G -Xmx4G -Xms2G"
              sbt ';project spark_embedded; ++ 2.13.6; testOnly ai.chronon.spark.test.TableUtilsTest'
        - store_test_results:
            path: /chronon/spark/target/test-reports
        - store_test_results:
            path: /chronon/aggregator/target/test-reports
        - run:
            name: Compress spark-warehouse
            command: |
              cd /tmp/ && tar -czvf spark-warehouse.tar.gz chronon/spark-warehouse
            when: on_fail
        - store_artifacts:
            path: /tmp/spark-warehouse.tar.gz
            destination: spark_warehouse.tar.gz
            when: on_fail

workflows:
    build_test_deploy:
        jobs:
            - "Pull Docker Image"
            - "Scala 11 -- Compile":
                requires:
                  - "Pull Docker Image"
            - "Scala 12 -- Spark 3 Tests":
                  requires:
                      - "Pull Docker Image"
            - "Scala 13 -- Tests":
                  requires:
                      - "Pull Docker Image"
            - "Scalafmt Check":
                  requires:
                      - "Pull Docker Image"
            - "Chronon Python Tests":
                  requires:
                      - "Pull Docker Image"
            - "Chronon Python Lint":
                  requires:
                      - "Pull Docker Image"
            - "Scala 13 -- Iceberg Format Tests":
                  requires:
                      - "Pull Docker Image"
            - "Scala 13 -- Iceberg Table Utils Tests":
                  requires:
                      - "Pull Docker Image"