Refactor itests to allow adding multiple transport sinks (#28)

* Refactor itests to be allow adding multiple transport sinks

* split out tests into base & transport

* organize all base tests together

* yaml lint

* unset context after test

.circleci/config.yml

@@ -17,7 +17,7 @@ step_library:
     command: |
       git submodule sync
       git submodule update --init || (rm -fr .git/config .git/modules && git submodule deinit -f . && git submodule update --init)
-      make itests-circleci
+      make itests
 
 - &step_build_artifact
   run:

.gitignore

@@ -9,6 +9,7 @@ _vendor-*/
 target/
 
 # exclude itest outputs
-itests/tests/*/output/*
+itests/tests/*/*/output/*
 itests/test_output/*
+itests/tests/*/*/perf_output/*
 itests/perf_output/*

Makefile

@@ -28,15 +28,9 @@ test: vendor generate vet
 	@echo "Executing tests ..."
 	go test -race -v ${GO_TEST_EXTRAS} ./...
 
-itests-circleci:
-	@echo "Running integration tests"
-	TEST_NAME=test_basic docker-compose -f itests/docker-compose.yml build
-	cd ./itests && ./circleci_split_itests.sh
-
 itests:
 	@echo "Running integration tests"
-	TEST_NAME=test_basic docker-compose -f itests/docker-compose.yml build
-	cd ./itests && ./integration_tests.bats -r tests
+	cd ./itests && ./itests_runner.sh
 
 clean:
 	@echo "Removing vendor deps"

itests/common.bash

@@ -55,7 +55,7 @@ _startup() {
   sleep 5
 
   _check_container postgres
-  _check_container kinesis
+  _check_container localstack
   _check_container bifrost
 
   log "Containers are running!"
@@ -221,7 +221,7 @@ _profile() {
 
 _wait() {
   log "Waiting for test to finish"
-  grep -q 'Records read' <(docker logs --follow kinesis-poller 2>&1)
+  grep -q 'Records read' <(docker logs --follow data-poller 2>&1)
   pkill -f "docker logs.*" || true
 }
 

itests/containers/kinesis-poller/Dockerfile → itests/containers/data-poller/Dockerfile

@@ -7,4 +7,4 @@ RUN \
 
 ADD app /app
 VOLUME /output
-CMD ["python", "/app/poller.py"]
+CMD ["sh", "/app/entrypoint.sh"]

itests/containers/data-poller/app/entrypoint.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+# Please set this in your docker run config to specify which poller is used.
+TRANSPORT_SINK=${TRANSPORT_SINK:-}
+
+python /app/poller-$TRANSPORT_SINK.py

itests/containers/kinesis-poller/app/poller.py → itests/containers/data-poller/app/poller-kinesis.py

@@ -15,7 +15,7 @@
 # Variables
 OUT_FILE = os.getenv('OUT_FILE', '/output/test')
 STREAM_NAME = os.getenv('STREAM_NAME', 'itests')
-ENDPOINT_URL = os.getenv('ENDPOINT_URL', 'http://kinesis:4568')
+ENDPOINT_URL = os.getenv('ENDPOINT_URL', 'http://localstack:4568')
 WAIT_TIME = int(os.getenv('WAIT_TIME', '90'))
 EXPECTED_COUNT = int(os.getenv('EXPECTED_COUNT', '1'))
 AWS_REGION = os.getenv('AWS_REGION', 'us-east-1')

itests/containers/defaults.env

@@ -9,4 +9,3 @@ BIFROST_KINESIS_STREAM=itests
 AWS_ACCESS_KEY_ID=DUMMYACCESSKEYID
 AWS_SECRET_ACCESS_KEY=DUMMYSECRETACCESSKEY
 AWS_REGION=us-east-1
-ENDPOINT=http://kinesis:4568

itests/contexts/kinesis.env

@@ -0,0 +1,3 @@
+TRANSPORT_SINK=kinesis
+LOCALSTACK_PORT=4568
+ENDPOINT=http://localstack:4568

itests/docker-compose.yml

@@ -10,16 +10,16 @@ services:
     networks:
       - net
 
-  kinesis:
-    container_name: kinesis
+  localstack:
+    container_name: localstack
     image: localstack/localstack:0.8.7
     env_file:
-    - "./containers/defaults.env"
-    - "./tests/${TEST_NAME}/envfile.env"
+      - "./containers/defaults.env"
+      - "./tests/${TEST_NAME}/envfile.env"
     environment:
-      - SERVICES=kinesis
+      - SERVICES=${TRANSPORT_SINK}
     ports:
-     - 4568:4568
+      - ${LOCALSTACK_PORT}:${LOCALSTACK_PORT}
     restart: always
     networks:
       - net
@@ -28,29 +28,31 @@ services:
     container_name: bifrost
     image: pg-bifrost:latest
     env_file:
-    - "./containers/defaults.env"
-    - "./tests/${TEST_NAME}/envfile.env"
+      - "./containers/defaults.env"
+      - "./tests/${TEST_NAME}/envfile.env"
+      - "./contexts/${TRANSPORT_SINK}.env"
     volumes:
       - "./tests/${TEST_NAME}/perf_output:/perf"
     networks:
       net:
         aliases:
-          - kinesis
+          - localstack
           - postgres
-    command: /pg-bifrost replicate kinesis
+    command: /pg-bifrost replicate ${TRANSPORT_SINK}
 
-  kinesis-poller:
-    container_name: kinesis-poller
-    build: containers/kinesis-poller
+  data-poller:
+    container_name: data-poller
+    build: containers/data-poller
     env_file:
-    - "./containers/defaults.env"
-    - "./tests/${TEST_NAME}/envfile.env"
+      - "./containers/defaults.env"
+      - "./tests/${TEST_NAME}/envfile.env"
+      - "./contexts/${TRANSPORT_SINK}.env"
     volumes:
-    - "./tests/${TEST_NAME}/output:/output"
+      - "./tests/${TEST_NAME}/output:/output"
     networks:
       net:
         aliases:
-          - kinesis
+          - localstack
 
 networks:
-  net:
+  net:

itests/itests_runner.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Copyright 2019 Nextdoor.com, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+get_testfiles() {
+    _transport_sink=$1
+
+    if [ "$CI" == "true" ]; then
+        _base_testfiles=$(cd tests/base && circleci tests glob "*" | circleci tests split --split-by=timings | sed -e 's/^/base\//')
+        _specific_testfiles=$(cd tests/${_transport_sink} && circleci tests glob "*" | circleci tests split --split-by=timings | sed -e "s/^/${_transport_sink}\//")
+    else
+        _base_testfiles=$(cd tests/base && ls -d */ | sed 's#/##' | sed -e 's/^/base\//')
+        _specific_testfiles=$(cd tests/${_transport_sink} && ls -d */ | sed 's#/##' | sed -e "s/^/${_transport_sink}\//")
+    fi
+
+    echo "${_base_testfiles} ${_specific_testfiles}"
+}
+
+# Kinesis
+set -a ; . contexts/kinesis.env ; set +a
+
+TEST_NAME=base/test_basic docker-compose -f docker-compose.yml build
+TESTFILES=$(get_testfiles $TRANSPORT_SINK)
+
+echo "TESTFILES:"
+echo $TESTFILES | tr ' ' '\n'
+
+for TEST in $TESTFILES
+do
+   echo "running test $TEST"
+   ./integration_tests.bats -r tests -f "$TEST"
+done
+
+unset $(cat contexts/kinesis.env | awk -F= '{print $1}' | xargs)

itests/tests/base/test_basic/test.bats

@@ -0,0 +1,5 @@
+load ../../../common
+
+@test "base/test_basic" {
+  do_test
+}

itests/tests/base/test_blacklist/test.bats

@@ -0,0 +1,5 @@
+load ../../../common
+
+@test "base/test_blacklist" {
+  do_test
+}

itests/tests/base/test_large_sleep/test.bats

@@ -0,0 +1,5 @@
+load ../../../common
+
+@test "base/test_large_sleep" {
+  do_test
+}

itests/tests/base/test_large_txn/test.bats

@@ -0,0 +1,5 @@
+load ../../../common
+
+@test "base/test_large_txn" {
+  do_test
+}

itests/tests/base/test_small_queue/test.bats

@@ -0,0 +1,5 @@
+load ../../../common
+
+@test "base/test_small_queue" {
+  do_test
+}

itests/tests/base/test_staggered/test.bats

@@ -0,0 +1,5 @@
+load ../../../common
+
+@test "base/test_staggered" {
+  do_test
+}

itests/tests/base/test_timeout/test.bats

@@ -0,0 +1,5 @@
+load ../../../common
+
+@test "base/test_timeout" {
+  do_test
+}

itests/tests/base/test_txn_shard_partition/test.bats

@@ -0,0 +1,5 @@
+load ../../../common
+
+@test "base/test_txn_shard_partition" {
+  do_test
+}

itests/tests/base/test_whitelist/test.bats

@@ -0,0 +1,5 @@
+load ../../../common
+
+@test "base/test_whitelist" {
+  do_test
+}

itests/tests/kinesis/test_high_kinesis_errors/test.bats

@@ -0,0 +1,5 @@
+load ../../../common
+
+@test "kinesis/test_high_kinesis_errors" {
+  do_test
+}

itests/tests/kinesis/test_multi_worker_high_kinesis_errors/test.bats

@@ -0,0 +1,5 @@
+load ../../../common
+
+@test "kinesis/test_multi_worker_high_kinesis_errors" {
+  do_test
+}

itests/tests/kinesis/test_multi_worker_no_kinesis_errors/test.bats

@@ -0,0 +1,5 @@
+load ../../../common
+
+@test "kinesis/test_multi_worker_no_kinesis_errors" {
+  do_test
+}

itests/tests/kinesis/test_strict_order_shared/test.bats

@@ -0,0 +1,5 @@
+load ../../../common
+
+@test "kinesis/test_strict_order_shared" {
+  SORT=false do_test
+}

itests/tests/kinesis/test_table_partitioning/test.bats

@@ -0,0 +1,5 @@
+load ../../../common
+
+@test "kinesis/test_table_partitioning" {
+  do_test
+}