Set up CI with Azure Pipelines

azure-pipelines.yml

@@ -0,0 +1,26 @@
+# Adapted from pandas version, which is in turn 
+# adapted from https://github.com/numba/numba/blob/master/azure-pipelines.yml
+jobs:
+# Mac and Linux could potentially use the same template
+# except it isn't clear how to use a different build matrix
+# for each, so for now they are separate
+- template: ci/azure/macos.yml
+  parameters:
+    name: macOS
+    vmImage: xcode9-macos10.13
+# - template: ci/azure/linux.yml
+#   parameters:
+#     name: Linux
+#     vmImage: ubuntu-16.04
+
+# Windows Python 2.7 needs VC 9.0 installed, and not sure
+# how to make that a conditional task, so for now these are
+# separate templates as well
+- template: ci/azure/windows.yml
+  parameters:
+    name: Windows
+    vmImage: vs2017-win2016
+- template: ci/azure/windows-py27.yml
+  parameters:
+    name: WindowsPy27
+    vmImage: vs2017-win2016

ci/azure-macos-35.yml

@@ -0,0 +1,21 @@
+name: sm2
+channels:
+  - defaults
+dependencies:
+  - python=3.5*
+  - cython>=0.28.2
+  - bottleneck
+  - matplotlib
+  - nomkl
+  - numexpr
+  - numpy=1.13
+  - scipy
+  #
+  - xarray
+  # universal
+  - pandas>=0.23.3
+  - pytest
+  - pytest-xdist
+  - pip:
+    - python-dateutil==2.5.3
+    - hypothesis>=3.58.0

ci/azure-windows-27.yml

@@ -0,0 +1,17 @@
+name: sm2
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  - python=2.7.*
+  - bottleneck
+  - matplotlib
+  - numexpr
+  - numpy=1.12*
+  - scipy
+  # universal
+  - cython>=0.28.2
+  - pandas>=0.23.3
+  - pytest
+  - pytest-xdist
+  - hypothesis>=3.58.0

ci/azure-windows-36.yml

@@ -0,0 +1,21 @@
+name: sm2
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  - python=3.6.*
+  - blosc
+  - bottleneck
+  - matplotlib
+  - numexpr
+  - numpy=1.14*
+  - scipy
+  #
+  - pyarrow
+  - thrift=0.10*
+  # universal
+  - pandas>=0.23.3
+  - cython>=0.28.2
+  - pytest
+  - pytest-xdist
+  - hypothesis>=3.58.0

ci/azure/macos.yml

@@ -0,0 +1,47 @@
+# adapted from pandas' version
+
+parameters:
+  name: ''
+  vmImage: ''
+
+jobs:
+- job: ${{ parameters.name }}
+  pool:
+    vmImage: ${{ parameters.vmImage }}
+  strategy:
+    maxParallel: 11
+    matrix:
+      py35_np_113:
+        ENV_FILE: ci/azure-macos-35.yml
+        CONDA_PY: "35"
+        CONDA_ENV: sm2
+        TEST_ARGS: "--skip-slow"
+
+  steps:
+    - script: |
+        if [ "$(uname)" == "Linux" ]; then sudo apt-get install -y libc6-dev-i386; fi
+        echo "Installing Miniconda"
+        ci/incremental/install_miniconda.sh
+        export PATH=$HOME/miniconda3/bin:$PATH
+        echo "Setting up Conda environment"
+        ci/incremental/setup_conda_environment.sh
+      displayName: 'Before Install'
+    - script: |
+        export PATH=$HOME/miniconda3/bin:$PATH
+        ci/incremental/build.sh
+      displayName: 'Build'
+    - script: |
+        export PATH=$HOME/miniconda3/bin:$PATH
+        ci/script_single.sh
+        echo $?
+        ci/script_multi.sh
+        echo $?
+        echo "[Test done]"
+      displayName: 'Test'
+    - script: |
+        export PATH=$HOME/miniconda3/bin:$PATH
+        source activate sm2 && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
+    - task: PublishTestResults@2
+      inputs:
+        #testResultsFiles: '/tmp/*.xml'
+        testRunTitle: 'MacOS-35'

ci/azure/windows-py27.yml

@@ -0,0 +1,47 @@
+# Adapted from pandas' version
+
+parameters:
+  name: ''
+  vmImage: ''
+
+jobs:
+- job: ${{ parameters.name }}
+  pool:
+    vmImage: ${{ parameters.vmImage }}
+  strategy:
+    maxParallel: 11
+    matrix:
+      py36_np14:
+        ENV_FILE: ci/azure-windows-27.yml
+        CONDA_PY: "27"
+        CONDA_ENV: sm2
+
+  steps:
+    - task: CondaEnvironment@1
+      inputs:
+        updateConda: no
+        packageSpecs: ''
+
+    # Need to install VC 9.0 only for Python 2.7
+    # Once we understand how to do tasks conditional on build matrix variables
+    # we could merge this into azure-windows.yml
+    - powershell: |
+        $wc = New-Object net.webclient
+        $wc.Downloadfile("https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi", "VCForPython27.msi")
+        Start-Process "VCForPython27.msi" /qn -Wait
+      displayName: 'Install VC 9.0'
+
+    - script: |
+        ci\\incremental\\setup_conda_environment.cmd
+      displayName: 'Before Install'
+    - script: |
+        ci\\incremental\\build.cmd
+      displayName: 'Build'
+    - script: |
+        call activate %CONDA_ENV%
+        pytest --junitxml=test-data.xml --skip-slow sm2 -n 2 -r sxX --strict %*
+      displayName: 'Test'
+    - task: PublishTestResults@2
+      inputs:
+        testResultsFiles: 'test-data.xml'
+        testRunTitle: 'Windows 27'

ci/azure/windows.yml

@@ -0,0 +1,38 @@
+# adapted from pandas' version
+
+parameters:
+  name: ''
+  vmImage: ''
+
+jobs:
+- job: ${{ parameters.name }}
+  pool:
+    vmImage: ${{ parameters.vmImage }}
+  strategy:
+    maxParallel: 11
+    matrix:
+      py36_np14:
+        ENV_FILE: ci/azure-windows-36.yml
+        CONDA_PY: "36"
+        CONDA_ENV: sm2
+
+  steps:
+    - task: CondaEnvironment@1
+      inputs:
+        updateConda: no
+        packageSpecs: ''
+
+    - script: |
+        ci\\incremental\\setup_conda_environment.cmd
+      displayName: 'Before Install'
+    - script: |
+        ci\\incremental\\build.cmd
+      displayName: 'Build'
+    - script: |
+        call activate %CONDA_ENV%
+        pytest --junitxml=test-data.xml --skip-slow sm2 -n 2 -r sxX --strict %*
+      displayName: 'Test'
+    - task: PublishTestResults@2
+      inputs:
+        testResultsFiles: 'test-data.xml'
+        testRunTitle: 'Windows 36'

ci/incremental/build.cmd

@@ -0,0 +1,13 @@
+@rem copied from pandas
+
+@rem https://github.com/numba/numba/blob/master/buildscripts/incremental/build.cmd
+call activate %CONDA_ENV%
+
+@rem Build numba extensions without silencing compile errors
+python setup.py build_ext -q --inplace
+
+@rem Install sm2 locally
+python -m pip install --upgrade pip
+python -m pip install -e .
+
+if %errorlevel% neq 0 exit /b %errorlevel%

ci/incremental/build.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+# copied from pandas
+
+source activate $CONDA_ENV
+
+# Make sure any error below is reported as such
+set -v -e
+
+echo "[building extensions]"
+python setup.py build_ext -q --inplace
+python -m pip install --upgrade pip
+python -m pip install -e .
+
+echo
+echo "[show environment]"
+conda list
+
+echo
+echo "[done]"
+exit 0

ci/incremental/install_miniconda.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+# Copied from pandas
+
+set -v -e
+
+# Install Miniconda
+unamestr=`uname`
+if [[ "$unamestr" == 'Linux' ]]; then
+    if [[ "$BITS32" == "yes" ]]; then
+        wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86.sh -O miniconda.sh
+    else
+        wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
+    fi
+elif [[ "$unamestr" == 'Darwin' ]]; then
+    wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh
+else
+  echo Error
+fi
+chmod +x miniconda.sh
+./miniconda.sh -b

ci/incremental/setup_conda_environment.cmd

@@ -0,0 +1,22 @@
+@rem based off of pandas version
+@rem https://github.com/numba/numba/blob/master/buildscripts/incremental/setup_conda_environment.cmd
+@rem The cmd /C hack circumvents a regression where conda installs a conda.bat
+@rem script in non-root environments.
+set CONDA_INSTALL=cmd /C conda install -q -y
+set PIP_INSTALL=pip install -q
+
+@echo on
+
+@rem Deactivate any environment
+call deactivate
+@rem Display root environment (for debugging)
+conda list
+@rem Clean up any left-over from a previous build
+conda remove --all -q -y -n %CONDA_ENV%
+@rem Scipy, CFFI, jinja2 and IPython are optional dependencies, but exercised in the test suite
+conda env create -n %CONDA_ENV% --file=ci\azure-windows-%CONDA_PY%.yml
+
+call activate %CONDA_ENV%
+conda list
+
+if %errorlevel% neq 0 exit /b %errorlevel%

ci/incremental/setup_conda_environment.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+# based on pandas' version
+
+set -v -e
+
+CONDA_INSTALL="conda install -q -y"
+PIP_INSTALL="pip install -q"
+
+# Deactivate any environment
+source deactivate
+# Display root environment (for debugging)
+conda list
+# Clean up any left-over from a previous build
+# (note workaround for https://github.com/conda/conda/issues/2679:
+#  `conda env remove` issue)
+conda remove --all -q -y -n $CONDA_ENV
+
+echo
+echo "[create env]"
+time conda env create -q -n "${CONDA_ENV}" --file="${ENV_FILE}" || exit 1
+
+# Activate first
+set +v
+source activate $CONDA_ENV
+set -v
+
+# remove any installed pandas package
+# w/o removing anything else
+#echo
+#echo "[removing installed pandas]"
+#conda remove pandas -y --force
+#pip uninstall -y pandas
+
+#echo
+#echo "[no installed pandas]"
+#conda list pandas
+
+# # Install the compiler toolchain
+# if [[ $(uname) == Linux ]]; then
+#     if [[ "$CONDA_SUBDIR" == "linux-32" || "$BITS32" == "yes" ]] ; then
+#         $CONDA_INSTALL gcc_linux-32 gxx_linux-32
+#     else
+#         $CONDA_INSTALL gcc_linux-64 gxx_linux-64
+#     fi
+# elif  [[ $(uname) == Darwin ]]; then
+#     $CONDA_INSTALL clang_osx-64 clangxx_osx-64
+#     # Install llvm-openmp and intel-openmp on OSX too
+#     $CONDA_INSTALL llvm-openmp intel-openmp
+# fi

ci/script_multi.sh

@@ -0,0 +1,43 @@
+#!/bin/bash -e
+
+echo "[script multi]"
+
+source activate sm2
+
+if [ -n "$LOCALE_OVERRIDE" ]; then
+    export LC_ALL="$LOCALE_OVERRIDE";
+    echo "Setting LC_ALL to $LOCALE_OVERRIDE"
+
+    pycmd='import pandas; print("pandas detected console encoding: %s" % pandas.get_option("display.encoding"))'
+    python -c "$pycmd"
+fi
+
+
+# Workaround for pytest-xdist flaky collection order
+# https://github.com/pytest-dev/pytest/issues/920
+# https://github.com/pytest-dev/pytest/issues/1075
+export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 4294967295))')
+echo PYTHONHASHSEED=$PYTHONHASHSEED
+
+if [ "$DOC" ]; then
+    echo "We are not running pytest as this is a doc-build"
+
+elif [ "$COVERAGE" ]; then
+    echo pytest -s -n 2 -m "not single" --cov=sm2 --cov-report xml:/tmp/cov-multiple.xml --junitxml=/tmp/multiple.xml --strict $TEST_ARGS sm2
+    pytest -s -n 2 -m "not single" --cov=sm2 --cov-report xml:/tmp/cov-multiple.xml --junitxml=/tmp/multiple.xml --strict $TEST_ARGS sm2
+
+elif [ "$SLOW" ]; then
+    TEST_ARGS="--only-slow"
+    echo pytest -r xX -m "not single and slow" -v --junitxml=/tmp/multiple.xml --strict $TEST_ARGS sm2
+    pytest -r xX -m "not single and slow" -v --junitxml=/tmp/multiple.xml --strict $TEST_ARGS sm2
+
+else
+    echo pytest -n 2 -r xX -m "not single" --junitxml=/tmp/multiple.xml --strict $TEST_ARGS sm2
+    pytest -n 2 -r xX -m "not single" --junitxml=/tmp/multiple.xml --strict $TEST_ARGS sm2 # TODO: doctest
+
+fi
+
+exit 1
+RET="$?"
+
+exit "$RET"