Add support for build from TileDB Core tarball (#627)

* Accomodate build from external tarball * Ensure LDFLAGS does not leak to cmake build, small edits * Also enable s3 when building from source tarball * Ensure default build works * Support download and use of dev.zip from GitHub * Add a new nightly yaml [ci skip] * Turn nightly on (and 'park' ci and windows for a moment) * Run tinytest::test_package after INSTALL step * Set back to default triggers now that nightly works [ci skip] * Add issue create action on failure to nightly job [ci skip] * Update NEWS and increase micro release version [ci skip]
TileDB-Inc · Dec 1, 2023 · ec5eb4f · ec5eb4f
1 parent 374269d
commit ec5eb4f
Show file tree

Hide file tree

Showing 8 changed files with 110 additions and 53 deletions.
diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml
@@ -0,0 +1,45 @@
+name: nightly
+
+on:
+  schedule:
+    - cron: "21 01 * * *"
+  workflow_dispatch:
+  # push:
+
+jobs:
+  dev:
+    runs-on: ubuntu-latest
+    container:
+      image: rocker/r2u:latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: SessionInfo
+        run: Rscript -e 'sessionInfo()'
+      - name: System Dependencies
+        # this particular package needs cmake and curl and git to build
+        run: apt update -qq && apt install --yes --no-install-recommends cmake curl git
+      - name: Package Dependencies
+        run: Rscript -e 'remotes::install_deps(".", dependencies=TRUE)'
+      - name: Build Package
+        run: R CMD build --no-build-vignettes --no-manual .
+      - name: Install Package
+        # running an install step gives better logging on standard out than R CMD check which tucks this away
+        run: R CMD INSTALL --configure-args="--with-download=https://github.com/TileDB-Inc/TileDB/archive/refs/heads/dev.zip" $(ls -1tr *.tar.gz | tail -1)
+      - name: Run Tests
+        # given that the package is installed and uses tinytest, we can easily run its tests
+        run: Rscript -e 'tinytest::test_package("tiledb")'
+
+  create_issue_on_fail:
+    permissions:
+      issues: write
+    runs-on: ubuntu-latest
+    needs: test
+    if: failure() || cancelled()
+    steps:
+      - uses: actions/checkout@v4
+      - name: Create Issue if Build Fails
+        uses: TileDB-Inc/github-actions/open-issue@main
+        with:
+          name: Nightly Dev Build
+          label: bug,nightly
+          assignee: eddelbuettel
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: tiledb
 Type: Package
-Version: 0.22.0
+Version: 0.22.0.1
 Title: Modern Database Engine for Multi-Modal Data via Sparse and Dense Multidimensional Arrays
 Authors@R: c(person("TileDB, Inc.", role = c("aut", "cph")),
  person("Dirk", "Eddelbuettel", email = "dirk@tiledb.com", role = "cre"))

diff --git a/NEWS.md b/NEWS.md
@@ -1,3 +1,10 @@
+# Ongoing development
+
+## Build and Test Systems
+
+* Builds from TileDB Core non-release tarballs are now supported via new configure option (#627)
+
+
 # tiledb 0.22.0
 
 * This release of the R package builds against [TileDB 2.18.0](https://github.com/TileDB-Inc/TileDB/releases/tag/2.18.0), [TileDB 2.18.1](https://github.com/TileDB-Inc/TileDB/releases/tag/2.18.1),  [TileDB 2.18.2](https://github.com/TileDB-Inc/TileDB/releases/tag/2.18.2) and has also been tested against earlier releases as well as the development version (#620,#621,#624)

diff --git a/configure b/configure
@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.71 for TileDB-R 0.21.1.
+# Generated by GNU Autoconf 2.71 for TileDB-R 0.22.0.
 #
 #
 # Copyright (C) 1992-1996, 1998-2017, 2020-2021 Free Software Foundation,
@@ -607,8 +607,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='TileDB-R'
 PACKAGE_TARNAME='tiledb-r'
-PACKAGE_VERSION='0.21.1'
-PACKAGE_STRING='TileDB-R 0.21.1'
+PACKAGE_VERSION='0.22.0'
+PACKAGE_STRING='TileDB-R 0.22.0'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''
 
@@ -1270,7 +1270,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures TileDB-R 0.21.1 to adapt to many kinds of systems.
+\`configure' configures TileDB-R 0.22.0 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1332,7 +1332,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of TileDB-R 0.21.1:";;
+     short | recursive ) echo "Configuration of TileDB-R 0.22.0:";;
    esac
   cat <<\_ACEOF
 
@@ -1425,7 +1425,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-TileDB-R configure 0.21.1
+TileDB-R configure 0.22.0
 generated by GNU Autoconf 2.71
 
 Copyright (C) 2021 Free Software Foundation, Inc.
@@ -1572,7 +1572,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by TileDB-R $as_me 0.21.1, which was
+It was created by TileDB-R $as_me 0.22.0, which was
 generated by GNU Autoconf 2.71.  Invocation command line was
 
   $ $0$ac_configure_args_raw
@@ -6026,16 +6026,18 @@ fi
 if test ${with_download+y}
 then :
   withval=$with_download; TILEDB_LIB_URL="${with_download}"
+	     enable_building="yes"
 fi
 
 
 ## Sanity checks
 if test ! -z "${with_tiledb}" -a x"${with_download}" = x"yes"; then
     as_fn_error $? "Conflicting choice of preinstalled TileDB and enabling download of TileDB library." "$LINENO" 5
 fi
-if test ! -z "${with_download}" -a x"${enable_building}" = x"yes"; then
-    as_fn_error $? "Conflicting choice of enabling download of TileDB and enabling building of TileDB." "$LINENO" 5
-fi
+## -- We allow this now as --with-download implies enable_building
+##if test ! -z "${with_download}" -a x"${enable_building}" = x"yes"; then
+##    AC_MSG_ERROR([Conflicting choice of enabling download of TileDB and enabling building of TileDB.])
+##fi
 TILEDB_LIB_URL="${with_download}"
 
 
@@ -6305,13 +6307,13 @@ fi
         { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: installing TileDB via building locally..." >&5
 printf "%s\n" "installing TileDB via building locally..." >&6; }
         if test x"${TILEDB_LIB_URL}" != x; then
-            { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: using ${TILEDB_LIB_URL} ..." >&5
-printf "%s\n" "using ${TILEDB_LIB_URL} ..." >&6; }
-            tools/buildTileDB.sh ${TILEBD_LIB_URL}
+            #AC_MSG_RESULT([using ${TILEDB_LIB_URL} ...])
+            ## Important: unset LDFLAGS to not leak -ltiledb from pkg-config
+            LDFLAGS="${origLDFLAGS}" tools/buildTileDB.sh ${TILEDB_LIB_URL}
         else
-            { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: using default url ..." >&5
-printf "%s\n" "using default url ..." >&6; }
-            tools/buildTileDB.sh default
+            #AC_MSG_RESULT([using default url ...])
+            ## Important: unset LDFLAGS to not leak -ltiledb from pkg-config
+            LDFLAGS="${origLDFLAGS}" tools/buildTileDB.sh default
         fi
 
     ## case two: download considered
@@ -6320,15 +6322,8 @@ printf "%s\n" "using default url ..." >&6; }
     ##   - otherwise error
     ##   - but also check we're on x86_64 before downloading
     else
-        ## download URL given by user
-        if test x"${TILEDB_LIB_URL}" != x; then
-            { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: downloading given TileDB library..." >&5
-printf "%s\n" "downloading given TileDB library..." >&6; }
-            ## we use the source download which takes this argument if one given
-            ## and downloads as the same file name
-            cd inst && ../tools/fetchTileDB.sh url ${TILEDB_LIB_URL} && cd ..
 
-        elif test x"${machine}" = x"x86_64" && test x"${uname}" = x"Linux" -o x"${uname}" = x"Darwin"; then
+        if test x"${machine}" = x"x86_64" && test x"${uname}" = x"Linux" -o x"${uname}" = x"Darwin"; then
             { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: downloading x86_64 TileDB library..." >&5
 printf "%s\n" "downloading x86_64 TileDB library..." >&6; }
 
@@ -6914,7 +6909,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by TileDB-R $as_me 0.21.1, which was
+This file was extended by TileDB-R $as_me 0.22.0, which was
 generated by GNU Autoconf 2.71.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -6969,7 +6964,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config='$ac_cs_config_escaped'
 ac_cs_version="\\
-TileDB-R config.status 0.21.1
+TileDB-R config.status 0.22.0
 configured by $0, generated by GNU Autoconf 2.71,
   with options \\"\$ac_cs_config\\"
 

diff --git a/configure.ac b/configure.ac
@@ -24,7 +24,7 @@
 
 ## require at least autoconf 2.69
 AC_PREREQ([2.69])
-AC_INIT([TileDB-R],[0.21.1])
+AC_INIT([TileDB-R],[0.22.0])
 
 ## -- Part 1: Setup -------------------------------------------------------------
 ##
@@ -140,16 +140,18 @@ fi
 ## (and sets by autconf standards the 'with_download' variable)
 AC_ARG_WITH([download],
             AS_HELP_STRING([--with-download=URL],[URL override for TileDB library to download]),
-            [TILEDB_LIB_URL="${with_download}"],
+            [TILEDB_LIB_URL="${with_download}"
+	     enable_building="yes"],
             [])
 
 ## Sanity checks
 if test ! -z "${with_tiledb}" -a x"${with_download}" = x"yes"; then
     AC_MSG_ERROR([Conflicting choice of preinstalled TileDB and enabling download of TileDB library.])
 fi
-if test ! -z "${with_download}" -a x"${enable_building}" = x"yes"; then
-    AC_MSG_ERROR([Conflicting choice of enabling download of TileDB and enabling building of TileDB.])
-fi
+## -- We allow this now as --with-download implies enable_building
+##if test ! -z "${with_download}" -a x"${enable_building}" = x"yes"; then
+##    AC_MSG_ERROR([Conflicting choice of enabling download of TileDB and enabling building of TileDB.])
+##fi
 AC_SUBST([TILEDB_LIB_URL], "${with_download}")
 
 ## Setup
@@ -227,11 +229,13 @@ if test x"${have_tiledb}" = x"no"; then
         fi
         AC_MSG_RESULT([installing TileDB via building locally...])
         if test x"${TILEDB_LIB_URL}" != x; then
-            AC_MSG_RESULT([using ${TILEDB_LIB_URL} ...])
-            tools/buildTileDB.sh ${TILEBD_LIB_URL}
+            #AC_MSG_RESULT([using ${TILEDB_LIB_URL} ...])
+            ## Important: unset LDFLAGS to not leak -ltiledb from pkg-config
+            LDFLAGS="${origLDFLAGS}" tools/buildTileDB.sh ${TILEDB_LIB_URL}
         else
-            AC_MSG_RESULT([using default url ...])
-            tools/buildTileDB.sh default
+            #AC_MSG_RESULT([using default url ...])
+            ## Important: unset LDFLAGS to not leak -ltiledb from pkg-config
+            LDFLAGS="${origLDFLAGS}" tools/buildTileDB.sh default
         fi
 
     ## case two: download considered
@@ -240,14 +244,8 @@ if test x"${have_tiledb}" = x"no"; then
     ##   - otherwise error
     ##   - but also check we're on x86_64 before downloading
     else
-        ## download URL given by user
-        if test x"${TILEDB_LIB_URL}" != x; then
-            AC_MSG_RESULT([downloading given TileDB library...])
-            ## we use the source download which takes this argument if one given
-            ## and downloads as the same file name
-            cd inst && ../tools/fetchTileDB.sh url ${TILEDB_LIB_URL} && cd ..
 
-        elif test x"${machine}" = x"x86_64" && test x"${uname}" = x"Linux" -o x"${uname}" = x"Darwin"; then
+        if test x"${machine}" = x"x86_64" && test x"${uname}" = x"Linux" -o x"${uname}" = x"Darwin"; then
             AC_MSG_RESULT([downloading x86_64 TileDB library...])
 
             osrel=`tools/getOsRelease.sh`

diff --git a/tools/buildTileDB.sh b/tools/buildTileDB.sh
@@ -20,7 +20,7 @@ fi
 cd src
 
 if [ ! -f tiledb.tar.gz ]; then
-    echo "Downloading ${url} as tiledb.tar.gz ..."
+    echo -n "Downloading ${url}: "
     ${R_HOME}/bin/Rscript ../tools/fetchTileDBSrc.R ${url}
 fi
 
@@ -46,9 +46,7 @@ fi
 ## Build
 mkdir build
 cd build
-../tiledb-src/bootstrap --force-build-all-deps --enable-serialization
-## NB: temporarily disabling and s3
-#../tiledb-src/bootstrap --force-build-all-deps
+../tiledb-src/bootstrap --force-build-all-deps --enable-s3 --enable-serialization
 make -j 2
 make -C tiledb install
 cd ..

diff --git a/tools/fetchTileDB.sh b/tools/fetchTileDB.sh
@@ -1,5 +1,7 @@
 #!/bin/sh
 
+set -eu
+
 if [ $# -lt 1 ] || { [ "$1" != "linux" ] && [ "$1" != "macos" ] && [ "$1" != "url" ]; }; then
     echo "Usage: fetchTileDB.sh (linux|macos|url) (givenurl|machine)"
     echo "where 'givenurl' and 'machine' are optional"
@@ -18,15 +20,14 @@ if [ ! -d tinytest ]; then
     exit 1
 fi
 
+## CRAN wants us permit different R binaries via different PATHs
+: ${R_HOME=`R RHOME`}
+
 tarball="tiledb.tar.gz"
 
 ## Download if need be
 if [ ! -f "${tarball}" ]; then
     ##echo "downloading '${tarball}'"
-    ## CRAN wants us permit different R binaries via different PATHs
-    if [ x"${R_HOME}" = x ]; then
-        R_HOME=`R RHOME`
-    fi
     ${R_HOME}/bin/Rscript ../tools/fetchTileDBLib.R ${os} ${url}
 fi
 

diff --git a/tools/fetchTileDBSrc.R b/tools/fetchTileDBSrc.R
@@ -30,11 +30,24 @@ if (arg == "default") {
     url <- arg
 }
 
-if (!file.exists("tiledb.tar,gz")) {
-    cat("Downloading", url, "\n")
+if (!file.exists("tiledb.tar.gz")) {
     op <- options()
     options(timeout=180)
+
+    cat(url, " ...\n")
     download.file(url, "tiledb.tar.gz", quiet=TRUE)
+
+    ## special case of nightly download from GitHub which comes only as a zipfile
+    ## so we first unzip the file now downloaded to tiledb.tar.gz, and then create the tar
+    if (grepl("zip$", url)) {
+        cat("Converting zip to tar.gz ...\n")
+        unzip("tiledb.tar.gz")
+        unlink("tiledb.tar.gz")
+        Sys.chmod("TileDB-dev/bootstrap", mode="0755")
+        options(warn=-1)
+        tar("tiledb.tar.gz", "TileDB-dev", compression="gzip")
+    }
+
     options(op)
 }
 q()