Fix build scripts for Ubuntu and update build guide

serenadeai · Apr 8, 2024 · b6cada2 · b6cada2
1 parent 6d16a1a
commit b6cada2
Show file tree

Hide file tree

Showing 13 changed files with 115 additions and 22 deletions.
diff --git a/client/package.json b/client/package.json
@@ -73,7 +73,9 @@
     "electron:serve": "wait-on http-get://localhost:4000/ && npm run dev:main",
     "package": "npm-run-all build package:dist",
     "package:unsigned": "cross-env SKIP_SIGN=1 npm-run-all build package:distunsigned",
-    "package:dist": "electron-builder",
+    "package:dist-linux": "electron-builder",
+    "package:dist-win": "electron-builder --win",
+    "package:dist-mac": "electron-builder --mac",
     "package:distunsigned": "electron-builder -c.mac.identity=null",
     "release": "npm-run-all build release:publish",
     "release:publish": "electron-builder --publish always",

diff --git a/docs/building.md → docs/building-client.md b/docs/building.md → docs/building-client.md
@@ -1,20 +1,79 @@
-# Building Serenade
+Serenade is built using the [Gradle](https://gradle.org) build system. We also have a few scripts useful for running
+various Serenade services.
 
-Serenade is built using the [Gradle](https://gradle.org) build system. We also have a few scripts useful for running various Serenade services.
+## Building the client
+Building Serenade is supported on Linux and Mac. The first thing you need to do is to download the source code.
 
-## Client
+```shell
+git clone https://github.com/serenadeai/serenade.git
+```
 
-To run the Serenade app, simply run:
+For Linux:
 
-    cd client
-    ./bin/dev.py
+- Use Ubuntu Jammy (22.04) 
+- Run the scripts to install the Ubuntu prerequisites. When prompted for configuration questions, just use the defaults.
+
+```shell
+cd serenade/
+./scripts/setup/setup-ubuntu.sh
+# If prompted for anything during the installation just press OK and continue.
+# As requested by the script, add the export lines to your ~/.bashrc~/ or .zshrc file and reload the
+# configs
+
+cd client
+./bin/build.py
+```
+
+Note: If you run into to dependency problems, trying to run the scripts from inside a fresh
+Ubuntu Jammy VM or container.
+
+For MacOS:
+
+- Run the scripts to install the MacOS prerequisites:
+
+```shell
+cd serenade/
+./scripts/setup/setup-mac.sh
+
+cd client
+cd ./bin/build.py
+```
+
+Once you have the build sucessfully completed, you can run the client locally, or package it:
+
+```shell
+cd client
+# To run the client locally:
+npm run dev
+
+# To generate the app image:
+npm run package:dist-linux
+
+# To generate the windows executable:
+npm run package:dist-win
+
+# To generate the mac executable:
+npm run package:dist-mac
+```
+
+```
 
 This will run a local version of the client that uses Serenade Cloud as the backend.
 
 If you'd instead like the client to connect to a specific endpoint (e.g., a local server you're running yourself), you can run:
 
     ENDPOINT=http://localhost:17200 ./bin/dev.py
 
+
+## Other Dependencies
+
+Now run the common dependency builder and installer:
+
+```shell
+./scripts/setup/build-dependencies.sh
+# Sit back and relax, this will take a while to finish
+```
+
 ## Service Setup
 
 ### Docker

diff --git a/docs/training-models.md b/docs/training-models.md
@@ -4,7 +4,7 @@ This document explains how to train the models that are used by Serenade. If you
 
 ## Setup
 
-Make sure all of the dependencies for training models are installed and built. You can do so by following the instructions in [Building](building.md)—make sure you do *not* use the `--minimal` flag or `serenade-minimal` Docker image (which are used only for running Serenade).
+Make sure all of the dependencies for training models are installed and built. You can do so by following the instructions in [Building](building-client.md)—make sure you do *not* use the `--minimal` flag or `serenade-minimal` Docker image (which are used only for running Serenade).
 
 Ensure that you have plenty of disk space. The language model for the speech engine is trained on all of the source code data available in all programming languages supported by Serenade, totaling to about 50 GB.
 

diff --git a/scripts/setup/build-dependencies.sh b/scripts/setup/build-dependencies.sh
@@ -45,7 +45,8 @@ pip3 install --upgrade \
   pyenchant \
   pyyaml \
   requests \
-  sentencepiece==0.1.95
+  sentencepiece==0.1.95 \
+  boto3
 
 sudo-non-docker npm install -g \
   prettier \
@@ -188,3 +189,4 @@ if [[ "$minimal" == "true" ]] ; then
   find kaldi -type f -name "*.so*" -delete
   find kaldi -type f -name "*.o" -delete
 fi
+
diff --git a/scripts/setup/setup-ubuntu.sh b/scripts/setup/setup-ubuntu.sh
@@ -1,6 +1,4 @@
-#!/bin/bash
-
-set -e
+#!/bin/bash -e
 
 HERE=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)
 . $HERE/paths.sh
@@ -15,6 +13,9 @@ while [[ $# -gt 0 ]]; do
     --cpu)
       gpu=false
       ;;
+    --debug)
+      set -x
+      ;;
     *)
       echo "Unknown argument: $1"
       exit 1
@@ -28,22 +29,29 @@ sudo-non-docker apt-get install --upgrade -y \
   apt-transport-https \
   curl \
   gnupg2 \
-  wget
+  wget \
+  ca-certificates
 
 if [[ "$gpu" == "true" ]] ; then
   sudo-non-docker apt-get install --upgrade -y ubuntu-drivers-common
   sudo-non-docker ubuntu-drivers autoinstall
 fi
 
-curl -sL https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB | sudo-non-docker apt-key add -
+curl -sL https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | sudo-non-docker gpg --dearmor -o /etc/apt/trusted.gpg.d/intel-mkl.gpg
 echo "deb https://apt.repos.intel.com/mkl all main" | sudo-non-docker tee /etc/apt/sources.list.d/intel-mkl.list
+
+NODE_MAJOR=18
+curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | sudo-non-docker gpg --dearmor -o /etc/apt/trusted.gpg.d/nodesource.gpg
+echo "deb [arch=amd64] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | sudo-non-docker tee /etc/apt/sources.list.d/nodesource.list
 sudo-non-docker apt-get update
+sudo-non-docker apt-get install nodejs -y
+
 sudo-non-docker apt-get install --upgrade -y \
   autoconf \
   automake \
   build-essential \
   ca-certificates \
-  clang-format-9 \
+  clang-format \
   cmake \
   ffmpeg \
   fonts-liberation \
@@ -53,16 +61,14 @@ sudo-non-docker apt-get install --upgrade -y \
   gfortran \
   git \
   groff \
-  intel-mkl-64bit-2020.2-108 \
+  intel-mkl \
   libasound2 \
   libc++-dev \
   libssl-dev \
   libpq-dev \
   libtool \
   logrotate \
   lsb-release \
-  nodejs \
-  npm \
   $([[ "$gpu" == "true" ]] && echo "nvidia-cuda-toolkit")  \
   postgresql-client \
   psmisc \
@@ -79,15 +85,27 @@ sudo-non-docker apt-get install --upgrade -y \
   vim \
   xdg-utils \
   yarn \
-  zlib1g-dev
+  zlib1g-dev \
+  pkg-config \
+  libx11-dev \
+  uglifyjs \
+  libxtst-dev \
+  libfuse2
 
 curl https://download.java.net/java/GA/jdk14.0.1/664493ef4a6946b186ff29eb326336a2/7/GPL/openjdk-14.0.1_linux-x64_bin.tar.gz -Lso jdk.tar.gz
 tar xf jdk.tar.gz
 rm jdk.tar.gz
 
-echo ""
+curl https://services.gradle.org/distributions/gradle-7.4.2-bin.zip -Lso gradle-7.4.2-bin.zip
+unzip -qq gradle-7.4.2-bin.zip
+rm gradle-7.4.2-bin.zip
+
+sudo ln -s $SERENADE_LIBRARY_ROOT/gradle-7.4.2/bin/gradle /usr/local/bin/gradle
+sudo ln -s jdk-14.0.1/bin/java /usr/local/bin/java
+
+echo "" && echo "" && echo ""
 echo "Install complete!"
-echo "Now, run build-dependencies.sh and add the following to your ~/.zshrc or ~/.bashrc:"
+echo "Now, run ./scripts/setup/build-dependencies.sh and add the following to your ~/.zshrc or ~/.bashrc:"
 echo "export PATH=\"$SERENADE_LIBRARY_ROOT/jdk-14.0.1/bin:$SERENADE_LIBRARY_ROOT/gradle-7.4.2/bin:\$PATH\""
 echo "export JAVA_HOME=\"$SERENADE_LIBRARY_ROOT/jdk-14.0.1\""
 

diff --git a/speech-engine/server/CMakeLists.txt b/speech-engine/server/CMakeLists.txt
@@ -139,7 +139,7 @@ if(APPLE)
   target_link_libraries(serenade-speech-engine "-framework Accelerate")
 else()
   target_link_libraries(serenade-speech-engine
-    "-Wl,--start-group;/opt/intel/mkl/lib/intel64/libmkl_intel_ilp64.a;/opt/intel/mkl/lib/intel64/libmkl_sequential.a;/opt/intel/mkl/lib/intel64/libmkl_core.a;-Wl,--end-group"
+    "-Wl,--start-group;/usr/lib/x86_64-linux-gnu/libmkl_intel_ilp64.a;/usr/lib/x86_64-linux-gnu/libmkl_sequential.a;/usr/lib/x86_64-linux-gnu/libmkl_core.a;-Wl,--end-group"
     dl
     gomp
   )

diff --git a/speech-engine/server/include/lattice_ops.h b/speech-engine/server/include/lattice_ops.h
@@ -1,6 +1,7 @@
 #ifndef LIBRARY_LATTICE_OPS_H
 #define LIBRARY_LATTICE_OPS_H
 
+#include <optional>
 #include "lat/kaldi-lattice.h"
 
 namespace speech_engine {

diff --git a/speech-engine/server/include/recognizer.h b/speech-engine/server/include/recognizer.h
@@ -12,6 +12,8 @@
 #include "recognizer.h"
 #include "recognizer_config.h"
 
+#include <optional>
+
 namespace speech_engine {
 
 class Recognizer {

diff --git a/speech-engine/server/include/rescorer.h b/speech-engine/server/include/rescorer.h
@@ -6,6 +6,8 @@
 #include "lat/kaldi-lattice.h"
 #include "lm/const-arpa-lm.h"
 
+#include <optional>
+
 namespace speech_engine {
 
 class Rescorer {

diff --git a/speech-engine/server/server/speech_engine_server.cc b/speech-engine/server/server/speech_engine_server.cc
@@ -7,6 +7,7 @@
 #include <chrono>
 #include <memory>
 #include <mutex>
+#include <optional>
 
 #include "base/kaldi-common.h"
 #include "crow.h"

diff --git a/speech-engine/server/src/lattice_ops.cc b/speech-engine/server/src/lattice_ops.cc
@@ -1,5 +1,7 @@
 #include "lattice_ops.h"
 
+#include <optional>
+
 using namespace kaldi;
 
 namespace speech_engine {

diff --git a/speech-engine/server/src/recognizer.cc b/speech-engine/server/src/recognizer.cc
@@ -7,6 +7,8 @@
 #include "fstext/table-matcher.h"
 #include "util/common-utils.h"
 
+#include <optional>
+
 using namespace kaldi;
 
 namespace speech_engine {

diff --git a/speech-engine/server/src/rescorer.cc b/speech-engine/server/src/rescorer.cc
@@ -4,6 +4,8 @@
 #include "lat/lattice-functions.h"
 #include "lattice_ops.h"
 
+#include <optional>
+
 using namespace kaldi;
 
 namespace speech_engine {