diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index f24342e993..b556905e58 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -105,7 +105,7 @@ jobs: run: | cp ranger-*.tar.gz dev-support/ranger-docker/dist cp version dev-support/ranger-docker/dist - + - name: Cache downloaded archives uses: actions/cache@v4 with: @@ -159,8 +159,9 @@ jobs: -f docker-compose.ranger-hive.yml \ -f docker-compose.ranger-knox.yml \ -f docker-compose.ranger-ozone.yml up -d + - name: Check status of containers and remove them - run: | + run: | sleep 60 containers=(ranger ranger-zk ranger-solr ranger-postgres ranger-usersync ranger-tagsync ranger-kms ranger-hadoop ranger-hbase ranger-kafka ranger-hive ranger-knox ozone-om ozone-scm ozone-datanode); flag=true; diff --git a/dev-support/ranger-docker/.env b/dev-support/ranger-docker/.env index ca27d0799e..0ef00da03e 100644 --- a/dev-support/ranger-docker/.env +++ b/dev-support/ranger-docker/.env @@ -36,6 +36,7 @@ UBI_VERSION=latest MARIADB_VERSION=10.7.3 POSTGRES_VERSION=12 ORACLE_VERSION=23.6 +SQLSERVER_VERSION=2019-latest ENABLE_DB_MOUNT=true ZK_VERSION=3.9.2 SOLR_VERSION=8.11.3 diff --git a/dev-support/ranger-docker/Dockerfile.ranger b/dev-support/ranger-docker/Dockerfile.ranger index 087430dd72..9d14a1eb9c 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger +++ b/dev-support/ranger-docker/Dockerfile.ranger @@ -58,6 +58,10 @@ FROM ranger AS ranger_oracle COPY ./downloads/ojdbc8.jar /home/ranger/dist/ RUN mv /home/ranger/dist/ojdbc8.jar /usr/share/java/oracle.jar +FROM ranger AS ranger_sqlserver +COPY ./downloads/mssql-jdbc-12.8.1.jre8.jar /home/ranger/dist/ +RUN mv /home/ranger/dist/mssql-jdbc-12.8.1.jre8.jar /usr/share/java/mssql.jar + FROM ranger_${RANGER_DB_TYPE} USER ranger diff --git a/dev-support/ranger-docker/Dockerfile.ranger-kms b/dev-support/ranger-docker/Dockerfile.ranger-kms index 5e70d0da53..be85bbcff8 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-kms +++ b/dev-support/ranger-docker/Dockerfile.ranger-kms @@ -58,6 +58,10 @@ FROM ranger-kms AS ranger_oracle COPY ./downloads/ojdbc8.jar /home/ranger/dist/ RUN mv /home/ranger/dist/ojdbc8.jar /usr/share/java/oracle.jar +FROM ranger-kms AS ranger_sqlserver +COPY ./downloads/mssql-jdbc-12.8.1.jre8.jar /home/ranger/dist/ +RUN mv /home/ranger/dist/mssql-jdbc-12.8.1.jre8.jar /usr/share/java/mssql.jar + FROM ranger_${RANGER_DB_TYPE} ENTRYPOINT [ "/home/ranger/scripts/ranger-kms.sh" ] diff --git a/dev-support/ranger-docker/Dockerfile.ranger-sqlserver b/dev-support/ranger-docker/Dockerfile.ranger-sqlserver new file mode 100644 index 0000000000..3559fa49dd --- /dev/null +++ b/dev-support/ranger-docker/Dockerfile.ranger-sqlserver @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG SQLSERVER_VERSION + +FROM mcr.microsoft.com/mssql/server:${SQLSERVER_VERSION} + +ENV ACCEPT_EULA=Y +ENV MSSQL_SA_PASSWORD=rangerR0cks! + +EXPOSE 1433 + +USER root + +RUN mkdir -p /docker-entrypoint-initdb.d +COPY config/init_mssql.sh /docker-entrypoint-initdb.d/ +RUN chown -R mssql /docker-entrypoint-initdb.d/ +RUN chmod +x /docker-entrypoint-initdb.d/init_mssql.sh + +USER mssql + +ENTRYPOINT ["/docker-entrypoint-initdb.d/init_mssql.sh"] diff --git a/dev-support/ranger-docker/config/init_mssql.sh b/dev-support/ranger-docker/config/init_mssql.sh new file mode 100644 index 0000000000..32036c9eae --- /dev/null +++ b/dev-support/ranger-docker/config/init_mssql.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +/opt/mssql/bin/sqlservr & + +# Wait for SQL Server to be ready +echo "Waiting for SQL Server to start..." +RETRIES=30 # Number of retries +SLEEP_INTERVAL=5 # Seconds to wait between retries +for i in $(seq 1 $RETRIES); do + # Try to connect to SQL Server + /opt/mssql-tools18/bin/sqlcmd -S localhost -U SA -P "rangerR0cks!" -Q "SELECT 1" -C > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo "SQL Server is ready!" + break + else + echo "SQL Server is not ready yet. Waiting..." + sleep $SLEEP_INTERVAL + fi +done + +if [ $i -eq $RETRIES ]; then + echo "SQL Server did not become ready in time. Exiting." + exit 1 +fi + +/opt/mssql-tools18/bin/sqlcmd -S localhost -U SA -P 'rangerR0cks!' -Q " + +-- Set the database context +USE master; + +-- Create databases +CREATE DATABASE ranger; +CREATE DATABASE rangerkms; +CREATE DATABASE hive; +GO + +-- Create users and assign permissions +USE ranger; +CREATE LOGIN rangeradmin WITH PASSWORD = 'rangerR0cks!'; +CREATE USER rangeradmin FOR LOGIN rangeradmin; +ALTER ROLE db_owner ADD MEMBER rangeradmin; -- Grant equivalent high-level permissions +GO + +USE rangerkms; +CREATE LOGIN rangerkms WITH PASSWORD = 'rangerR0cks!'; +CREATE USER rangerkms FOR LOGIN rangerkms; +ALTER ROLE db_owner ADD MEMBER rangerkms; -- Grant equivalent high-level permissions +GO + +USE hive; +CREATE LOGIN hive WITH PASSWORD = 'rangerR0cks!'; +CREATE USER hive FOR LOGIN hive; +ALTER ROLE db_owner ADD MEMBER hive; -- Grant equivalent high-level permissions +GO +" -C + +# Bring SQL Server to the foreground +wait -n +exec /opt/mssql/bin/sqlservr diff --git a/dev-support/ranger-docker/docker-compose.ranger-sqlserver.yml b/dev-support/ranger-docker/docker-compose.ranger-sqlserver.yml new file mode 100644 index 0000000000..f1e790019f --- /dev/null +++ b/dev-support/ranger-docker/docker-compose.ranger-sqlserver.yml @@ -0,0 +1,25 @@ +services: + ranger-db: + build: + context: . + dockerfile: Dockerfile.ranger-sqlserver + args: + - SQLSERVER_VERSION=${SQLSERVER_VERSION} + image: ranger-sqlserver + container_name: ranger-sqlserver + hostname: ranger-db.example.com + networks: + - ranger + healthcheck: + test: [ + "CMD-SHELL", + "/opt/mssql-tools18/bin/sqlcmd -S localhost -U SA -P rangerR0cks! -Q \"SELECT 1\" -C" # -C bypasses SSL validation + ] + interval: 15s + timeout: 10s + retries: 3 + start_period: 10s + +networks: + ranger: + name: rangernw diff --git a/dev-support/ranger-docker/download-archives.sh b/dev-support/ranger-docker/download-archives.sh index e7a4bf217b..0bfca41ade 100755 --- a/dev-support/ranger-docker/download-archives.sh +++ b/dev-support/ranger-docker/download-archives.sh @@ -44,6 +44,7 @@ downloadIfNotPresent() { downloadIfNotPresent postgresql-42.2.16.jre7.jar "https://search.maven.org/remotecontent?filepath=org/postgresql/postgresql/42.2.16.jre7" downloadIfNotPresent mysql-connector-java-8.0.28.jar "https://search.maven.org/remotecontent?filepath=mysql/mysql-connector-java/8.0.28" downloadIfNotPresent ojdbc8.jar https://download.oracle.com/otn-pub/otn_software/jdbc/236 +downloadIfNotPresent mssql-jdbc-12.8.1.jre8.jar https://repo1.maven.org/maven2/com/microsoft/sqlserver/mssql-jdbc/12.8.1.jre8 downloadIfNotPresent log4jdbc-1.2.jar https://repo1.maven.org/maven2/com/googlecode/log4jdbc/log4jdbc/1.2 if [[ $# -eq 0 ]] diff --git a/dev-support/ranger-docker/scripts/hive-site-sqlserver.xml b/dev-support/ranger-docker/scripts/hive-site-sqlserver.xml new file mode 100644 index 0000000000..5bed21c2d8 --- /dev/null +++ b/dev-support/ranger-docker/scripts/hive-site-sqlserver.xml @@ -0,0 +1,50 @@ + + + + + javax.jdo.option.ConnectionURL + jdbc:sqlserver://ranger-db/hive + + + javax.jdo.option.ConnectionDriverName + com.microsoft.sqlserver.jdbc.SQLServerDriver + + + javax.jdo.option.ConnectionUserName + hive + + + javax.jdo.option.ConnectionPassword + rangerR0cks! + + + hive.server2.enable.doAs + false + + + hive.zookeeper.quorum + ranger-zk.example.com + + + hive.zookeeper.client.port + 2181 + + diff --git a/dev-support/ranger-docker/scripts/ranger-admin-install-sqlserver.properties b/dev-support/ranger-docker/scripts/ranger-admin-install-sqlserver.properties new file mode 100644 index 0000000000..b69e22d1e4 --- /dev/null +++ b/dev-support/ranger-docker/scripts/ranger-admin-install-sqlserver.properties @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# This file provides a list of the deployment variables for the Policy Manager Web Application +# + +PYTHON_COMMAND_INVOKER=python3 +RANGER_ADMIN_LOG_DIR=/var/log/ranger +RANGER_PID_DIR_PATH=/var/run/ranger +DB_FLAVOR=MSSQL +SQL_CONNECTOR_JAR=/usr/share/java/mssql.jar +CONNECTION_STRING_ADDITIONAL_PARAMS="trustServerCertificate=true;" +RANGER_ADMIN_LOGBACK_CONF_FILE=/opt/ranger/admin/ews/webapp/WEB-INF/classes/conf/logback.xml + +db_root_user=sa +db_root_password=rangerR0cks! +db_host=ranger-db + +db_name=ranger +db_user=rangeradmin +db_password=rangerR0cks! + +postgres_core_file=db/postgres/optimized/current/ranger_core_db_postgres.sql +postgres_audit_file=db/postgres/xa_audit_db_postgres.sql +mysql_core_file=db/mysql/optimized/current/ranger_core_db_mysql.sql +mysql_audit_file=db/mysql/xa_audit_db.sql +oracle_core_file=db/oracle/optimized/current/ranger_core_db_oracle.sql +oracle_audit_file=db/oracle/xa_audit_db_oracle.sql +sqlserver_core_file=db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql +sqlserver_audit_file=db/sqlserver/xa_audit_db_sqlserver.sql + +# For over-riding the jdbc url +is_override_db_connection_string=true +db_override_jdbc_connection_string="jdbc:sqlserver://ranger-db;databaseName=ranger;trustServerCertificate=true;" + +rangerAdmin_password=rangerR0cks! +rangerTagsync_password=rangerR0cks! +rangerUsersync_password=rangerR0cks! +keyadmin_password=rangerR0cks! + + +audit_store=solr +audit_solr_urls=http://ranger-solr:8983/solr/ranger_audits +audit_solr_collection_name=ranger_audits + +# audit_store=elasticsearch +audit_elasticsearch_urls= +audit_elasticsearch_port=9200 +audit_elasticsearch_protocol=http +audit_elasticsearch_user=elastic +audit_elasticsearch_password=elasticsearch +audit_elasticsearch_index=ranger_audits +audit_elasticsearch_bootstrap_enabled=true + +policymgr_external_url=http://ranger-admin:6080 +policymgr_http_enabled=true + +unix_user=ranger +unix_user_pwd=ranger +unix_group=ranger + +# Following variables are referenced in db_setup.py. Do not remove these +sqlanywhere_core_file= +cred_keystore_filename= + +# ################# DO NOT MODIFY ANY VARIABLES BELOW ######################### +# +# --- These deployment variables are not to be modified unless you understand the full impact of the changes +# +################################################################################ +XAPOLICYMGR_DIR=$PWD +app_home=$PWD/ews/webapp +TMPFILE=$PWD/.fi_tmp +LOGFILE=$PWD/logfile +LOGFILES="$LOGFILE" + +JAVA_BIN='java' +JAVA_VERSION_REQUIRED='1.8' + +ranger_admin_max_heap_size=1g +#retry DB and Java patches after the given time in seconds. +PATCH_RETRY_INTERVAL=120 +STALE_PATCH_ENTRY_HOLD_TIME=10 + +hadoop_conf= +authentication_method=UNIX diff --git a/dev-support/ranger-docker/scripts/ranger-kms-install-sqlserver.properties b/dev-support/ranger-docker/scripts/ranger-kms-install-sqlserver.properties new file mode 100644 index 0000000000..04c96989be --- /dev/null +++ b/dev-support/ranger-docker/scripts/ranger-kms-install-sqlserver.properties @@ -0,0 +1,223 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# This file provides a list of the deployment variables for the Ranger KMS Web Application +# + +PYTHON_COMMAND_INVOKER=python3 +DB_FLAVOR=MSSQL +SQL_CONNECTOR_JAR=/usr/share/java/mssql.jar +CONNECTION_STRING_ADDITIONAL_PARAMS="trustServerCertificate=true;" + +db_root_user=sa +db_root_password=rangerR0cks! +db_host=ranger-db + +db_name=rangerkms +db_user=rangerkms +db_password=rangerR0cks! + +# Following variables are referenced in db_setup.py. Do not remove these +mysql_core_file=db/mysql/kms_core_db.sql +postgres_core_file=db/postgres/kms_core_db_postgres.sql +oracle_core_file=db/oracle/kms_core_db_oracle.sql +sqlserver_core_file=db/sqlserver/kms_core_db_sqlserver.sql +sqlanywhere_core_file= + +# For over-riding the jdbc url +is_override_db_connection_string=true +db_override_jdbc_connection_string="jdbc:sqlserver://ranger-db;databaseName=rangerkms;trustServerCertificate=true;" + + +#SSL config +db_ssl_enabled=false +db_ssl_required=false +db_ssl_verifyServerCertificate=false +#db_ssl_auth_type=1-way|2-way, where 1-way represents standard one way ssl authentication and 2-way represents mutual ssl authentication +db_ssl_auth_type=2-way +javax_net_ssl_keyStore= +javax_net_ssl_keyStorePassword= +javax_net_ssl_trustStore= +javax_net_ssl_trustStorePassword= +javax_net_ssl_trustStore_type=jks +javax_net_ssl_keyStore_type=jks + +# For postgresql db +db_ssl_certificate_file= + +#------------------------- DB CONFIG - END ---------------------------------- +#KMS Server config +ranger_kms_http_enabled=true +ranger_kms_https_keystore_file= +ranger_kms_https_keystore_keyalias=rangerkms +ranger_kms_https_keystore_password= + +#------------------------- RANGER KMS Install Dir ------------------ +COMPONENT_INSTALL_DIR_NAME=/opt/ranger/kms + +#------------------------- RANGER KMS Master Key Crypt Key ------------------ +KMS_MASTER_KEY_PASSWD=Str0ngPassw0rd + +#------------------------- Ranger KMS Kerberos Configuration --------------------------- +kms_principal= +kms_keytab= +hadoop_conf= + +#------------------------- Ranger KMS HSM CONFIG ------------------------------ +HSM_TYPE=LunaProvider +HSM_ENABLED=false +HSM_PARTITION_NAME=par19 +HSM_PARTITION_PASSWORD=S@fenet123 + +#------------------------- Ranger SAFENET KEYSECURE CONFIG ------------------------------ +KEYSECURE_ENABLED=false +KEYSECURE_USER_PASSWORD_AUTHENTICATION=true +KEYSECURE_MASTERKEY_NAME=safenetkeysecure +KEYSECURE_USERNAME=user1 +KEYSECURE_PASSWORD=t1e2s3t4 +KEYSECURE_HOSTNAME=SunPKCS11-keysecurehn +KEYSECURE_MASTER_KEY_SIZE=256 +KEYSECURE_LIB_CONFIG_PATH=/opt/safenetConf/64/8.3.1/sunpkcs11.cfg + +#------------------------- Ranger Azure Key Vault ------------------------------ +AZURE_KEYVAULT_ENABLED=false +AZURE_KEYVAULT_SSL_ENABLED=false +AZURE_CLIENT_ID=50fd7ca6-fd4f-4785-a13f-1a6cc4e95e42 +AZURE_CLIENT_SECRET= +AZURE_AUTH_KEYVAULT_CERTIFICATE_PATH=/home/machine/Desktop/azureAuthCertificate/keyvault-MyCert.pfx +# Initialize below prop if your certificate file has any password +#AZURE_AUTH_KEYVAULT_CERTIFICATE_PASSWORD=certPass +AZURE_MASTERKEY_NAME=RangerMasterKey +# E.G. RSA, RSA_HSM, EC, EC_HSM, OCT +AZURE_MASTER_KEY_TYPE=RSA +# E.G. RSA_OAEP, RSA_OAEP_256, RSA1_5, RSA_OAEP +ZONE_KEY_ENCRYPTION_ALGO=RSA_OAEP +AZURE_KEYVAULT_URL=https://shahkeyvault.vault.azure.net/ + +#------------------------- Ranger Google Cloud HSM ------------------------------ +IS_GCP_ENABLED=false +GCP_KEYRING_ID= +GCP_CRED_JSON_FILE=/full/path/to/credfile.json +GCP_PROJECT_ID= +GCP_LOCATION_ID= +GCP_MASTER_KEY_NAME=MyMasterKeyNameChangeIt + +#------------------------- Ranger Tencent KMS ------------------------------ +TENCENT_KMS_ENABLED=false +TENCENT_MASTERKEY_ID=b756b016-6e11-11ec-a735-525400fe0300 +TENCENT_CLIENT_ID=AKIDrXx6ybx2qNdiaBWaNs76pGQJvFJ6crpW +TENCENT_CLIENT_SECRET= +TENCENT_CLIENT_REGION=ap-beijing + +# ------- UNIX User CONFIG ---------------- +# +unix_user=rangerkms +unix_user_pwd=kms +unix_group=ranger + +# +# ------- UNIX User CONFIG - END ---------------- +# + +POLICY_MGR_URL=http://ranger:6080 +REPOSITORY_NAME=dev_kms + +# AUDIT configuration with V3 properties +XAAUDIT.SOLR.IS_ENABLED=true +XAAUDIT.SOLR.MAX_QUEUE_SIZE=1 +XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000 +XAAUDIT.SOLR.SOLR_URL=http://ranger-solr:8983/solr/ranger_audits +XAAUDIT.SUMMARY.ENABLE=true + +# Following properties are needed to get past installation script! Please don't remove +XAAUDIT.HDFS.IS_ENABLED=false +XAAUDIT.HDFS.DESTINATION_DIRECTORY=/ranger/audit +XAAUDIT.HDFS.DESTINTATION_FILE=hive +XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900 +XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400 +XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/var/log/hive/audit +XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/var/log/hive/audit/archive +XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log +XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 +XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 + +XAAUDIT.SOLR.ENABLE=true +XAAUDIT.SOLR.URL=http://ranger-solr:8983/solr/ranger_audits +XAAUDIT.SOLR.USER=NONE +XAAUDIT.SOLR.PASSWORD=NONE +XAAUDIT.SOLR.ZOOKEEPER=NONE +XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/hive/audit/solr/spool + +XAAUDIT.ELASTICSEARCH.ENABLE=false +XAAUDIT.ELASTICSEARCH.URL=NONE +XAAUDIT.ELASTICSEARCH.USER=NONE +XAAUDIT.ELASTICSEARCH.PASSWORD=NONE +XAAUDIT.ELASTICSEARCH.INDEX=NONE +XAAUDIT.ELASTICSEARCH.PORT=NONE +XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE + +XAAUDIT.HDFS.ENABLE=true +XAAUDIT.HDFS.HDFS_DIR=hdfs://ranger-hadoop:9000/ranger/audit +XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/hive/audit/hdfs/spool + +XAAUDIT.HDFS.AZURE_ACCOUNTNAME=__REPLACE_AZURE_ACCOUNT_NAME +XAAUDIT.HDFS.AZURE_ACCOUNTKEY=__REPLACE_AZURE_ACCOUNT_KEY +XAAUDIT.HDFS.AZURE_SHELL_KEY_PROVIDER=__REPLACE_AZURE_SHELL_KEY_PROVIDER +XAAUDIT.HDFS.AZURE_ACCOUNTKEY_PROVIDER=__REPLACE_AZURE_ACCOUNT_KEY_PROVIDER + +XAAUDIT.LOG4J.ENABLE=false +XAAUDIT.LOG4J.IS_ASYNC=false +XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240 +XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000 +XAAUDIT.LOG4J.DESTINATION.LOG4J=false +XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit + +XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false +XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE +XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE +XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE +XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE + +SSL_KEYSTORE_FILE_PATH=/etc/hive/conf/ranger-plugin-keystore.jks +SSL_KEYSTORE_PASSWORD=myKeyFilePassword +SSL_TRUSTSTORE_FILE_PATH=/etc/hive/conf/ranger-plugin-truststore.jks +SSL_TRUSTSTORE_PASSWORD=changeit + + +# Custom log directory path +RANGER_KMS_LOG_DIR=/var/log/ranger/kms + +#PID file path +RANGER_KMS_PID_DIR_PATH=/var/run/ranger_kms +# ################# DO NOT MODIFY ANY VARIABLES BELOW ######################### +# +# --- These deployment variables are not to be modified unless you understand the full impact of the changes +# +################################################################################ +KMS_DIR=$PWD +app_home=$PWD/ews/webapp +TMPFILE=$PWD/.fi_tmp +LOGFILE=$PWD/logfile + +JAVA_BIN='java' +JAVA_VERSION_REQUIRED='1.8' +JAVA_ORACLE='Java(TM) SE Runtime Environment' + +cred_keystore_filename=$app_home/WEB-INF/classes/conf/.jceks/rangerkms.jceks + +KMS_BLACKLIST_DECRYPT_EEK=hdfs diff --git a/kms/scripts/dba_script.py b/kms/scripts/dba_script.py index 7e7b4e1ce3..5d19a7d215 100755 --- a/kms/scripts/dba_script.py +++ b/kms/scripts/dba_script.py @@ -825,19 +825,27 @@ def writeDrymodeCmd(self, xa_db_root_user, xa_db_root_password, db_user, db_pass class SqlServerConf(BaseDB): # Constructor - def __init__(self, host, SQL_CONNECTOR_JAR, JAVA_BIN): + def __init__(self, host, SQL_CONNECTOR_JAR, JAVA_BIN, is_db_override_jdbc_connection_string, db_override_jdbc_connection_string): self.host = host self.SQL_CONNECTOR_JAR = SQL_CONNECTOR_JAR self.JAVA_BIN = JAVA_BIN + self.is_db_override_jdbc_connection_string = is_db_override_jdbc_connection_string + self.db_override_jdbc_connection_string = db_override_jdbc_connection_string def get_jisql_cmd(self, user, password, db_name): #TODO: User array for forming command path = RANGER_KMS_HOME self.JAVA_BIN = self.JAVA_BIN.strip("'") if is_unix: - jisql_cmd = "%s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -user %s -p '%s' -driver mssql -cstring jdbc:sqlserver://%s\\;databaseName=%s -noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path,user, password, self.host,db_name) + if self.is_db_override_jdbc_connection_string == 'true' and self.db_override_jdbc_connection_string is not None and len(self.db_override_jdbc_connection_string) > 0: + jisql_cmd = "%s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -user %s -p '%s' -driver mssql -cstring %s -noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path, user, password, self.db_override_jdbc_connection_string) + else: + jisql_cmd = "%s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -user %s -p '%s' -driver mssql -cstring jdbc:sqlserver://%s\\;databaseName=%s -noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path,user, password, self.host,db_name) elif os_name == "WINDOWS": - jisql_cmd = "%s -cp %s;%s\\jisql\\lib\\* org.apache.util.sql.Jisql -user %s -p \"%s\" -driver mssql -cstring jdbc:sqlserver://%s;databaseName=%s -noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path, user, password, self.host,db_name) + if self.is_db_override_jdbc_connection_string == 'true' and self.db_override_jdbc_connection_string is not None and len(self.db_override_jdbc_connection_string) > 0: + jisql_cmd = "%s -cp %s;%s\\jisql\\lib\\* org.apache.util.sql.Jisql -user %s -p \"%s\" -driver mssql -cstring %s -noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path, user, password, self.db_override_jdbc_connection_string) + else: + jisql_cmd = "%s -cp %s;%s\\jisql\\lib\\* org.apache.util.sql.Jisql -user %s -p \"%s\" -driver mssql -cstring jdbc:sqlserver://%s;databaseName=%s -noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path, user, password, self.host,db_name) return jisql_cmd def verify_user(self, root_user, db_root_password, db_user,dryMode): @@ -1398,6 +1406,14 @@ def main(argv): javax_net_ssl_keyStorePassword='' javax_net_ssl_trustStore='' javax_net_ssl_trustStorePassword='' + + is_override_db_connection_string='false' + db_override_jdbc_connection_string='' + if 'is_override_db_connection_string' in globalDict: + is_override_db_connection_string=globalDict['is_override_db_connection_string'].lower() + if 'db_override_jdbc_connection_string' in globalDict: + db_override_jdbc_connection_string=globalDict['db_override_jdbc_connection_string'].strip() + if XA_DB_FLAVOR == "MYSQL" or XA_DB_FLAVOR == "POSTGRES": if 'db_ssl_enabled' in globalDict: db_ssl_enabled=globalDict['db_ssl_enabled'].lower() @@ -1451,7 +1467,7 @@ def main(argv): elif XA_DB_FLAVOR == "MSSQL": SQLSERVER_CONNECTOR_JAR=CONNECTOR_JAR - xa_sqlObj = SqlServerConf(xa_db_host, SQLSERVER_CONNECTOR_JAR, JAVA_BIN) + xa_sqlObj = SqlServerConf(xa_db_host, SQLSERVER_CONNECTOR_JAR, JAVA_BIN, is_override_db_connection_string, db_override_jdbc_connection_string) xa_db_core_file = os.path.join(RANGER_KMS_HOME,sqlserver_core_file) elif XA_DB_FLAVOR == "SQLA": diff --git a/kms/scripts/setup.sh b/kms/scripts/setup.sh index 9e31f5d347..b547b1751b 100755 --- a/kms/scripts/setup.sh +++ b/kms/scripts/setup.sh @@ -66,8 +66,10 @@ get_prop_or_default() { } PYTHON_COMMAND_INVOKER=$(get_prop 'PYTHON_COMMAND_INVOKER' $PROPFILE) + DB_FLAVOR=$(get_prop 'DB_FLAVOR' $PROPFILE) SQL_CONNECTOR_JAR=$(get_prop 'SQL_CONNECTOR_JAR' $PROPFILE) +CONNECTION_STRING_ADDITIONAL_PARAMS=$(get_prop 'CONNECTION_STRING_ADDITIONAL_PARAMS' $PROPFILE) db_root_user=$(get_prop 'db_root_user' $PROPFILE) db_root_password=$(get_prop 'db_root_password' $PROPFILE) db_host=$(get_prop 'db_host' $PROPFILE) @@ -605,7 +607,12 @@ update_properties() { if [ "${DB_FLAVOR}" == "MSSQL" ] then propertyName=ranger.ks.jpa.jdbc.url - newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name}" + if [ "${CONNECTION_STRING_ADDITIONAL_PARAMS}" != "" ] + then + newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name};${CONNECTION_STRING_ADDITIONAL_PARAMS}" + else + newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name}" + fi updatePropertyToFilePy $propertyName $newPropertyValue $to_file propertyName=ranger.ks.jpa.jdbc.dialect diff --git a/security-admin/scripts/dba_script.py b/security-admin/scripts/dba_script.py index 5f96ef26d5..8a334de063 100644 --- a/security-admin/scripts/dba_script.py +++ b/security-admin/scripts/dba_script.py @@ -1004,19 +1004,27 @@ def writeDrymodeCmd(self, xa_db_host, audit_db_host, xa_db_root_user, xa_db_root class SqlServerConf(BaseDB): # Constructor - def __init__(self, host, SQL_CONNECTOR_JAR, JAVA_BIN): + def __init__(self, host, SQL_CONNECTOR_JAR, JAVA_BIN, is_db_override_jdbc_connection_string, db_override_jdbc_connection_string): self.host = host self.SQL_CONNECTOR_JAR = SQL_CONNECTOR_JAR self.JAVA_BIN = JAVA_BIN + self.is_db_override_jdbc_connection_string = is_db_override_jdbc_connection_string + self.db_override_jdbc_connection_string = db_override_jdbc_connection_string def get_jisql_cmd(self, user, password, db_name): #TODO: User array for forming command path = RANGER_ADMIN_HOME self.JAVA_BIN = self.JAVA_BIN.strip("'") if is_unix: - jisql_cmd = "%s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -user %s -p '%s' -driver mssql -cstring jdbc:sqlserver://%s\\;databaseName=%s -noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path,user, password, self.host,db_name) + if self.is_db_override_jdbc_connection_string == 'true' and self.db_override_jdbc_connection_string is not None and len(self.db_override_jdbc_connection_string) > 0: + jisql_cmd = "%s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -user %s -p '%s' -driver mssql -cstring %s -noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path, user, password, self.db_override_jdbc_connection_string) + else: + jisql_cmd = "%s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -user %s -p '%s' -driver mssql -cstring jdbc:sqlserver://%s\\;databaseName=%s -noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path,user, password, self.host,db_name) elif os_name == "WINDOWS": - jisql_cmd = "%s -cp %s;%s\\jisql\\lib\\* org.apache.util.sql.Jisql -user %s -p \"%s\" -driver mssql -cstring jdbc:sqlserver://%s;databaseName=%s -noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path, user, password, self.host,db_name) + if self.is_db_override_jdbc_connection_string == 'true' and self.db_override_jdbc_connection_string is not None and len(self.db_override_jdbc_connection_string) > 0: + jisql_cmd = "%s -cp %s;%s\\jisql\\lib\\* org.apache.util.sql.Jisql -user %s -p \"%s\" -driver mssql -cstring %s -noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path, user, password, self.db_override_jdbc_connection_string) + else: + jisql_cmd = "%s -cp %s;%s\\jisql\\lib\\* org.apache.util.sql.Jisql -user %s -p \"%s\" -driver mssql -cstring jdbc:sqlserver://%s;databaseName=%s -noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path, user, password, self.host,db_name) return jisql_cmd def verify_user(self, root_user, db_root_password, db_user,dryMode): @@ -1714,6 +1722,13 @@ def main(argv): log("[E] Invalid ssl keystore password!","error") sys.exit(1) + is_override_db_connection_string='false' + db_override_jdbc_connection_string='' + if 'is_override_db_connection_string' in globalDict: + is_override_db_connection_string=globalDict['is_override_db_connection_string'].lower() + if 'db_override_jdbc_connection_string' in globalDict: + db_override_jdbc_connection_string=globalDict['db_override_jdbc_connection_string'].strip() + if XA_DB_FLAVOR == "MYSQL": MYSQL_CONNECTOR_JAR=CONNECTOR_JAR xa_sqlObj = MysqlConf(xa_db_host, MYSQL_CONNECTOR_JAR, JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type) @@ -1740,7 +1755,7 @@ def main(argv): elif XA_DB_FLAVOR == "MSSQL": SQLSERVER_CONNECTOR_JAR=CONNECTOR_JAR - xa_sqlObj = SqlServerConf(xa_db_host, SQLSERVER_CONNECTOR_JAR, JAVA_BIN) + xa_sqlObj = SqlServerConf(xa_db_host, SQLSERVER_CONNECTOR_JAR, JAVA_BIN, is_override_db_connection_string, db_override_jdbc_connection_string) xa_db_version_file = os.path.join(RANGER_ADMIN_HOME,sqlserver_dbversion_catalog) xa_db_core_file = os.path.join(RANGER_ADMIN_HOME,sqlserver_core_file) xa_patch_file = os.path.join(RANGER_ADMIN_HOME,sqlserver_patches) @@ -1779,7 +1794,7 @@ def main(argv): elif AUDIT_DB_FLAVOR == "MSSQL": SQLSERVER_CONNECTOR_JAR=CONNECTOR_JAR - audit_sqlObj = SqlServerConf(audit_db_host, SQLSERVER_CONNECTOR_JAR, JAVA_BIN) + audit_sqlObj = SqlServerConf(audit_db_host, SQLSERVER_CONNECTOR_JAR, JAVA_BIN, is_override_db_connection_string, db_override_jdbc_connection_string) audit_db_file = os.path.join(RANGER_ADMIN_HOME,sqlserver_audit_file) elif AUDIT_DB_FLAVOR == "SQLA": diff --git a/security-admin/scripts/setup.sh b/security-admin/scripts/setup.sh index b5eec25dd8..1fef99ee45 100755 --- a/security-admin/scripts/setup.sh +++ b/security-admin/scripts/setup.sh @@ -69,8 +69,10 @@ fi LOGFILE=$(eval echo " $(get_prop 'LOGFILE' $PROPFILE)") PYTHON_COMMAND_INVOKER=$(get_prop 'PYTHON_COMMAND_INVOKER' $PROPFILE) + DB_FLAVOR=$(get_prop 'DB_FLAVOR' $PROPFILE) SQL_CONNECTOR_JAR=$(get_prop 'SQL_CONNECTOR_JAR' $PROPFILE) +CONNECTION_STRING_ADDITIONAL_PARAMS=$(get_prop 'CONNECTION_STRING_ADDITIONAL_PARAMS' $PROPFILE) db_root_user=$(get_prop 'db_root_user' $PROPFILE) db_root_password=$(get_prop 'db_root_password' $PROPFILE) db_host=$(get_prop 'db_host' $PROPFILE) @@ -746,7 +748,12 @@ update_properties() { if [ "${DB_FLAVOR}" == "MSSQL" ] then propertyName=ranger.jpa.jdbc.url - newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name}" + if [ "${CONNECTION_STRING_ADDITIONAL_PARAMS}" != "" ] + then + newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name};${CONNECTION_STRING_ADDITIONAL_PARAMS}" + else + newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name}" + fi updatePropertyToFilePy $propertyName $newPropertyValue $to_file_ranger propertyName=ranger.jpa.jdbc.dialect