added test words, remember to delete #490
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #------------------------------------------------------------------------------- | |
| # Copyright 2023-2024 Norconex Inc. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| #------------------------------------------------------------------------------- | |
| # This workflow automates the continuous integration (CI) and | |
| # continuous delivery (CD) process for the Maven project. It includes | |
| # building, testing, and analyzing changes in pull requests, as well | |
| # as deploying snapshot releases to artifact repositories upon | |
| # pushes to the main branch. The workflow ensures that code changes | |
| # are validated and deployed efficiently, maintaining the project's | |
| # quality and reliability. | |
| name: Maven CI/CD Workflow | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - release/* | |
| # Only run this workflow if one or more files were changed and not part of | |
| # 'paths-ignore'. | |
| paths-ignore: | |
| - ./V4_MIGRATION.md | |
| - '**/README.md' | |
| - '**/TODO.txt' | |
| pull_request: | |
| branches: | |
| - '*' | |
| types: [opened, synchronize, reopened] | |
| # Unfortunately GitHub actions does not yet support Yaml anchors and aliases | |
| # so we have to repeat the paths-ignore here. | |
| paths-ignore: | |
| - ./V4_MIGRATION.md | |
| - '**/README.md' | |
| - '**/TODO.txt' | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| # Env. vars. defined here are available to all steps. | |
| env: | |
| SHOULD_DEPLOY: ${{ github.repository == 'Norconex/crawlers' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} | |
| SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} | |
| permissions: | |
| id-token: write | |
| # write is required to publish the dependency graph | |
| contents: write | |
| steps: | |
| - run: | | |
| echo "Should deploy: ${{ env.SHOULD_DEPLOY }}" | |
| #--- Step ------------------------------------------------------------------ | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| # depth 0 means checkout all commits... we need that | |
| # in case there are many commits in a push/PR | |
| fetch-depth: 0 | |
| #--- Step ------------------------------------------------------------------ | |
| - name: Merge main|release into PR branch | |
| if: > | |
| github.event_name == 'pull_request' && ( | |
| github.event.pull_request.base.ref == 'main' || | |
| startsWith(github.event.pull_request.base.ref, 'release/') | |
| ) | |
| run: git merge --ff-only "origin/${{ github.event.pull_request.base.ref }}" | |
| #--- Step ------------------------------------------------------------------ | |
| - name: Cache Maven installation | |
| id: cache-maven | |
| uses: actions/cache@v4 | |
| with: | |
| path: /opt/maven | |
| key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven- | |
| #--- Step ------------------------------------------------------------------ | |
| - name: Install recent Maven | |
| if: ${{ steps.cache-maven.outputs.cache-hit != 'true' }} | |
| run: | | |
| MAVEN_VERSION=3.9.9 | |
| wget https://downloads.apache.org/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz | |
| tar xzvf apache-maven-$MAVEN_VERSION-bin.tar.gz | |
| sudo mv apache-maven-$MAVEN_VERSION /opt/maven | |
| sudo rm -f /usr/bin/mvn | |
| sudo ln -s /opt/maven/bin/mvn /usr/bin/mvn | |
| #--- Step ------------------------------------------------------------------ | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '17' | |
| maven-version: '3.9.9' | |
| # Server ID as in the pom.xml (distributionManagement + sonatype plugin) | |
| server-id: ossrh | |
| # Env variable name for username in deploy | |
| server-username: OSSRH_USERNAME | |
| # Env variable name for token in deploy | |
| server-password: OSSRH_TOKEN | |
| # Value of the GPG private key to import | |
| gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} | |
| # Env variable name for GPG private key passphrase | |
| gpg-passphrase: MAVEN_GPG_PASSPHRASE | |
| - run: mvn --version | |
| #--- Step ------------------------------------------------------------------ | |
| - name: Cache Maven repository | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.m2/repository | |
| !~/.m2/repository/com/norconex | |
| key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} | |
| ${{ runner.os }}-maven- | |
| #--- Step ------------------------------------------------------------------ | |
| - name: Cache SonarCloud analysis | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.sonar/cache | |
| key: ${{ runner.os }}-sonar | |
| restore-keys: ${{ runner.os }}-sonar | |
| #--- Step ------------------------------------------------------------------ | |
| # The shaded libs are dependencies to some other modules so we have | |
| # to install them in local maven repo first. | |
| - name: Build and Install Shaded Libs | |
| run: | | |
| mvn clean install \ | |
| -Dgpg.skip=true \ | |
| -Dmaven.wagon.rss.buffer.size=2048 \ | |
| -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ | |
| -pl shaded | |
| #--- Step ------------------------------------------------------------------ | |
| # Resolving before most mvn commands will fail early if there is an issue. | |
| # Downloaded items won't be downloaded again in this workflow. | |
| - name: Resolve Maven Dependencies | |
| run: | | |
| mvn dependency:resolve -U \ | |
| -Dmaven.wagon.rss.buffer.size=2048 \ | |
| -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn | |
| #--- Step ------------------------------------------------------------------ | |
| - name: Show Maven Dependency Tree | |
| run: mvn dependency:tree -Dverbose | |
| #--- Step ------------------------------------------------------------------ | |
| - name: Find Changed Modules (only if not deploying) | |
| if: ${{ env.SHOULD_DEPLOY == 'false' }} | |
| id: find-modules | |
| run: | | |
| chmod +x .github/workflows/scripts/changed-modules2.sh | |
| ./.github/workflows/scripts/changed-modules2.sh | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| #--- Step ------------------------------------------------------------------ | |
| - name: Build and test | |
| run: | | |
| changed_modules="${{ steps.find-modules.outputs.changed_modules }}" | |
| if [ -n "$changed_modules" ]; then | |
| pl_argument="$changed_modules" | |
| am_argument="-am" | |
| else | |
| pl_argument="!shaded" | |
| am_argument="" | |
| fi | |
| mvn clean -B verify \ | |
| -Dmaven.compiler.maxmem=3072m \ | |
| -DargLine="-Xmx3g" \ | |
| -Dgpg.skip=true \ | |
| -Dmaven.javadoc.skip=true \ | |
| -Dmaven.wagon.rss.buffer.size=2048 \ | |
| -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ | |
| -Pjacoco \ | |
| -pl "$pl_argument" \ | |
| -C ${am_argument} | |
| #--- Step ------------------------------------------------------------------ | |
| # Note: As of this writing, for SonarCloud to work with monorepos, | |
| # each projects must be analyzed separately. | |
| - name: Sonar Analysis | |
| run: | | |
| # Figure out which modules to process | |
| changed_modules="${{ steps.find-modules.outputs.changed_modules }}" | |
| if [ -n "$changed_modules" ]; then | |
| IFS=',' read -r -a modules_to_process <<< "$changed_modules" | |
| else | |
| mapfile -t modules_to_process < <(grep -oP '<module>\K.*?(?=</module>)' pom.xml | grep -v 'shaded') | |
| fi | |
| # Loop through resolved modules and run Sonar analysis | |
| for module in "${modules_to_process[@]}"; do | |
| echo "Running Sonar analysis for module: $module" | |
| cd "$module" || exit | |
| mvn sonar:sonar \ | |
| -Dsonar.host.url=https://sonarcloud.io \ | |
| -Dsonar.verbose=true \ | |
| -Dmaven.wagon.rss.buffer.size=2048 | |
| cd - || exit | |
| done | |
| #--- Step ------------------------------------------------------------------ | |
| # Uploads the full dependency graph to GitHub to improve the quality | |
| # of Dependabot alerts this repository can receive | |
| - name: Update Maven dependency graph | |
| if: ${{ env.SHOULD_DEPLOY == 'true' }} | |
| uses: advanced-security/maven-dependency-submission-action@v4 | |
| #--- Step ------------------------------------------------------------------ | |
| - name: Deploy snapshots to Sonatype | |
| if: ${{ env.SHOULD_DEPLOY == 'true' }} | |
| run: mvn deploy -DskipTests -Dskip.sonar=true -X | |
| env: | |
| OSSRH_USERNAME: ${{ secrets.OSSRH_USERNAME }} | |
| OSSRH_TOKEN: ${{ secrets.OSSRH_TOKEN }} | |
| MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} |