diff --git a/master-nonlatent/master.cfg b/master-nonlatent/master.cfg index 0470f0d4..85e07f80 100644 --- a/master-nonlatent/master.cfg +++ b/master-nonlatent/master.cfg @@ -1,27 +1,27 @@ # -*- python -*- # ex: set filetype=python: -from buildbot.plugins import * -from buildbot.process.properties import Property, Properties -from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCommand -from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver -from buildbot.steps.source.github import GitHub -from buildbot.process.remotecommand import RemoteCommand -from datetime import timedelta -from twisted.internet import defer - -import docker import os -import sys -import yaml - -sys.setrecursionlimit(10000) +from datetime import timedelta -sys.path.insert(0, "/srv/buildbot/master") -from common_factories import * -from locks import * -from schedulers_definition import SCHEDULERS -from utils import * +from buildbot.plugins import steps, util, worker +from buildbot.process.properties import Property +from common_factories import addWinTests, getQuickBuildFactory +from constants import MTR_ENV, SAVED_PACKAGE_BRANCHES +from master_common import base_master_config +from utils import ( + canStartBuild, + createVar, + getHTMLLogString, + getSourceTarball, + hasFailed, + ls2list, + moveMTRLogs, + mtrJobsMultiplier, + nextBuild, + prioritizeBuilders, + savePackageIfBranchMatch, +) ####### VARIABLES vsWarningPattern = "^.*: warning C.*$" @@ -35,7 +35,7 @@ cfg_dir = os.path.abspath(os.path.dirname(__file__)) #     └── master.cfg # # Non autogen masters load from for now. -base_dir = os.path.abspath(f'{cfg_dir}/../') +base_dir = os.path.abspath(f"{cfg_dir}/../") # Load the slave, database passwords and 3rd-party tokens from an external private file, so # that the rest of the configuration can be public. @@ -45,36 +45,7 @@ with open(os.path.join(base_dir, "master-private.cfg"), "r") as file: # This is the dictionary that the buildmaster pays attention to. We also use # a shorter alias to save typing. -c = BuildmasterConfig = {} - - -####### PROJECT IDENTITY - -# the 'title' string will appear at the top of this buildbot installation's -# home pages (linked to the 'titleURL'). -c["title"] = os.environ["TITLE"] -c["titleURL"] = os.environ["TITLE_URL"] - -# the 'buildbotURL' string should point to the location where the buildbot's -# internal web server is visible. This typically uses the port number set in -# the 'www' entry below, but with an externally-visible host name which the -# buildbot cannot figure out without some help. -c["buildbotURL"] = os.environ["BUILDMASTER_URL"] - -# 'protocols' contains information about protocols which master will use for -# communicating with workers. You must define at least 'port' option that workers -# could connect to your master with this protocol. -# 'port' must match the value configured into the workers (with their -# --master option) -port = int(os.environ["PORT"]) -c["protocols"] = {"pb": {"port": port}} - -####### DB URL - -c["db"] = { - # This specifies what database buildbot uses to store its state. - "db_url": config["private"]["db_url"] -} +c = BuildmasterConfig = base_master_config(config) mtrDbPool = util.EqConnectionPool( "MySQLdb", @@ -84,33 +55,10 @@ mtrDbPool = util.EqConnectionPool( config["private"]["db_mtr_db"], ) -####### Disable net usage reports from being sent to buildbot.net -c["buildbotNetUsageData"] = None -####### Services -c["services"] = [] -context = util.Interpolate("buildbot/%(prop:buildername)s") -gs = reporters.GitHubStatusPush( - token=config["private"]["gh_mdbci"]["access_token"], - context=context, - startDescription="Build started.", - endDescription="Build done.", - verbose=True, - builders=GITHUB_STATUS_BUILDERS, -) -c["services"].append(gs) -c["secretsProviders"] = [ - secrets.SecretInAFile( - dirname=os.environ["MASTER_CREDENTIALS_DIR"] - ) -] ####### Builder priority c["prioritizeBuilders"] = prioritizeBuilders -####### SCHEDULERS - -# Configure the Schedulers, which decide how to react to incoming changes. -c["schedulers"] = SCHEDULERS ####### WORKERS @@ -177,9 +125,6 @@ f_windows_env = { } f_windows_env.update(MTR_ENV) -# f_quick_build = getQuickBuildFactory(mtrDbPool) -f_rpm_autobake = getRpmAutobakeFactory(mtrDbPool) - ## f_windows f_windows = util.BuildFactory() f_windows.addStep( @@ -258,7 +203,7 @@ f_windows.addStep( "dojob", '"', util.Interpolate( - '"C:\Program Files (x86)\Microsoft Visual Studio\\2022\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=%(kw:arch)s && cmake . -A %(kw:arch_cmake)s -DPLUGIN_ROCKSDB=NO -DMYSQL_MAINTAINER_MODE=ERR -Wno-dev', + '"C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\BuildTools\\Common7\\Tools\\VsDevCmd.bat" -arch=%(kw:arch)s && cmake . -A %(kw:arch_cmake)s -DPLUGIN_ROCKSDB=NO -DMYSQL_MAINTAINER_MODE=ERR -Wno-dev', arch=util.Property("arch", default="x64"), arch_cmake=util.Property("arch_cmake", default="x64"), ), @@ -275,7 +220,7 @@ f_windows.addStep( "dojob", '"', util.Interpolate( - '"C:\Program Files (x86)\Microsoft Visual Studio\\2022\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=%(kw:arch)s && cmake --build . --verbose --config Debug -- -m', + '"C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\BuildTools\\Common7\\Tools\\VsDevCmd.bat" -arch=%(kw:arch)s && cmake --build . --verbose --config Debug -- -m', arch=util.Property("arch", default="x64"), ), '"', @@ -316,14 +261,8 @@ f_windows.addStep( + "%(prop:buildername)s" ), url=util.Interpolate( - f'{os.environ["ARTIFACTS_URL"]}' - "/" - "%(prop:tarbuildnum)s" - "/" - "logs" - "/" - "%(prop:buildername)s" - "/" + os.environ["ARTIFACTS_URL"], + "/" "%(prop:tarbuildnum)s" "/" "logs" "/" "%(prop:buildername)s" "/", ), ) ) @@ -336,7 +275,7 @@ f_windows.addStep( "powershell", "-command", "Remove-Item", - '"$pwd\*"', + '"$pwd\\*"', "-Recurse", "-Force", '"', @@ -386,7 +325,7 @@ f_windows_msi.addStep( "powershell", "-command", "Remove-Item", - '"$pwd\*"', + '"$pwd\\*"', "-Recurse", "-Force", '"', @@ -438,7 +377,7 @@ f_windows_msi.addStep( "dojob", '"', util.Interpolate( - '"C:\Program Files (x86)\Microsoft Visual Studio\\2022\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=%(kw:arch)s && cmake . -G "Visual Studio 17 2022" -A %(kw:arch_cmake)s -DBUILD_CONFIG=mysql_release -DWITH_THIRD_PARTY=HeidiSQL -DWITH_EMBEDDED_SERVER=0 -DSIGNCODE=ON -DSIGNTOOL_PARAMETERS="/fd;SHA256;/a;/t;http://timestamp.globalsign.com/?signature=sha2" -DWITH_UNIT_TESTS=0 -DMYSQL_MAINTAINER_MODE=ERR', + '"C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\BuildTools\\Common7\\Tools\\VsDevCmd.bat" -arch=%(kw:arch)s && cmake . -G "Visual Studio 17 2022" -A %(kw:arch_cmake)s -DBUILD_CONFIG=mysql_release -DWITH_THIRD_PARTY=HeidiSQL -DWITH_EMBEDDED_SERVER=0 -DSIGNCODE=ON -DSIGNTOOL_PARAMETERS="/fd;SHA256;/a;/t;http://timestamp.globalsign.com/?signature=sha2" -DWITH_UNIT_TESTS=0 -DMYSQL_MAINTAINER_MODE=ERR', arch=util.Property("arch", default="x64"), arch_cmake=util.Property("arch_cmake", default="x64"), ), @@ -455,7 +394,7 @@ f_windows_msi.addStep( "dojob", '"', util.Interpolate( - '"C:\Program Files (x86)\Microsoft Visual Studio\\2022\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=%(kw:arch)s && cmake --build . --verbose --config RelWithDebInfo -- -m', + '"C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\BuildTools\\Common7\\Tools\\VsDevCmd.bat" -arch=%(kw:arch)s && cmake --build . --verbose --config RelWithDebInfo -- -m', arch=util.Property("arch", default="x64"), ), '"', @@ -472,7 +411,7 @@ f_windows_msi.addStep( "dojob", '"', util.Interpolate( - '"C:\Program Files (x86)\Microsoft Visual Studio\\2022\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=%(kw:arch)s && cmake --build . --config RelWithDebInfo --target win_package && cmake --build . --config RelWithDebInfo --target MSI', + '"C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\BuildTools\\Common7\\Tools\\VsDevCmd.bat" -arch=%(kw:arch)s && cmake --build . --config RelWithDebInfo --target win_package && cmake --build . --config RelWithDebInfo --target MSI', arch=util.Property("arch", default="x64"), ), '"', @@ -530,7 +469,7 @@ f_windows_msi.addStep( command=[ "powershell", "-command", - 'Get-ChildItem .\* -Include @("*.msi", "*.zip") | Get-FileHash | Select-Object Hash, @{Name="Name";Expression={[System.IO.Path]::GetFileName($_.Path)}} | Format-Table -HideTableHeaders | Out-File sha256sums.txt', + 'Get-ChildItem .\\* -Include @("*.msi", "*.zip") | Get-FileHash | Select-Object Hash, @{Name="Name";Expression={[System.IO.Path]::GetFileName($_.Path)}} | Format-Table -HideTableHeaders | Out-File sha256sums.txt', ], ) ) @@ -592,7 +531,7 @@ f_windows_msi.addStep( "powershell", "-command", "Remove-Item", - '"$pwd\*"', + '"$pwd\\*"', "-Recurse", "-Force", '"', @@ -707,9 +646,7 @@ f_dockerlibrary.addStep( f_dockerlibrary.addStep( steps.ShellCommand( name="building MariaDB docker library test image", - env={ - "ARTIFACTS_URL": os.environ["ARTIFACTS_URL"] - }, + env={"ARTIFACTS_URL": os.environ["ARTIFACTS_URL"]}, command=[ "bash", "-xc", @@ -968,7 +905,7 @@ def get_macos_factory(compile_only=False): util.Interpolate( """ cd mysql-test && - exec perl mysql-test-run.pl --verbose-restart --force --retry=3 --max-save-core=2 --max-save-datadir=10 --max-test-fail=20 --parallel=$(expr %(kw:jobs)s \* 2) %(kw:mtr_additional_args)s + exec perl mysql-test-run.pl --verbose-restart --force --retry=3 --max-save-core=2 --max-save-datadir=10 --max-test-fail=20 --parallel=$(expr %(kw:jobs)s \\* 2) %(kw:mtr_additional_args)s """, mtr_additional_args=util.Property( "mtr_additional_args", default="" @@ -1056,9 +993,7 @@ c["builders"].append( name=os.environ["MASTER_NONLATENT_DOCKERLIBRARY_WORKER"] .replace("bb", "amd64") .replace("docker", "wordpress"), - workernames=[ - os.environ["MASTER_NONLATENT_DOCKERLIBRARY_WORKER"] - ], + workernames=[os.environ["MASTER_NONLATENT_DOCKERLIBRARY_WORKER"]], tags=["RHEL"], collapseRequests=True, nextBuild=nextBuild, @@ -1071,9 +1006,7 @@ c["builders"].append( util.BuilderConfig( name=os.environ["MASTER_NONLATENT_DOCKERLIBRARY_WORKER"].replace("bb", "amd64") + "library", - workernames=[ - os.environ["MASTER_NONLATENT_DOCKERLIBRARY_WORKER"] - ], + workernames=[os.environ["MASTER_NONLATENT_DOCKERLIBRARY_WORKER"]], tags=["RHEL"], collapseRequests=True, nextBuild=nextBuild, @@ -1161,15 +1094,3 @@ c["builders"].append( # Add a Janitor configurator that removes old logs c["configurators"] = [util.JanitorConfigurator(logHorizon=timedelta(weeks=6), hour=23)] - -c["logEncoding"] = "utf-8" - -c["multiMaster"] = True - -c["mq"] = { # Need to enable multimaster aware mq. Wamp is the only option for now. - "type": "wamp", - "router_url": os.environ["MQ_ROUTER_URL"], - "realm": "realm1", - # valid are: none, critical, error, warn, info, debug, trace - "wamp_debug_level": "info", -}