Skip to content

Commit

Permalink
Move master-nonlatent to use common master class
Browse files Browse the repository at this point in the history 
* Also fixup unescaped backslashes in strings
  • Loading branch information
@cvicentiu
cvicentiu committed Jan 12, 2025
1 parent 8ad133a commit 6b4567b
Showing 1 changed file with 36 additions and 115 deletions.
151 changes: 36 additions & 115 deletions master-nonlatent/master.cfg
View file
Original file line number Diff line number Diff line change
@@ -1,27 +1,27 @@
# -*- python -*-
# ex: set filetype=python:

from buildbot.plugins import *
from buildbot.process.properties import Property, Properties
from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCommand
from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver
from buildbot.steps.source.github import GitHub
from buildbot.process.remotecommand import RemoteCommand
from datetime import timedelta
from twisted.internet import defer

import docker
import os
import sys
import yaml

sys.setrecursionlimit(10000)
from datetime import timedelta

sys.path.insert(0, "/srv/buildbot/master")
from common_factories import *
from locks import *
from schedulers_definition import SCHEDULERS
from utils import *
from buildbot.plugins import steps, util, worker
from buildbot.process.properties import Property
from common_factories import addWinTests, getQuickBuildFactory
from constants import MTR_ENV, SAVED_PACKAGE_BRANCHES
from master_common import base_master_config
from utils import (
canStartBuild,
createVar,
getHTMLLogString,
getSourceTarball,
hasFailed,
ls2list,
moveMTRLogs,
mtrJobsMultiplier,
nextBuild,
prioritizeBuilders,
savePackageIfBranchMatch,
)

####### VARIABLES
vsWarningPattern = "^.*: warning C.*$"
Expand All @@ -35,7 +35,7 @@ cfg_dir = os.path.abspath(os.path.dirname(__file__))
#     └── master.cfg
#
# Non autogen masters load from <srcdir> for now.
base_dir = os.path.abspath(f'{cfg_dir}/../')
base_dir = os.path.abspath(f"{cfg_dir}/../")

# Load the slave, database passwords and 3rd-party tokens from an external private file, so
# that the rest of the configuration can be public.
Expand All @@ -45,36 +45,7 @@ with open(os.path.join(base_dir, "master-private.cfg"), "r") as file:

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}


####### PROJECT IDENTITY

# the 'title' string will appear at the top of this buildbot installation's
# home pages (linked to the 'titleURL').
c["title"] = os.environ["TITLE"]
c["titleURL"] = os.environ["TITLE_URL"]

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server is visible. This typically uses the port number set in
# the 'www' entry below, but with an externally-visible host name which the
# buildbot cannot figure out without some help.
c["buildbotURL"] = os.environ["BUILDMASTER_URL"]

# 'protocols' contains information about protocols which master will use for
# communicating with workers. You must define at least 'port' option that workers
# could connect to your master with this protocol.
# 'port' must match the value configured into the workers (with their
# --master option)
port = int(os.environ["PORT"])
c["protocols"] = {"pb": {"port": port}}

####### DB URL

c["db"] = {
# This specifies what database buildbot uses to store its state.
"db_url": config["private"]["db_url"]
}
c = BuildmasterConfig = base_master_config(config)

mtrDbPool = util.EqConnectionPool(
"MySQLdb",
Expand All @@ -84,33 +55,10 @@ mtrDbPool = util.EqConnectionPool(
config["private"]["db_mtr_db"],
)

####### Disable net usage reports from being sent to buildbot.net
c["buildbotNetUsageData"] = None

####### Services
c["services"] = []
context = util.Interpolate("buildbot/%(prop:buildername)s")
gs = reporters.GitHubStatusPush(
token=config["private"]["gh_mdbci"]["access_token"],
context=context,
startDescription="Build started.",
endDescription="Build done.",
verbose=True,
builders=GITHUB_STATUS_BUILDERS,
)
c["services"].append(gs)
c["secretsProviders"] = [
secrets.SecretInAFile(
dirname=os.environ["MASTER_CREDENTIALS_DIR"]
)
]
####### Builder priority
c["prioritizeBuilders"] = prioritizeBuilders

####### SCHEDULERS

# Configure the Schedulers, which decide how to react to incoming changes.
c["schedulers"] = SCHEDULERS

####### WORKERS

Expand Down Expand Up @@ -177,9 +125,6 @@ f_windows_env = {
}
f_windows_env.update(MTR_ENV)

# f_quick_build = getQuickBuildFactory(mtrDbPool)
f_rpm_autobake = getRpmAutobakeFactory(mtrDbPool)

## f_windows
f_windows = util.BuildFactory()
f_windows.addStep(
Expand Down Expand Up @@ -258,7 +203,7 @@ f_windows.addStep(
"dojob",
'"',
util.Interpolate(
'"C:\Program Files (x86)\Microsoft Visual Studio\\2022\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=%(kw:arch)s && cmake . -A %(kw:arch_cmake)s -DPLUGIN_ROCKSDB=NO -DMYSQL_MAINTAINER_MODE=ERR -Wno-dev',
'"C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\BuildTools\\Common7\\Tools\\VsDevCmd.bat" -arch=%(kw:arch)s && cmake . -A %(kw:arch_cmake)s -DPLUGIN_ROCKSDB=NO -DMYSQL_MAINTAINER_MODE=ERR -Wno-dev',
arch=util.Property("arch", default="x64"),
arch_cmake=util.Property("arch_cmake", default="x64"),
),
Expand All @@ -275,7 +220,7 @@ f_windows.addStep(
"dojob",
'"',
util.Interpolate(
'"C:\Program Files (x86)\Microsoft Visual Studio\\2022\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=%(kw:arch)s && cmake --build . --verbose --config Debug -- -m',
'"C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\BuildTools\\Common7\\Tools\\VsDevCmd.bat" -arch=%(kw:arch)s && cmake --build . --verbose --config Debug -- -m',
arch=util.Property("arch", default="x64"),
),
'"',
Expand Down Expand Up @@ -316,14 +261,8 @@ f_windows.addStep(
+ "%(prop:buildername)s"
),
url=util.Interpolate(
f'{os.environ["ARTIFACTS_URL"]}'
"/"
"%(prop:tarbuildnum)s"
"/"
"logs"
"/"
"%(prop:buildername)s"
"/"
os.environ["ARTIFACTS_URL"],
"/" "%(prop:tarbuildnum)s" "/" "logs" "/" "%(prop:buildername)s" "/",
),
)
)
Expand All @@ -336,7 +275,7 @@ f_windows.addStep(
"powershell",
"-command",
"Remove-Item",
'"$pwd\*"',
'"$pwd\\*"',
"-Recurse",
"-Force",
'"',
Expand Down Expand Up @@ -386,7 +325,7 @@ f_windows_msi.addStep(
"powershell",
"-command",
"Remove-Item",
'"$pwd\*"',
'"$pwd\\*"',
"-Recurse",
"-Force",
'"',
Expand Down Expand Up @@ -438,7 +377,7 @@ f_windows_msi.addStep(
"dojob",
'"',
util.Interpolate(
'"C:\Program Files (x86)\Microsoft Visual Studio\\2022\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=%(kw:arch)s && cmake . -G "Visual Studio 17 2022" -A %(kw:arch_cmake)s -DBUILD_CONFIG=mysql_release -DWITH_THIRD_PARTY=HeidiSQL -DWITH_EMBEDDED_SERVER=0 -DSIGNCODE=ON -DSIGNTOOL_PARAMETERS="/fd;SHA256;/a;/t;http://timestamp.globalsign.com/?signature=sha2" -DWITH_UNIT_TESTS=0 -DMYSQL_MAINTAINER_MODE=ERR',
'"C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\BuildTools\\Common7\\Tools\\VsDevCmd.bat" -arch=%(kw:arch)s && cmake . -G "Visual Studio 17 2022" -A %(kw:arch_cmake)s -DBUILD_CONFIG=mysql_release -DWITH_THIRD_PARTY=HeidiSQL -DWITH_EMBEDDED_SERVER=0 -DSIGNCODE=ON -DSIGNTOOL_PARAMETERS="/fd;SHA256;/a;/t;http://timestamp.globalsign.com/?signature=sha2" -DWITH_UNIT_TESTS=0 -DMYSQL_MAINTAINER_MODE=ERR',
arch=util.Property("arch", default="x64"),
arch_cmake=util.Property("arch_cmake", default="x64"),
),
Expand All @@ -455,7 +394,7 @@ f_windows_msi.addStep(
"dojob",
'"',
util.Interpolate(
'"C:\Program Files (x86)\Microsoft Visual Studio\\2022\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=%(kw:arch)s && cmake --build . --verbose --config RelWithDebInfo -- -m',
'"C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\BuildTools\\Common7\\Tools\\VsDevCmd.bat" -arch=%(kw:arch)s && cmake --build . --verbose --config RelWithDebInfo -- -m',
arch=util.Property("arch", default="x64"),
),
'"',
Expand All @@ -472,7 +411,7 @@ f_windows_msi.addStep(
"dojob",
'"',
util.Interpolate(
'"C:\Program Files (x86)\Microsoft Visual Studio\\2022\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=%(kw:arch)s && cmake --build . --config RelWithDebInfo --target win_package && cmake --build . --config RelWithDebInfo --target MSI',
'"C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\BuildTools\\Common7\\Tools\\VsDevCmd.bat" -arch=%(kw:arch)s && cmake --build . --config RelWithDebInfo --target win_package && cmake --build . --config RelWithDebInfo --target MSI',
arch=util.Property("arch", default="x64"),
),
'"',
Expand Down Expand Up @@ -530,7 +469,7 @@ f_windows_msi.addStep(
command=[
"powershell",
"-command",
'Get-ChildItem .\* -Include @("*.msi", "*.zip") | Get-FileHash | Select-Object Hash, @{Name="Name";Expression={[System.IO.Path]::GetFileName($_.Path)}} | Format-Table -HideTableHeaders | Out-File sha256sums.txt',
'Get-ChildItem .\\* -Include @("*.msi", "*.zip") | Get-FileHash | Select-Object Hash, @{Name="Name";Expression={[System.IO.Path]::GetFileName($_.Path)}} | Format-Table -HideTableHeaders | Out-File sha256sums.txt',
],
)
)
Expand Down Expand Up @@ -592,7 +531,7 @@ f_windows_msi.addStep(
"powershell",
"-command",
"Remove-Item",
'"$pwd\*"',
'"$pwd\\*"',
"-Recurse",
"-Force",
'"',
Expand Down Expand Up @@ -707,9 +646,7 @@ f_dockerlibrary.addStep(
f_dockerlibrary.addStep(
steps.ShellCommand(
name="building MariaDB docker library test image",
env={
"ARTIFACTS_URL": os.environ["ARTIFACTS_URL"]
},
env={"ARTIFACTS_URL": os.environ["ARTIFACTS_URL"]},
command=[
"bash",
"-xc",
Expand Down Expand Up @@ -968,7 +905,7 @@ def get_macos_factory(compile_only=False):
util.Interpolate(
"""
cd mysql-test &&
exec perl mysql-test-run.pl --verbose-restart --force --retry=3 --max-save-core=2 --max-save-datadir=10 --max-test-fail=20 --parallel=$(expr %(kw:jobs)s \* 2) %(kw:mtr_additional_args)s
exec perl mysql-test-run.pl --verbose-restart --force --retry=3 --max-save-core=2 --max-save-datadir=10 --max-test-fail=20 --parallel=$(expr %(kw:jobs)s \\* 2) %(kw:mtr_additional_args)s
""",
mtr_additional_args=util.Property(
"mtr_additional_args", default=""
Expand Down Expand Up @@ -1056,9 +993,7 @@ c["builders"].append(
name=os.environ["MASTER_NONLATENT_DOCKERLIBRARY_WORKER"]
.replace("bb", "amd64")
.replace("docker", "wordpress"),
workernames=[
os.environ["MASTER_NONLATENT_DOCKERLIBRARY_WORKER"]
],
workernames=[os.environ["MASTER_NONLATENT_DOCKERLIBRARY_WORKER"]],
tags=["RHEL"],
collapseRequests=True,
nextBuild=nextBuild,
Expand All @@ -1071,9 +1006,7 @@ c["builders"].append(
util.BuilderConfig(
name=os.environ["MASTER_NONLATENT_DOCKERLIBRARY_WORKER"].replace("bb", "amd64")
+ "library",
workernames=[
os.environ["MASTER_NONLATENT_DOCKERLIBRARY_WORKER"]
],
workernames=[os.environ["MASTER_NONLATENT_DOCKERLIBRARY_WORKER"]],
tags=["RHEL"],
collapseRequests=True,
nextBuild=nextBuild,
Expand Down Expand Up @@ -1161,15 +1094,3 @@ c["builders"].append(

# Add a Janitor configurator that removes old logs
c["configurators"] = [util.JanitorConfigurator(logHorizon=timedelta(weeks=6), hour=23)]

c["logEncoding"] = "utf-8"

c["multiMaster"] = True

c["mq"] = { # Need to enable multimaster aware mq. Wamp is the only option for now.
"type": "wamp",
"router_url": os.environ["MQ_ROUTER_URL"],
"realm": "realm1",
# valid are: none, critical, error, warn, info, debug, trace
"wamp_debug_level": "info",
}

0 comments on commit 6b4567b

Please sign in to comment.