From 58309662a87a2a751bb61c57d4e9833357816ecb Mon Sep 17 00:00:00 2001 From: Gregory Kopels Date: Fri, 17 Jan 2025 12:08:46 +0200 Subject: [PATCH] cnf:network-add-nftables-test-cases --- tests/cnf/core/network/internal/cmd/cmd.go | 30 ++ tests/cnf/core/network/internal/define/nad.go | 35 +- .../core/network/internal/frrconfig/consts.go | 90 +++++ .../network/internal/frrconfig/frrconfig.go | 26 ++ .../core/network/internal/netenv/netenv.go | 82 ++++ .../cnf/core/network/internal/netenv/sriov.go | 18 - .../core/network/metallb/internal/frr/frr.go | 33 -- .../metallb/internal/tsparams/consts.go | 84 ---- .../core/network/metallb/tests/bfd-test.go | 23 +- .../metallb/tests/bgp-connect-time-tests.go | 12 +- .../metallb/tests/bgp-gracefulrestart.go | 16 +- .../metallb/tests/bgp-remote-as-dynamic.go | 47 ++- .../core/network/metallb/tests/bgp-tests.go | 16 +- .../cnf/core/network/metallb/tests/common.go | 34 +- .../core/network/metallb/tests/frrk8-tests.go | 134 ++++--- .../core/network/metallb/tests/layer2-test.go | 4 +- .../security/internal/tsparams/consts.go | 27 ++ .../internal/tsparams/securityvars.go | 24 ++ .../network/security/security_suite_test.go | 59 +++ .../tests/nftables-custom-firewall-test.go | 364 ++++++++++++++++++ 20 files changed, 881 insertions(+), 277 deletions(-) create mode 100644 tests/cnf/core/network/internal/frrconfig/consts.go create mode 100644 tests/cnf/core/network/internal/frrconfig/frrconfig.go create mode 100644 tests/cnf/core/network/security/internal/tsparams/consts.go create mode 100644 tests/cnf/core/network/security/internal/tsparams/securityvars.go create mode 100644 tests/cnf/core/network/security/security_suite_test.go create mode 100644 tests/cnf/core/network/security/tests/nftables-custom-firewall-test.go diff --git a/tests/cnf/core/network/internal/cmd/cmd.go b/tests/cnf/core/network/internal/cmd/cmd.go index d4cbed2a4..ab848994d 100644 --- a/tests/cnf/core/network/internal/cmd/cmd.go +++ b/tests/cnf/core/network/internal/cmd/cmd.go @@ -9,6 +9,7 @@ import ( "github.com/golang/glog" "github.com/openshift-kni/eco-goinfra/pkg/pod" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/ipaddr" . "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netinittools" "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netparam" ) @@ -114,6 +115,23 @@ func RxTrafficOnClientPod(clientPod *pod.Builder, clientRxCmd string) error { return nil } +// ValidateTCPTraffic runs the testcmd with tcp and specified interface, port and destination. +// The receiving client needs to be listening to the specified port. +func ValidateTCPTraffic(clientPod *pod.Builder, destIPAddrs []string, interfaceName, + containerName string, portNum int) error { + for _, destIPAddr := range RemovePrefixFromIPList(destIPAddrs) { + glog.V(90).Infof("Validate tcp traffic to %d to destination server IP %s", portNum, destIPAddr) + + command := fmt.Sprintf("testcmd -interface %s -protocol tcp -port %d -server %s", interfaceName, + portNum, destIPAddr) + _, err := clientPod.ExecCommand([]string{"bash", "-c", command}, containerName) + + return err + } + + return nil +} + // checkRxOnly checks the number of incoming packets. func checkRxOnly(out string) bool { lines := strings.Split(out, "\n") @@ -160,3 +178,15 @@ func getNumberOfPackets(line, firstFieldSubstr string) int { return numberOfPackets } + +// RemovePrefixFromIPList removes the prefix from a list of IP addresses with prefixes. +func RemovePrefixFromIPList(ipAddressList []string) []string { + var ipAddressListWithoutPrefix []string + + for _, ipaddress := range ipAddressList { + glog.V(90).Infof("Remove the network prefix from IP address %s", ipaddress) + ipAddressListWithoutPrefix = append(ipAddressListWithoutPrefix, ipaddr.RemovePrefix(ipaddress)) + } + + return ipAddressListWithoutPrefix +} diff --git a/tests/cnf/core/network/internal/define/nad.go b/tests/cnf/core/network/internal/define/nad.go index d44742741..15b051c1a 100644 --- a/tests/cnf/core/network/internal/define/nad.go +++ b/tests/cnf/core/network/internal/define/nad.go @@ -4,6 +4,7 @@ import ( "github.com/golang/glog" "github.com/openshift-kni/eco-goinfra/pkg/clients" "github.com/openshift-kni/eco-goinfra/pkg/nad" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/internal/coreparams" ) // MasterNadPlugin sets NetworkAttachmentDefinition master plugin based on given input. @@ -54,7 +55,7 @@ func MacVlanNad(apiClient *clients.Settings, name, nsName, intName string, ipam return nil, err } - return createNadWithMasterPlugin(apiClient, name, nsName, masterPlugin) + return CreateNadWithMasterPlugin(apiClient, name, nsName, masterPlugin) } // VlanNad defines and creates Vlan NetworkAttachmentDefinition on a cluster. @@ -67,7 +68,7 @@ func VlanNad( return nil, err } - return createNadWithMasterPlugin(apiClient, name, nsName, masterPlugin) + return CreateNadWithMasterPlugin(apiClient, name, nsName, masterPlugin) } // IPVlanNad defines and creates IP-Vlan NetworkAttachmentDefinition on a cluster. @@ -78,10 +79,11 @@ func IPVlanNad(apiClient *clients.Settings, name, nsName, intName string, ipam * return nil, err } - return createNadWithMasterPlugin(apiClient, name, nsName, masterPlugin) + return CreateNadWithMasterPlugin(apiClient, name, nsName, masterPlugin) } -func createNadWithMasterPlugin( +// CreateNadWithMasterPlugin creates a MasterPlugin network attachment definition. +func CreateNadWithMasterPlugin( apiClient *clients.Settings, name, nsName string, masterPlugin *nad.MasterPlugin) (*nad.Builder, error) { createdNad, err := nad.NewBuilder(apiClient, name, nsName).WithMasterPlugin(masterPlugin).Create() if err != nil { @@ -90,3 +92,28 @@ func createNadWithMasterPlugin( return createdNad, nil } + +// CreateExternalNad creats an external network-attchment-definition using the br-ex interface. +func CreateExternalNad(apiClient *clients.Settings, name, testNameSpace string) error { + glog.V(90).Info("Creating external BR-EX NetworkAttachmentDefinition") + + // Define the master NAD plugin + macVlanPlugin, err := MasterNadPlugin(coreparams.OvnExternalBridge, "bridge", nad.IPAMStatic()) + if err != nil { + glog.V(90).Infof("Failed to define master NAD plugin: %v", err) + + return err + } + + // Create the NetworkAttachmentDefinition + _, err = CreateNadWithMasterPlugin(apiClient, name, testNameSpace, macVlanPlugin) + if err != nil { + glog.V(90).Infof("Failed to create external NetworkAttachmentDefinition: %v", err) + + return err + } + + glog.V(90).Infof("Successfully created external NetworkAttachmentDefinition: %s", name) + + return nil +} diff --git a/tests/cnf/core/network/internal/frrconfig/consts.go b/tests/cnf/core/network/internal/frrconfig/consts.go new file mode 100644 index 000000000..95c75395d --- /dev/null +++ b/tests/cnf/core/network/internal/frrconfig/consts.go @@ -0,0 +1,90 @@ +package frrconfig + +const ( + // ContainerName is the container name running in the frr pod. + ContainerName = "frr" + // DaemonsFile represents FRR default daemon configuration template. + DaemonsFile = ` + # This file tells the frr package which daemons to start. + # + # Sample configurations for these daemons can be found in + # /usr/share/doc/frr/examples/. + # + # ATTENTION: + # + # When activating a daemon for the first time, a config file, even if it is + # empty, has to be present *and* be owned by the user and group "frr", else + # the daemon will not be started by /etc/init.d/frr. The permissions should + # be u=rw,g=r,o=. + # When using "vtysh" such a config file is also needed. It should be owned by + # group "frrvty" and set to ug=rw,o= though. Check /etc/pam.d/frr, too. + # + # The watchfrr, zebra and staticd daemons are always started. + # + bgpd=yes + ospfd=no + ospf6d=no + ripd=no + ripngd=no + isisd=no + pimd=no + ldpd=no + nhrpd=no + eigrpd=no + babeld=no + sharpd=no + pbrd=no + bfdd=yes + fabricd=no + vrrpd=no + pathd=no + # + # If this option is set the /etc/init.d/frr script automatically loads + # the config via "vtysh -b" when the servers are started. + # Check /etc/pam.d/frr if you intend to use "vtysh"! + # + vtysh_enable=yes + zebra_options=" -A 127.0.0.1 -s 90000000" + bgpd_options=" -A 127.0.0.1" + ospfd_options=" -A 127.0.0.1" + ospf6d_options=" -A ::1" + ripd_options=" -A 127.0.0.1" + ripngd_options=" -A ::1" + isisd_options=" -A 127.0.0.1" + pimd_options=" -A 127.0.0.1" + ldpd_options=" -A 127.0.0.1" + nhrpd_options=" -A 127.0.0.1" + eigrpd_options=" -A 127.0.0.1" + babeld_options=" -A 127.0.0.1" + sharpd_options=" -A 127.0.0.1" + pbrd_options=" -A 127.0.0.1" + staticd_options="-A 127.0.0.1" + bfdd_options=" -A 127.0.0.1" + fabricd_options="-A 127.0.0.1" + vrrpd_options=" -A 127.0.0.1" + pathd_options=" -A 127.0.0.1" + # configuration profile + # + #frr_profile="traditional" + #frr_profile="datacenter" + # + # This is the maximum number of FD's that will be available. + # Upon startup this is read by the control files and ulimit + # is called. Uncomment and use a reasonable value for your + # setup if you are expecting a large number of peers in + # say BGP. + #MAX_FDS=1024 + # The list of daemons to watch is automatically generated by the init script. + #watchfrr_options="" + # To make watchfrr create/join the specified netns, use the following option: + #watchfrr_options="--netns" + # This only has an effect in /etc/frr//daemons, and you need to + # start FRR with "/usr/lib/frr/frrinit.sh start ". + # for debugging purposes, you can specify a "wrap" command to start instead + # of starting the daemon directly, e.g. to use valgrind on ospfd: + # ospfd_wrap="/usr/bin/valgrind" + # or you can use "all_wrap" for all daemons, e.g. to use perf record: + # all_wrap="/usr/bin/perf record --call-graph -" + # the normal daemon command is added to this at the end. + ` +) diff --git a/tests/cnf/core/network/internal/frrconfig/frrconfig.go b/tests/cnf/core/network/internal/frrconfig/frrconfig.go new file mode 100644 index 000000000..26a8b7638 --- /dev/null +++ b/tests/cnf/core/network/internal/frrconfig/frrconfig.go @@ -0,0 +1,26 @@ +package frrconfig + +import ( + "github.com/openshift-kni/eco-goinfra/pkg/pod" + "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types" +) + +// DefineBaseConfig creates a map of strings for the frr configuration. +func DefineBaseConfig(daemonsConfig, frrConfig, vtyShConfig string) map[string]string { + configMapData := make(map[string]string) + configMapData["daemons"] = daemonsConfig + configMapData["frr.conf"] = frrConfig + configMapData["vtysh.conf"] = vtyShConfig + + return configMapData +} + +// CreateStaticIPAnnotations creates a static ip annotation used together with the nad in a pod for IP configuration. +func CreateStaticIPAnnotations(internalNADName, externalNADName string, internalIPAddresses, + externalIPAddresses []string) []*types.NetworkSelectionElement { + ipAnnotation := pod.StaticIPAnnotation(internalNADName, internalIPAddresses) + ipAnnotation = append(ipAnnotation, + pod.StaticIPAnnotation(externalNADName, externalIPAddresses)...) + + return ipAnnotation +} diff --git a/tests/cnf/core/network/internal/netenv/netenv.go b/tests/cnf/core/network/internal/netenv/netenv.go index cede5d458..0fb9520c2 100644 --- a/tests/cnf/core/network/internal/netenv/netenv.go +++ b/tests/cnf/core/network/internal/netenv/netenv.go @@ -3,6 +3,7 @@ package netenv import ( "encoding/json" "fmt" + "strings" "time" "github.com/golang/glog" @@ -234,3 +235,84 @@ func RemoveAllPoliciesAndWaitForSriovAndMCPStable() error { netinittools.APIClient, netparam.MCOWaitTimeout, time.Minute, netinittools.NetConfig.CnfMcpLabel, netinittools.NetConfig.SriovOperatorNamespace) } + +// BuildRoutesMapWithSpecificRoutes creates a route map with specific routes. +func BuildRoutesMapWithSpecificRoutes(podList []*pod.Builder, workerNodeList []*nodes.Builder, + nextHopList []string) (map[string]string, error) { + if len(podList) == 0 { + glog.V(90).Infof("Pod list is empty") + + return nil, fmt.Errorf("pod list is empty") + } + + if len(nextHopList) == 0 { + glog.V(90).Infof("Nexthop IP addresses list is empty") + + return nil, fmt.Errorf("nexthop IP addresses list is empty") + } + + if len(nextHopList) < len(podList) { + glog.V(90).Infof("Number of speaker IP addresses[%d] is less than the number of pods[%d]", + len(nextHopList), len(podList)) + + return nil, fmt.Errorf("insufficient speaker IP addresses: got %d, need at least %d", + len(nextHopList), len(podList)) + } + + routesMap := make(map[string]string) + + for _, frrPod := range podList { + if frrPod.Definition.Spec.NodeName == workerNodeList[0].Definition.Name { + routesMap[frrPod.Definition.Spec.NodeName] = nextHopList[1] + } else { + routesMap[frrPod.Definition.Spec.NodeName] = nextHopList[0] + } + } + + return routesMap, nil +} + +// SetStaticRoute could set or delete static route on all Speaker pods. +func SetStaticRoute(frrPod *pod.Builder, action, destIP, containerName string, + nextHopMap map[string]string) (string, error) { + buffer, err := frrPod.ExecCommand( + []string{"ip", "route", action, destIP, "via", nextHopMap[frrPod.Definition.Spec.NodeName]}, containerName) + if err != nil { + if strings.Contains(buffer.String(), "File exists") { + glog.V(90).Infof("Warning: Route to %s already exist", destIP) + + return buffer.String(), nil + } + + if strings.Contains(buffer.String(), "No such process") { + glog.V(90).Infof("Warning: Route to %s already absent", destIP) + + return buffer.String(), nil + } + + return buffer.String(), err + } + + return buffer.String(), nil +} + +// WaitForMcpStable waits for the stability of the MCP with the given name. +func WaitForMcpStable(apiClient *clients.Settings, waitingTime, stableDuration time.Duration, mcpName string) error { + mcp, err := mco.Pull(apiClient, mcpName) + + glog.V(90).Info("Waiting for mcp to be stable") + + if err != nil { + return fmt.Errorf("fail to pull mcp %s from cluster due to: %s", mcpName, err.Error()) + } + + err = mcp.WaitToBeStableFor(stableDuration, waitingTime) + + if err != nil { + glog.V(90).Infof("Failed to wait for mcp to become stable %v", err) + + return err + } + + return nil +} diff --git a/tests/cnf/core/network/internal/netenv/sriov.go b/tests/cnf/core/network/internal/netenv/sriov.go index 4624ba1d5..aaba41b5c 100644 --- a/tests/cnf/core/network/internal/netenv/sriov.go +++ b/tests/cnf/core/network/internal/netenv/sriov.go @@ -6,7 +6,6 @@ import ( "github.com/golang/glog" "github.com/openshift-kni/eco-goinfra/pkg/clients" - "github.com/openshift-kni/eco-goinfra/pkg/mco" "github.com/openshift-kni/eco-goinfra/pkg/sriov" ) @@ -50,20 +49,3 @@ func WaitForSriovStable(apiClient *clients.Settings, waitingTime time.Duration, return nil } - -// WaitForMcpStable waits for the stability of the MCP with the given name. -func WaitForMcpStable(apiClient *clients.Settings, waitingTime, stableDuration time.Duration, mcpName string) error { - mcp, err := mco.Pull(apiClient, mcpName) - - if err != nil { - return fmt.Errorf("fail to pull mcp %s from cluster due to: %s", mcpName, err.Error()) - } - - err = mcp.WaitToBeStableFor(stableDuration, waitingTime) - - if err != nil { - return fmt.Errorf("cluster is not stable: %s", err.Error()) - } - - return nil -} diff --git a/tests/cnf/core/network/metallb/internal/frr/frr.go b/tests/cnf/core/network/metallb/internal/frr/frr.go index 5f80e1ec8..854b02bc4 100644 --- a/tests/cnf/core/network/metallb/internal/frr/frr.go +++ b/tests/cnf/core/network/metallb/internal/frr/frr.go @@ -119,16 +119,6 @@ type ( BGPNeighborGRStatus map[string]GRStatus ) -// DefineBaseConfig defines minimal required FRR configuration. -func DefineBaseConfig(daemonsConfig, frrConfig, vtyShConfig string) map[string]string { - configMapData := make(map[string]string) - configMapData["daemons"] = daemonsConfig - configMapData["frr.conf"] = frrConfig - configMapData["vtysh.conf"] = vtyShConfig - - return configMapData -} - // DefineBGPConfig returns string which represents BGP config file peering to all given IP addresses. func DefineBGPConfig(localBGPASN, remoteBGPASN int, neighborsIPAddresses []string, multiHop, bfd bool) string { bgpConfig := tsparams.FRRBaseConfig + @@ -272,29 +262,6 @@ func GetMetricsByPrefix(frrPod *pod.Builder, metricPrefix string) ([]string, err return collectedMetrics, nil } -// SetStaticRoute could set or delete static route on all Speaker pods. -func SetStaticRoute(frrPod *pod.Builder, action, destIP string, nextHopMap map[string]string) (string, error) { - buffer, err := frrPod.ExecCommand( - []string{"ip", "route", action, destIP, "via", nextHopMap[frrPod.Definition.Spec.NodeName]}, "frr") - if err != nil { - if strings.Contains(buffer.String(), "File exists") { - glog.V(90).Infof("Warning: Route to %s already exist", destIP) - - return buffer.String(), nil - } - - if strings.Contains(buffer.String(), "No such process") { - glog.V(90).Infof("Warning: Route to %s already absent", destIP) - - return buffer.String(), nil - } - - return buffer.String(), err - } - - return buffer.String(), nil -} - // GetBGPStatus returns bgp status output from frr pod. func GetBGPStatus(frrPod *pod.Builder, protocolVersion string) (*bgpStatus, error) { glog.V(90).Infof("Getting bgp status from pod: %s", frrPod.Definition.Name) diff --git a/tests/cnf/core/network/metallb/internal/tsparams/consts.go b/tests/cnf/core/network/metallb/internal/tsparams/consts.go index 64887b8e2..d8f141160 100644 --- a/tests/cnf/core/network/metallb/internal/tsparams/consts.go +++ b/tests/cnf/core/network/metallb/internal/tsparams/consts.go @@ -20,90 +20,6 @@ const ( // MlbAddressListError an error message when the ECO_CNF_CORE_NET_MLB_ADDR_LIST is incorrect. MlbAddressListError = "An unexpected error occurred while " + "determining the IP addresses from the ECO_CNF_CORE_NET_MLB_ADDR_LIST environment variable." - // DaemonsFile represents FRR default daemon configuration template. - DaemonsFile = ` - # This file tells the frr package which daemons to start. - # - # Sample configurations for these daemons can be found in - # /usr/share/doc/frr/examples/. - # - # ATTENTION: - # - # When activating a daemon for the first time, a config file, even if it is - # empty, has to be present *and* be owned by the user and group "frr", else - # the daemon will not be started by /etc/init.d/frr. The permissions should - # be u=rw,g=r,o=. - # When using "vtysh" such a config file is also needed. It should be owned by - # group "frrvty" and set to ug=rw,o= though. Check /etc/pam.d/frr, too. - # - # The watchfrr, zebra and staticd daemons are always started. - # - bgpd=yes - ospfd=no - ospf6d=no - ripd=no - ripngd=no - isisd=no - pimd=no - ldpd=no - nhrpd=no - eigrpd=no - babeld=no - sharpd=no - pbrd=no - bfdd=yes - fabricd=no - vrrpd=no - pathd=no - # - # If this option is set the /etc/init.d/frr script automatically loads - # the config via "vtysh -b" when the servers are started. - # Check /etc/pam.d/frr if you intend to use "vtysh"! - # - vtysh_enable=yes - zebra_options=" -A 127.0.0.1 -s 90000000" - bgpd_options=" -A 127.0.0.1" - ospfd_options=" -A 127.0.0.1" - ospf6d_options=" -A ::1" - ripd_options=" -A 127.0.0.1" - ripngd_options=" -A ::1" - isisd_options=" -A 127.0.0.1" - pimd_options=" -A 127.0.0.1" - ldpd_options=" -A 127.0.0.1" - nhrpd_options=" -A 127.0.0.1" - eigrpd_options=" -A 127.0.0.1" - babeld_options=" -A 127.0.0.1" - sharpd_options=" -A 127.0.0.1" - pbrd_options=" -A 127.0.0.1" - staticd_options="-A 127.0.0.1" - bfdd_options=" -A 127.0.0.1" - fabricd_options="-A 127.0.0.1" - vrrpd_options=" -A 127.0.0.1" - pathd_options=" -A 127.0.0.1" - # configuration profile - # - #frr_profile="traditional" - #frr_profile="datacenter" - # - # This is the maximum number of FD's that will be available. - # Upon startup this is read by the control files and ulimit - # is called. Uncomment and use a reasonable value for your - # setup if you are expecting a large number of peers in - # say BGP. - #MAX_FDS=1024 - # The list of daemons to watch is automatically generated by the init script. - #watchfrr_options="" - # To make watchfrr create/join the specified netns, use the following option: - #watchfrr_options="--netns" - # This only has an effect in /etc/frr//daemons, and you need to - # start FRR with "/usr/lib/frr/frrinit.sh start ". - # for debugging purposes, you can specify a "wrap" command to start instead - # of starting the daemon directly, e.g. to use valgrind on ospfd: - # ospfd_wrap="/usr/bin/valgrind" - # or you can use "all_wrap" for all daemons, e.g. to use perf record: - # all_wrap="/usr/bin/perf record --call-graph -" - # the normal daemon command is added to this at the end. - ` // FRRBaseConfig represents FRR daemon minimal configuration. FRRBaseConfig = `! frr defaults traditional diff --git a/tests/cnf/core/network/metallb/tests/bfd-test.go b/tests/cnf/core/network/metallb/tests/bfd-test.go index 2b552358d..2a2e51413 100644 --- a/tests/cnf/core/network/metallb/tests/bfd-test.go +++ b/tests/cnf/core/network/metallb/tests/bfd-test.go @@ -17,6 +17,9 @@ import ( "github.com/openshift-kni/eco-goinfra/pkg/pod" "github.com/openshift-kni/eco-goinfra/pkg/reportxml" "github.com/openshift-kni/eco-goinfra/pkg/service" + netcmd "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/cmd" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/define" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/frrconfig" "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/ipaddr" "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netenv" . "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netinittools" @@ -59,7 +62,8 @@ var _ = Describe("BFD", Ordered, Label(tsparams.LabelBFDTestCases), ContinueOnFa err = metallbenv.IsEnvVarMetalLbIPinNodeExtNetRange(ipv4NodeAddrList, ipv4metalLbIPList, nil) Expect(err).ToNot(HaveOccurred(), "Failed to validate metalLb exported ip address") - createExternalNad(tsparams.ExternalMacVlanNADName) + err = define.CreateExternalNad(APIClient, tsparams.ExternalMacVlanNADName, tsparams.TestNamespaceName) + Expect(err).ToNot(HaveOccurred(), "Failed to create a network-attachment-definition") }) Context("single hop", Label("singlehop"), func() { @@ -80,7 +84,7 @@ var _ = Describe("BFD", Ordered, Label(tsparams.LabelBFDTestCases), ContinueOnFa By("Creating static ip annotation") staticIPAnnotation := pod.StaticIPAnnotation( - externalNad.Definition.Name, []string{fmt.Sprintf("%s/24", ipv4metalLbIPList[0])}) + tsparams.ExternalMacVlanNADName, []string{fmt.Sprintf("%s/24", ipv4metalLbIPList[0])}) By("Listing control-plane nodes") masterNodeList, err := nodes.List(APIClient, @@ -185,7 +189,8 @@ var _ = Describe("BFD", Ordered, Label(tsparams.LabelBFDTestCases), ContinueOnFa }) Expect(err).ToNot(HaveOccurred(), "Failed to list pods") for _, frrk8sPod := range frrk8sPods { - out, err := frr.SetStaticRoute(frrk8sPod, "del", "172.16.0.1", speakerRoutesMap) + out, err := netenv.SetStaticRoute(frrk8sPod, "del", "172.16.0.1", + frrconfig.ContainerName, speakerRoutesMap) Expect(err).ToNot(HaveOccurred(), out) } @@ -211,7 +216,8 @@ var _ = Describe("BFD", Ordered, Label(tsparams.LabelBFDTestCases), ContinueOnFa DescribeTable("should provide fast link failure detection", reportxml.ID("47186"), func(bgpProtocol, ipStack string, externalTrafficPolicy corev1.ServiceExternalTrafficPolicyType) { - createExternalNad(tsparams.ExternalMacVlanNADName) + err := define.CreateExternalNad(APIClient, tsparams.ExternalMacVlanNADName, tsparams.TestNamespaceName) + Expect(err).ToNot(HaveOccurred(), "Failed to create a network-attachment-definition") By("Verifying that speaker route map is not empty") Expect(speakerRoutesMap).ToNot(BeNil(), "Speaker route map is empty") @@ -289,18 +295,19 @@ var _ = Describe("BFD", Ordered, Label(tsparams.LabelBFDTestCases), ContinueOnFa pod.StaticIPAnnotation(bridgeNad.Definition.Name, []string{fmt.Sprintf("%s/%s", masterClientPodIP, subMast)})) // Add static routes from client towards Speaker via router internal IPs - for index, workerAddress := range removePrefixFromIPList(nodeAddrList) { + for index, workerAddress := range netcmd.RemovePrefixFromIPList(nodeAddrList) { buffer, err := cmd.SetRouteOnPod(frrPod, workerAddress, frrMasterIPs[index]) Expect(err).ToNot(HaveOccurred(), buffer.String()) } By("Adding static routes to the speakers") for _, frrk8sPod := range frrk8sPods { - out, err := frr.SetStaticRoute(frrk8sPod, "add", masterClientPodIP, speakerRoutesMap) + out, err := netenv.SetStaticRoute(frrk8sPod, "add", masterClientPodIP, + frrconfig.ContainerName, speakerRoutesMap) Expect(err).ToNot(HaveOccurred(), out) } By("Checking that BGP and BFD sessions are established and up") - verifyMetalLbBFDAndBGPSessionsAreUPOnFrrPod(frrPod, removePrefixFromIPList(nodeAddrList)) + verifyMetalLbBFDAndBGPSessionsAreUPOnFrrPod(frrPod, netcmd.RemovePrefixFromIPList(nodeAddrList)) By("Running http check") httpOutput, err := cmd.Curl(frrPod, masterClientPodIP, addressPool[0], ipStack, tsparams.FRRSecondContainerName) @@ -498,7 +505,7 @@ func setLocalGWMode(status bool) { } func verifyMetalLbBFDAndBGPSessionsAreUPOnFrrPod(frrPod *pod.Builder, peerAddrList []string) { - for _, peerAddress := range removePrefixFromIPList(peerAddrList) { + for _, peerAddress := range netcmd.RemovePrefixFromIPList(peerAddrList) { Eventually(frr.BGPNeighborshipHasState, time.Minute*3, tsparams.DefaultRetryInterval). WithArguments(frrPod, peerAddress, "Established").Should( diff --git a/tests/cnf/core/network/metallb/tests/bgp-connect-time-tests.go b/tests/cnf/core/network/metallb/tests/bgp-connect-time-tests.go index b571489c4..f54606b69 100644 --- a/tests/cnf/core/network/metallb/tests/bgp-connect-time-tests.go +++ b/tests/cnf/core/network/metallb/tests/bgp-connect-time-tests.go @@ -14,6 +14,8 @@ import ( "github.com/openshift-kni/eco-goinfra/pkg/pod" "github.com/openshift-kni/eco-goinfra/pkg/reportxml" "github.com/openshift-kni/eco-goinfra/pkg/service" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/cmd" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/define" . "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netinittools" "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/metallb/internal/frr" "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/metallb/internal/metallbenv" @@ -139,7 +141,7 @@ var _ = Describe("FRR", Ordered, Label(tsparams.LabelBGPTestCases), ContinueOnFa Expect(err).ToNot(HaveOccurred(), "Failed to reset BGP connection") By("Verify that BGP session is re-established and up in less then 10 seconds") - verifyMaxReConnectTime(frrPod, removePrefixFromIPList(ipv4NodeAddrList), time.Second*10) + verifyMaxReConnectTime(frrPod, cmd.RemovePrefixFromIPList(ipv4NodeAddrList), time.Second*10) }) It("Update the timer to less then the default on an existing BGP connection", @@ -182,12 +184,14 @@ var _ = Describe("FRR", Ordered, Label(tsparams.LabelBGPTestCases), ContinueOnFa func createAndDeployFRRPod() *pod.Builder { By("Creating External NAD") - createExternalNad(tsparams.ExternalMacVlanNADName) + + err := define.CreateExternalNad(APIClient, tsparams.ExternalMacVlanNADName, tsparams.TestNamespaceName) + Expect(err).ToNot(HaveOccurred(), "Failed to create a network-attachment-definition") By("Creating static ip annotation") staticIPAnnotation := pod.StaticIPAnnotation( - externalNad.Definition.Name, []string{fmt.Sprintf("%s/%s", ipv4metalLbIPList[0], "24")}) + tsparams.ExternalMacVlanNADName, []string{fmt.Sprintf("%s/%s", ipv4metalLbIPList[0], "24")}) By("Creating MetalLb configMap") @@ -202,7 +206,7 @@ func createAndDeployFRRPod() *pod.Builder { } func verifyMaxReConnectTime(frrPod *pod.Builder, peerAddrList []string, maxConnectTime time.Duration) { - for _, peerAddress := range removePrefixFromIPList(peerAddrList) { + for _, peerAddress := range cmd.RemovePrefixFromIPList(peerAddrList) { Eventually(frr.BGPNeighborshipHasState, maxConnectTime, time.Second). WithArguments(frrPod, peerAddress, "Established").Should( diff --git a/tests/cnf/core/network/metallb/tests/bgp-gracefulrestart.go b/tests/cnf/core/network/metallb/tests/bgp-gracefulrestart.go index 12fb5c138..6da93e120 100644 --- a/tests/cnf/core/network/metallb/tests/bgp-gracefulrestart.go +++ b/tests/cnf/core/network/metallb/tests/bgp-gracefulrestart.go @@ -14,6 +14,8 @@ import ( "github.com/openshift-kni/eco-goinfra/pkg/pod" "github.com/openshift-kni/eco-goinfra/pkg/reportxml" + netcmd "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/cmd" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/define" "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/ipaddr" . "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netinittools" "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netparam" @@ -62,7 +64,8 @@ var _ = Describe("BGP Graceful Restart", Ordered, Label(tsparams.LabelGRTestCase err = metallbenv.IsEnvVarMetalLbIPinNodeExtNetRange(ipv4NodeAddrList, ipv4MetalLbIPList, nil) Expect(err).ToNot(HaveOccurred(), "Failed to validate metalLb exported ip address") - createExternalNad(tsparams.ExternalMacVlanNADName) + err = define.CreateExternalNad(APIClient, tsparams.ExternalMacVlanNADName, tsparams.TestNamespaceName) + Expect(err).ToNot(HaveOccurred(), "Failed to create a network-attachment-definition") By("Listing master nodes") masterNodeList, err = nodes.List(APIClient, @@ -82,11 +85,12 @@ var _ = Describe("BGP Graceful Restart", Ordered, Label(tsparams.LabelGRTestCase Expect(err).ToNot(HaveOccurred(), "Failed to recreate metalLb daemonset") By("Creating External NAD") - createExternalNad(tsparams.ExternalMacVlanNADName) + err = define.CreateExternalNad(APIClient, tsparams.ExternalMacVlanNADName, tsparams.TestNamespaceName) + Expect(err).ToNot(HaveOccurred(), "Failed to create a network-attachment-definition") By("Creating static ip annotation") staticIPAnnotation := pod.StaticIPAnnotation( - externalNad.Definition.Name, []string{fmt.Sprintf("%s/%s", ipv4MetalLbIPList[0], "24")}) + tsparams.ExternalMacVlanNADName, []string{fmt.Sprintf("%s/%s", ipv4MetalLbIPList[0], "24")}) By("Creating MetalLb configMap") masterConfigMap := createConfigMap(tsparams.LocalBGPASN, ipv4NodeAddrList, false, false) @@ -105,7 +109,7 @@ var _ = Describe("BGP Graceful Restart", Ordered, Label(tsparams.LabelGRTestCase By("Creating static ip annotation") pod.StaticIPAnnotation( - externalNad.Definition.Name, []string{fmt.Sprintf("%s/%s", ipv4MetalLbIPList[0], netparam.IPSubnet24)}) + tsparams.ExternalMacVlanNADName, []string{fmt.Sprintf("%s/%s", ipv4MetalLbIPList[0], netparam.IPSubnet24)}) By("Creating an IPAddressPool and BGPAdvertisement") ipAddressPool = setupBgpAdvertisement(addressPool, 32) @@ -206,7 +210,7 @@ func gracefulRestartTest( } func verifyMetalLbBGPSessionsAreDown(frrPod *pod.Builder, peerAddrList []string) { - for _, peerAddress := range removePrefixFromIPList(peerAddrList) { + for _, peerAddress := range netcmd.RemovePrefixFromIPList(peerAddrList) { Eventually(frr.BGPNeighborshipHasState, 30*time.Second, tsparams.DefaultRetryInterval). WithArguments(frrPod, peerAddress, "Established").ShouldNot( @@ -215,7 +219,7 @@ func verifyMetalLbBGPSessionsAreDown(frrPod *pod.Builder, peerAddrList []string) } func verifyGREnabledOnNeighbors(frrPod *pod.Builder, peerAddrList []string) { - for _, peerAddress := range removePrefixFromIPList(peerAddrList) { + for _, peerAddress := range netcmd.RemovePrefixFromIPList(peerAddrList) { grStatus, err := frr.GetGracefulRestartStatus(frrPod, peerAddress) Expect(err).ToNot(HaveOccurred(), "Failed to get GracefulRestart status") diff --git a/tests/cnf/core/network/metallb/tests/bgp-remote-as-dynamic.go b/tests/cnf/core/network/metallb/tests/bgp-remote-as-dynamic.go index 10353d902..4ba83a051 100644 --- a/tests/cnf/core/network/metallb/tests/bgp-remote-as-dynamic.go +++ b/tests/cnf/core/network/metallb/tests/bgp-remote-as-dynamic.go @@ -11,6 +11,10 @@ import ( "github.com/openshift-kni/eco-goinfra/pkg/pod" "github.com/openshift-kni/eco-goinfra/pkg/reportxml" "github.com/openshift-kni/eco-goinfra/pkg/schemes/metallb/mlbtypesv1beta2" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/cmd" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/define" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/frrconfig" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netenv" . "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netinittools" "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netparam" "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/metallb/internal/frr" @@ -81,7 +85,7 @@ var _ = Describe("BGP remote-dynamicAS", Ordered, Label(tsparams.LabelDynamicRem externalAdvertisedIPv6Routes, dynamicASeBGP, tsparams.RemoteBGPASN) By("Checking that BGP session is established and up") - verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod, removePrefixFromIPList(ipv4NodeAddrList)) + verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod, cmd.RemovePrefixFromIPList(ipv4NodeAddrList)) By("Validating external FRR AS number received on the FRR nodes") Eventually(func() error { @@ -97,7 +101,7 @@ var _ = Describe("BGP remote-dynamicAS", Ordered, Label(tsparams.LabelDynamicRem externalAdvertisedIPv6Routes, dynamicASiBGP, tsparams.LocalBGPASN) By("Checking that BGP session is established and up") - verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod, removePrefixFromIPList(ipv4NodeAddrList)) + verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod, cmd.RemovePrefixFromIPList(ipv4NodeAddrList)) By("Validating external FRR AS number received on the FRR nodes") Eventually(func() error { @@ -113,7 +117,7 @@ var _ = Describe("BGP remote-dynamicAS", Ordered, Label(tsparams.LabelDynamicRem externalAdvertisedIPv6Routes, dynamicASiBGP, tsparams.RemoteBGPASN) By("Checking that BGP session is down") - verifyMetalLbBGPSessionsAreDownOnFrrPod(frrPod, removePrefixFromIPList(ipv4NodeAddrList)) + verifyMetalLbBGPSessionsAreDownOnFrrPod(frrPod, cmd.RemovePrefixFromIPList(ipv4NodeAddrList)) By("Validating external FRR AS number received is incorrect and marked as 0 on the FRR nodes") Eventually(func() error { @@ -136,11 +140,14 @@ var _ = Describe("BGP remote-dynamicAS", Ordered, Label(tsparams.LabelDynamicRem }) Expect(err).ToNot(HaveOccurred(), "Failed to list pods") - speakerRoutesMap := buildRoutesMapWithSpecificRoutes(frrk8sPods, []string{ipv4metalLbIPList[0], - ipv4metalLbIPList[1], frrNodeSecIntIPv4Addresses[0], frrNodeSecIntIPv4Addresses[1]}) + speakerRoutesMap, err := netenv.BuildRoutesMapWithSpecificRoutes(frrk8sPods, workerNodeList, + []string{ipv4metalLbIPList[0], ipv4metalLbIPList[1], frrNodeSecIntIPv4Addresses[0], + frrNodeSecIntIPv4Addresses[1]}) + Expect(err).ToNot(HaveOccurred(), "Failed to create route map with specific routes") for _, frrk8sPod := range frrk8sPods { - out, err := frr.SetStaticRoute(frrk8sPod, "del", frrExternalMasterIPAddress, speakerRoutesMap) + out, err := netenv.SetStaticRoute(frrk8sPod, "del", frrExternalMasterIPAddress, + frrconfig.ContainerName, speakerRoutesMap) Expect(err).ToNot(HaveOccurred(), out) } @@ -158,7 +165,7 @@ var _ = Describe("BGP remote-dynamicAS", Ordered, Label(tsparams.LabelDynamicRem createBGPPeerWithDynamicASN(frrExternalMasterIPAddress, dynamicASiBGP, false) By("Checking that BGP session is established and up") - verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod, removePrefixFromIPList(ipv4NodeAddrList)) + verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod, cmd.RemovePrefixFromIPList(ipv4NodeAddrList)) By("Validating external FRR AS number received on the FRR nodes") Eventually(func() error { @@ -177,7 +184,7 @@ var _ = Describe("BGP remote-dynamicAS", Ordered, Label(tsparams.LabelDynamicRem createBGPPeerWithDynamicASN(frrExternalMasterIPAddress, dynamicASeBGP, true) By("Checking that BGP session is established and up") - verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod, removePrefixFromIPList(ipv4NodeAddrList)) + verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod, cmd.RemovePrefixFromIPList(ipv4NodeAddrList)) By("Validating external FRR AS number received on the FRR nodes") Eventually(func() error { @@ -202,12 +209,14 @@ func createBGPPeerWithDynamicASN(peerIP, dynamicASN string, eBgpMultiHop bool) { func deployFrrExternalPod(hubIPAddresses, externalAdvertisedIPv4Routes, externalAdvertisedIPv6Routes []string, localAS int) *pod.Builder { By("Creating External NAD") - createExternalNad(tsparams.ExternalMacVlanNADName) + + err := define.CreateExternalNad(APIClient, tsparams.ExternalMacVlanNADName, tsparams.TestNamespaceName) + Expect(err).ToNot(HaveOccurred(), "Failed to create a network-attachment-definition") By("Creating static ip annotation") staticIPAnnotation := pod.StaticIPAnnotation( - externalNad.Definition.Name, []string{fmt.Sprintf("%s/%s", ipv4metalLbIPList[0], "24")}) + tsparams.ExternalMacVlanNADName, []string{fmt.Sprintf("%s/%s", ipv4metalLbIPList[0], "24")}) By("Creating MetalLb configMap") @@ -269,21 +278,25 @@ func setupBGPRemoteASMultiHopTest(ipv4metalLbIPList, hubIPv4ExternalAddresses, e Expect(err).ToNot(HaveOccurred(), "Fail to set iteration parameters") By("Creating External NAD for master FRR pod") - createExternalNad(tsparams.ExternalMacVlanNADName) + + err = define.CreateExternalNad(APIClient, tsparams.ExternalMacVlanNADName, tsparams.TestNamespaceName) + Expect(err).ToNot(HaveOccurred(), "Failed to create a network-attachment-definition") By("Creating External NAD for hub FRR pods") - createExternalNad(tsparams.HubMacVlanNADName) + + err = define.CreateExternalNad(APIClient, tsparams.HubMacVlanNADName, tsparams.TestNamespaceName) + Expect(err).ToNot(HaveOccurred(), "Failed to create a network-attachment-definition") By("Creating static ip annotation for hub0") - hub0BRstaticIPAnnotation := createStaticIPAnnotations(tsparams.ExternalMacVlanNADName, + hub0BRstaticIPAnnotation := frrconfig.CreateStaticIPAnnotations(tsparams.ExternalMacVlanNADName, tsparams.HubMacVlanNADName, []string{fmt.Sprintf("%s/24", ipv4metalLbIPList[0])}, []string{fmt.Sprintf("%s/24", hubIPv4ExternalAddresses[0])}) By("Creating static ip annotation for hub1") - hub1BRstaticIPAnnotation := createStaticIPAnnotations(tsparams.ExternalMacVlanNADName, + hub1BRstaticIPAnnotation := frrconfig.CreateStaticIPAnnotations(tsparams.ExternalMacVlanNADName, tsparams.HubMacVlanNADName, []string{fmt.Sprintf("%s/24", ipv4metalLbIPList[1])}, []string{fmt.Sprintf("%s/24", hubIPv4ExternalAddresses[1])}) @@ -309,10 +322,12 @@ func setupBGPRemoteASMultiHopTest(ipv4metalLbIPList, hubIPv4ExternalAddresses, e By("Adding static routes to the speakers") - speakerRoutesMap := buildRoutesMapWithSpecificRoutes(frrk8sPods, ipv4metalLbIPList) + speakerRoutesMap, err := netenv.BuildRoutesMapWithSpecificRoutes(frrk8sPods, workerNodeList, ipv4metalLbIPList) + Expect(err).ToNot(HaveOccurred(), "Failed to create route map with specific routes") for _, frrk8sPod := range frrk8sPods { - out, err := frr.SetStaticRoute(frrk8sPod, "add", masterClientPodIP, speakerRoutesMap) + out, err := netenv.SetStaticRoute(frrk8sPod, "add", masterClientPodIP, frrconfig.ContainerName, + speakerRoutesMap) Expect(err).ToNot(HaveOccurred(), out) } diff --git a/tests/cnf/core/network/metallb/tests/bgp-tests.go b/tests/cnf/core/network/metallb/tests/bgp-tests.go index 542a036c2..def13bd22 100644 --- a/tests/cnf/core/network/metallb/tests/bgp-tests.go +++ b/tests/cnf/core/network/metallb/tests/bgp-tests.go @@ -16,6 +16,8 @@ import ( "github.com/openshift-kni/eco-goinfra/pkg/pod" "github.com/openshift-kni/eco-goinfra/pkg/reportxml" "github.com/openshift-kni/eco-goinfra/pkg/service" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/cmd" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/define" . "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netinittools" "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netparam" "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/metallb/internal/frr" @@ -63,10 +65,10 @@ var _ = Describe("BGP", Ordered, Label(tsparams.LabelBGPTestCases), ContinueOnFa BeforeEach(func() { By("Creating External NAD") - createExternalNad(tsparams.ExternalMacVlanNADName) + err := define.CreateExternalNad(APIClient, tsparams.ExternalMacVlanNADName, tsparams.TestNamespaceName) + Expect(err).ToNot(HaveOccurred(), "Failed to create a network-attachment-definition") By("Listing metalLb speakers pod") - var err error frrk8sPods, err = pod.List(APIClient, NetConfig.MlbOperatorNamespace, metav1.ListOptions{ LabelSelector: tsparams.FRRK8sDefaultLabel, }) @@ -123,7 +125,7 @@ var _ = Describe("BGP", Ordered, Label(tsparams.LabelBGPTestCases), ContinueOnFa By("Creating static ip annotation") staticIPAnnotation := pod.StaticIPAnnotation( - externalNad.Definition.Name, []string{fmt.Sprintf("%s/%s", mlbAddressList[0], subMask)}) + tsparams.ExternalMacVlanNADName, []string{fmt.Sprintf("%s/%s", mlbAddressList[0], subMask)}) By("Creating FRR Pod") frrPod := createFrrPod( @@ -139,10 +141,10 @@ var _ = Describe("BGP", Ordered, Label(tsparams.LabelBGPTestCases), ContinueOnFa setupNGNXPod(workerNodeList[0].Definition.Name) By("Checking that BGP session is established and up") - verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod, removePrefixFromIPList(nodeAddrList)) + verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod, cmd.RemovePrefixFromIPList(nodeAddrList)) By("Validating BGP route prefix") - validatePrefix(frrPod, ipStack, removePrefixFromIPList(nodeAddrList), addressPool, prefixLen) + validatePrefix(frrPod, ipStack, cmd.RemovePrefixFromIPList(nodeAddrList), addressPool, prefixLen) }, Entry("", netparam.IPV4Family, 32, @@ -162,7 +164,7 @@ var _ = Describe("BGP", Ordered, Label(tsparams.LabelBGPTestCases), ContinueOnFa It("provides Prometheus BGP metrics", reportxml.ID("47202"), func() { By("Creating static ip annotation") staticIPAnnotation := pod.StaticIPAnnotation( - externalNad.Definition.Name, []string{fmt.Sprintf("%s/%s", ipv4metalLbIPList[0], "24")}) + tsparams.ExternalMacVlanNADName, []string{fmt.Sprintf("%s/%s", ipv4metalLbIPList[0], "24")}) By("Creating MetalLb configMap") masterConfigMap := createConfigMap(tsparams.LocalBGPASN, ipv4NodeAddrList, false, false) @@ -176,7 +178,7 @@ var _ = Describe("BGP", Ordered, Label(tsparams.LabelBGPTestCases), ContinueOnFa frrk8sPods) By("Checking that BGP session is established and up") - verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod, removePrefixFromIPList(ipv4NodeAddrList)) + verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod, cmd.RemovePrefixFromIPList(ipv4NodeAddrList)) By("Label namespace") testNs, err := namespace.Pull(APIClient, NetConfig.MlbOperatorNamespace) diff --git a/tests/cnf/core/network/metallb/tests/common.go b/tests/cnf/core/network/metallb/tests/common.go index f767c0b77..89e2958d4 100644 --- a/tests/cnf/core/network/metallb/tests/common.go +++ b/tests/cnf/core/network/metallb/tests/common.go @@ -15,9 +15,9 @@ import ( "github.com/openshift-kni/eco-goinfra/pkg/nodes" "github.com/openshift-kni/eco-goinfra/pkg/pod" "github.com/openshift-kni/eco-goinfra/pkg/service" - "github.com/openshift-kni/eco-gotests/tests/cnf/core/internal/coreparams" + netcmd "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/cmd" "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/define" - "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/ipaddr" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/frrconfig" "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netenv" . "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netinittools" "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/metallb/internal/cmd" @@ -83,8 +83,8 @@ func setWorkerNodeListAndLabelForBfdTests( func createConfigMap( bgpAsn int, nodeAddrList []string, enableMultiHop, enableBFD bool) *configmap.Builder { frrBFDConfig := frr.DefineBGPConfig( - bgpAsn, tsparams.LocalBGPASN, removePrefixFromIPList(nodeAddrList), enableMultiHop, enableBFD) - configMapData := frr.DefineBaseConfig(tsparams.DaemonsFile, frrBFDConfig, "") + bgpAsn, tsparams.LocalBGPASN, netcmd.RemovePrefixFromIPList(nodeAddrList), enableMultiHop, enableBFD) + configMapData := frrconfig.DefineBaseConfig(frrconfig.DaemonsFile, frrBFDConfig, "") masterConfigMap, err := configmap.NewBuilder(APIClient, "frr-master-node-config", tsparams.TestNamespaceName). WithData(configMapData).Create() Expect(err).ToNot(HaveOccurred(), "Failed to create config map") @@ -95,24 +95,13 @@ func createConfigMap( func createHubConfigMap(name string) *configmap.Builder { frrBFDConfig := frr.DefineBGPConfig( tsparams.LocalBGPASN, tsparams.LocalBGPASN, []string{"10.10.0.10"}, false, false) - configMapData := frr.DefineBaseConfig(tsparams.DaemonsFile, frrBFDConfig, "") + configMapData := frrconfig.DefineBaseConfig(frrconfig.DaemonsFile, frrBFDConfig, "") hubConfigMap, err := configmap.NewBuilder(APIClient, name, tsparams.TestNamespaceName).WithData(configMapData).Create() Expect(err).ToNot(HaveOccurred(), "Failed to create hub config map") return hubConfigMap } -func createExternalNad(name string) { - By("Creating external BR-EX NetworkAttachmentDefinition") - - macVlanPlugin, err := define.MasterNadPlugin(coreparams.OvnExternalBridge, "bridge", nad.IPAMStatic()) - Expect(err).ToNot(HaveOccurred(), "Failed to define master nad plugin") - externalNad, err = nad.NewBuilder(APIClient, name, tsparams.TestNamespaceName). - WithMasterPlugin(macVlanPlugin).Create() - Expect(err).ToNot(HaveOccurred(), "Failed to create external NetworkAttachmentDefinition") - Expect(externalNad.Exists()).To(BeTrue(), "Failed to detect external NetworkAttachmentDefinition") -} - func createExternalNadWithMasterInterface(name, masterInterface string) { By("Creating external BR-EX NetworkAttachmentDefinition") @@ -189,7 +178,7 @@ func setupL2Advertisement(addressPool []string) *metallb.IPAddressPoolBuilder { } func verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod *pod.Builder, peerAddrList []string) { - for _, peerAddress := range removePrefixFromIPList(peerAddrList) { + for _, peerAddress := range netcmd.RemovePrefixFromIPList(peerAddrList) { Eventually(frr.BGPNeighborshipHasState, time.Minute*3, tsparams.DefaultRetryInterval). WithArguments(frrPod, peerAddress, "Established").Should( @@ -198,7 +187,7 @@ func verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod *pod.Builder, peerAddrList []s } func verifyMetalLbBGPSessionsAreDownOnFrrPod(frrPod *pod.Builder, peerAddrList []string) { - for _, peerAddress := range removePrefixFromIPList(peerAddrList) { + for _, peerAddress := range netcmd.RemovePrefixFromIPList(peerAddrList) { Consistently(frr.BGPNeighborshipHasState, time.Minute, tsparams.DefaultRetryInterval). WithArguments(frrPod, peerAddress, "Established").Should( @@ -298,15 +287,6 @@ func setupNGNXPod(nodeName string) { Expect(err).ToNot(HaveOccurred(), "Failed to create nginx test pod") } -func removePrefixFromIPList(ipAddressList []string) []string { - var ipAddressListWithoutPrefix []string - for _, ipaddress := range ipAddressList { - ipAddressListWithoutPrefix = append(ipAddressListWithoutPrefix, ipaddr.RemovePrefix(ipaddress)) - } - - return ipAddressListWithoutPrefix -} - func verifyMetricPresentInPrometheus( frrk8sPods []*pod.Builder, prometheusPod *pod.Builder, metricPrefix string, expectedMetrics ...[]string) { By("Verifying if metrics are present in Prometheus database") diff --git a/tests/cnf/core/network/metallb/tests/frrk8-tests.go b/tests/cnf/core/network/metallb/tests/frrk8-tests.go index 3dd1915a4..86bb17779 100644 --- a/tests/cnf/core/network/metallb/tests/frrk8-tests.go +++ b/tests/cnf/core/network/metallb/tests/frrk8-tests.go @@ -16,13 +16,16 @@ import ( "github.com/openshift-kni/eco-goinfra/pkg/nodes" "github.com/openshift-kni/eco-goinfra/pkg/pod" "github.com/openshift-kni/eco-goinfra/pkg/reportxml" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/cmd" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/define" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/frrconfig" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netenv" . "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netinittools" "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netnmstate" "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netparam" "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/metallb/internal/frr" "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/metallb/internal/metallbenv" "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/metallb/internal/tsparams" - "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" @@ -130,10 +133,11 @@ var _ = Describe("FRR", Ordered, Label(tsparams.LabelFRRTestCases), ContinueOnFa false, 0, frrk8sPods) By("Checking that BGP session is established and up") - verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod, removePrefixFromIPList(ipv4NodeAddrList)) + verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod, cmd.RemovePrefixFromIPList(ipv4NodeAddrList)) By("Validating BGP route prefix") - validatePrefix(frrPod, netparam.IPV4Family, removePrefixFromIPList(nodeAddrList), addressPool, 32) + validatePrefix(frrPod, netparam.IPV4Family, cmd.RemovePrefixFromIPList(nodeAddrList), + addressPool, 32) By("Create a frrconfiguration allow all") createFrrConfiguration(frrCongigAllowAll, ipv4metalLbIPList[0], @@ -176,10 +180,11 @@ var _ = Describe("FRR", Ordered, Label(tsparams.LabelFRRTestCases), ContinueOnFa false, 0, frrk8sPods) By("Checking that BGP session is established and up") - verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod, removePrefixFromIPList(ipv4NodeAddrList)) + verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod, cmd.RemovePrefixFromIPList(ipv4NodeAddrList)) By("Validating BGP route prefix") - validatePrefix(frrPod, netparam.IPV4Family, removePrefixFromIPList(nodeAddrList), addressPool, 32) + validatePrefix(frrPod, netparam.IPV4Family, cmd.RemovePrefixFromIPList(nodeAddrList), + addressPool, 32) By("Create a frrconfiguration with prefix filter") @@ -223,10 +228,11 @@ var _ = Describe("FRR", Ordered, Label(tsparams.LabelFRRTestCases), ContinueOnFa false, 0, frrk8sPods) By("Checking that BGP session is established and up") - verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod, removePrefixFromIPList(ipv4NodeAddrList)) + verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod, cmd.RemovePrefixFromIPList(ipv4NodeAddrList)) By("Validating BGP route prefix") - validatePrefix(frrPod, netparam.IPV4Family, removePrefixFromIPList(nodeAddrList), addressPool, 32) + validatePrefix(frrPod, netparam.IPV4Family, cmd.RemovePrefixFromIPList(nodeAddrList), + addressPool, 32) By("Create a frrconfiguration allow all") createFrrConfiguration(frrCongigAllowAll, ipv4metalLbIPList[0], tsparams.LocalBGPASN, @@ -267,10 +273,11 @@ var _ = Describe("FRR", Ordered, Label(tsparams.LabelFRRTestCases), ContinueOnFa false, 0, frrk8sPods) By("Checking that BGP session is established and up") - verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod, removePrefixFromIPList(ipv4NodeAddrList)) + verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod, cmd.RemovePrefixFromIPList(ipv4NodeAddrList)) By("Validating BGP route prefix") - validatePrefix(frrPod, netparam.IPV4Family, removePrefixFromIPList(nodeAddrList), addressPool, 32) + validatePrefix(frrPod, netparam.IPV4Family, cmd.RemovePrefixFromIPList(nodeAddrList), + addressPool, 32) By("Create first frrconfiguration that receieves a single route") createFrrConfiguration(frrConfigFiltered1, ipv4metalLbIPList[0], tsparams.LocalBGPASN, @@ -438,11 +445,13 @@ var _ = Describe("FRR", Ordered, Label(tsparams.LabelFRRTestCases), ContinueOnFa }) Expect(err).ToNot(HaveOccurred(), "Failed to list pods") - speakerRoutesMap := buildRoutesMapWithSpecificRoutes(frrk8sPods, []string{ipv4metalLbIPList[0], - ipv4metalLbIPList[1], frrNodeSecIntIPv4Addresses[0], frrNodeSecIntIPv4Addresses[1]}) + speakerRoutesMap, err := netenv.BuildRoutesMapWithSpecificRoutes(frrk8sPods, workerNodeList, + []string{ipv4metalLbIPList[0], ipv4metalLbIPList[1], frrNodeSecIntIPv4Addresses[0], frrNodeSecIntIPv4Addresses[1]}) + Expect(err).ToNot(HaveOccurred(), "Failed to create route map with specific routes") for _, frrk8sPod := range frrk8sPods { - out, err := frr.SetStaticRoute(frrk8sPod, "del", frrExternalMasterIPAddress, speakerRoutesMap) + out, err := netenv.SetStaticRoute(frrk8sPod, "del", frrExternalMasterIPAddress, + frrconfig.ContainerName, speakerRoutesMap) Expect(err).ToNot(HaveOccurred(), out) } @@ -477,27 +486,32 @@ var _ = Describe("FRR", Ordered, Label(tsparams.LabelFRRTestCases), ContinueOnFa reportxml.ID("74278"), func() { By("Adding static routes to the speakers") - speakerRoutesMap := buildRoutesMapWithSpecificRoutes(frrk8sPods, ipv4metalLbIPList) + speakerRoutesMap, err := netenv.BuildRoutesMapWithSpecificRoutes(frrk8sPods, workerNodeList, + ipv4metalLbIPList) + Expect(err).ToNot(HaveOccurred(), "Failed to create route map with specific routes") for _, frrk8sPod := range frrk8sPods { - out, err := frr.SetStaticRoute(frrk8sPod, "add", masterClientPodIP, speakerRoutesMap) + out, err := netenv.SetStaticRoute(frrk8sPod, "add", masterClientPodIP, + frrconfig.ContainerName, speakerRoutesMap) Expect(err).ToNot(HaveOccurred(), out) } By("Creating External NAD for master FRR pod") - createExternalNad(tsparams.ExternalMacVlanNADName) + err = define.CreateExternalNad(APIClient, tsparams.ExternalMacVlanNADName, tsparams.TestNamespaceName) + Expect(err).ToNot(HaveOccurred(), "Failed to create a network-attachment-definition") By("Creating External NAD for hub FRR pods") - createExternalNad(tsparams.HubMacVlanNADName) + err = define.CreateExternalNad(APIClient, tsparams.HubMacVlanNADName, tsparams.TestNamespaceName) + Expect(err).ToNot(HaveOccurred(), "Failed to create a network-attachment-definition") By("Creating static ip annotation for hub0") - hub0BRstaticIPAnnotation := createStaticIPAnnotations(tsparams.ExternalMacVlanNADName, + hub0BRstaticIPAnnotation := frrconfig.CreateStaticIPAnnotations(tsparams.ExternalMacVlanNADName, tsparams.HubMacVlanNADName, []string{fmt.Sprintf("%s/24", ipv4metalLbIPList[0])}, []string{fmt.Sprintf("%s/24", hubIPv4ExternalAddresses[0])}) By("Creating static ip annotation for hub1") - hub1BRstaticIPAnnotation := createStaticIPAnnotations(tsparams.ExternalMacVlanNADName, + hub1BRstaticIPAnnotation := frrconfig.CreateStaticIPAnnotations(tsparams.ExternalMacVlanNADName, tsparams.HubMacVlanNADName, []string{fmt.Sprintf("%s/24", ipv4metalLbIPList[1])}, []string{fmt.Sprintf("%s/24", hubIPv4ExternalAddresses[1])}) @@ -523,10 +537,10 @@ var _ = Describe("FRR", Ordered, Label(tsparams.LabelFRRTestCases), ContinueOnFa false, 0, frrk8sPods) By("Checking that BGP session is established and up") - verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod, removePrefixFromIPList(ipv4NodeAddrList)) + verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod, cmd.RemovePrefixFromIPList(ipv4NodeAddrList)) By("Validating BGP route prefix") - validatePrefix(frrPod, netparam.IPV4Family, removePrefixFromIPList(nodeAddrList), + validatePrefix(frrPod, netparam.IPV4Family, cmd.RemovePrefixFromIPList(nodeAddrList), addressPool, 32) By("Create a frrconfiguration allow all for EBGP multihop") @@ -546,27 +560,32 @@ var _ = Describe("FRR", Ordered, Label(tsparams.LabelFRRTestCases), ContinueOnFa reportxml.ID("47279"), func() { By("Adding static routes to the speakers") - speakerRoutesMap := buildRoutesMapWithSpecificRoutes(frrk8sPods, ipv4metalLbIPList) + speakerRoutesMap, err := netenv.BuildRoutesMapWithSpecificRoutes(frrk8sPods, workerNodeList, + ipv4metalLbIPList) + Expect(err).ToNot(HaveOccurred(), "Failed to create route map with specific routes") for _, frrk8sPod := range frrk8sPods { - out, err := frr.SetStaticRoute(frrk8sPod, "add", masterClientPodIP, speakerRoutesMap) + out, err := netenv.SetStaticRoute(frrk8sPod, "add", masterClientPodIP, + frrconfig.ContainerName, speakerRoutesMap) Expect(err).ToNot(HaveOccurred(), out) } By("Creating External NAD for master FRR pod") - createExternalNad(tsparams.ExternalMacVlanNADName) + err = define.CreateExternalNad(APIClient, tsparams.ExternalMacVlanNADName, tsparams.TestNamespaceName) + Expect(err).ToNot(HaveOccurred(), "Failed to create a network-attachment-definition") By("Creating External NAD for hub FRR pods") - createExternalNad(tsparams.HubMacVlanNADName) + err = define.CreateExternalNad(APIClient, tsparams.HubMacVlanNADName, tsparams.TestNamespaceName) + Expect(err).ToNot(HaveOccurred(), "Failed to create a network-attachment-definition") By("Creating static ip annotation for hub0") - hub0BRstaticIPAnnotation := createStaticIPAnnotations(tsparams.ExternalMacVlanNADName, + hub0BRstaticIPAnnotation := frrconfig.CreateStaticIPAnnotations(tsparams.ExternalMacVlanNADName, tsparams.HubMacVlanNADName, []string{fmt.Sprintf("%s/24", ipv4metalLbIPList[0])}, []string{fmt.Sprintf("%s/24", hubIPv4ExternalAddresses[0])}) By("Creating static ip annotation for hub1") - hub1BRstaticIPAnnotation := createStaticIPAnnotations(tsparams.ExternalMacVlanNADName, + hub1BRstaticIPAnnotation := frrconfig.CreateStaticIPAnnotations(tsparams.ExternalMacVlanNADName, tsparams.HubMacVlanNADName, []string{fmt.Sprintf("%s/24", ipv4metalLbIPList[1])}, []string{fmt.Sprintf("%s/24", hubIPv4ExternalAddresses[1])}) @@ -592,10 +611,10 @@ var _ = Describe("FRR", Ordered, Label(tsparams.LabelFRRTestCases), ContinueOnFa true, 0, frrk8sPods) By("Checking that BGP session is established and up") - verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod, removePrefixFromIPList(ipv4NodeAddrList)) + verifyMetalLbBGPSessionsAreUPOnFrrPod(frrPod, cmd.RemovePrefixFromIPList(ipv4NodeAddrList)) By("Validating BGP route prefix") - validatePrefix(frrPod, netparam.IPV4Family, removePrefixFromIPList(nodeAddrList), + validatePrefix(frrPod, netparam.IPV4Family, cmd.RemovePrefixFromIPList(nodeAddrList), addressPool, 32) By("Create a frrconfiguration allow all for EBGP multihop") @@ -628,14 +647,16 @@ var _ = Describe("FRR", Ordered, Label(tsparams.LabelFRRTestCases), ContinueOnFa srIovInterfacesUnderTest[0], frrNodeSecIntIPv4Addresses[1], "2001:100::253", vlanID) By("Adding static routes to the speakers") - speakerRoutesMap := buildRoutesMapWithSpecificRoutes(frrk8sPods, hubSecIntIPv4Addresses) + speakerRoutesMap, err := netenv.BuildRoutesMapWithSpecificRoutes(frrk8sPods, workerNodeList, + hubSecIntIPv4Addresses) + Expect(err).ToNot(HaveOccurred(), "Failed to create route map with specific routes") for _, frrk8sPod := range frrk8sPods { // Wait until the interface is created before adding the static route Eventually(func() error { // Here you can add logic to check if the interface exists - out, err := frr.SetStaticRoute(frrk8sPod, "add", fmt.Sprintf("%s/32", - frrExternalMasterIPAddress), speakerRoutesMap) + out, err := netenv.SetStaticRoute(frrk8sPod, "add", fmt.Sprintf("%s/32", + frrExternalMasterIPAddress), frrconfig.ContainerName, speakerRoutesMap) if err != nil { return fmt.Errorf("error adding static route: %s", out) } @@ -651,22 +672,24 @@ var _ = Describe("FRR", Ordered, Label(tsparams.LabelFRRTestCases), ContinueOnFa createExternalNadWithMasterInterface(tsparams.HubMacVlanNADSecIntName, interfaceNameWithVlan) By("Creating External NAD for master FRR pod") - createExternalNad(tsparams.ExternalMacVlanNADName) + err = define.CreateExternalNad(APIClient, tsparams.ExternalMacVlanNADName, tsparams.TestNamespaceName) + Expect(err).ToNot(HaveOccurred(), "Failed to create a network-attachment-definition") By("Creating External NAD for hub FRR pods") - createExternalNad(tsparams.HubMacVlanNADName) + err = define.CreateExternalNad(APIClient, tsparams.HubMacVlanNADName, tsparams.TestNamespaceName) + Expect(err).ToNot(HaveOccurred(), "Failed to create a network-attachment-definition") By("Creating MetalLb Hub pod configMap") createHubConfigMapSecInt := createHubConfigMap("frr-hub-node-config") By("Creating static ip annotation for hub0") - hub0BRStaticSecIntIPAnnotation := createStaticIPAnnotations(tsparams.HubMacVlanNADSecIntName, + hub0BRStaticSecIntIPAnnotation := frrconfig.CreateStaticIPAnnotations(tsparams.HubMacVlanNADSecIntName, tsparams.HubMacVlanNADName, []string{fmt.Sprintf("%s/24", hubSecIntIPv4Addresses[0])}, []string{fmt.Sprintf("%s/24", hubIPv4ExternalAddresses[0])}) By("Creating static ip annotation for hub1") - hub1SecIntIPAnnotation := createStaticIPAnnotations(tsparams.HubMacVlanNADSecIntName, + hub1SecIntIPAnnotation := frrconfig.CreateStaticIPAnnotations(tsparams.HubMacVlanNADSecIntName, tsparams.HubMacVlanNADName, []string{fmt.Sprintf("%s/24", hubSecIntIPv4Addresses[1])}, []string{fmt.Sprintf("%s/24", hubIPv4ExternalAddresses[1])}) @@ -770,12 +793,14 @@ func deployTestPods(addressPool, hubIPAddresses, externalAdvertisedIPv4Routes, setupNGNXPod(workerNodeList[0].Definition.Name) By("Creating External NAD") - createExternalNad(tsparams.ExternalMacVlanNADName) + + err := define.CreateExternalNad(APIClient, tsparams.ExternalMacVlanNADName, tsparams.TestNamespaceName) + Expect(err).ToNot(HaveOccurred(), "Failed to create a network-attachment-definition") By("Creating static ip annotation") staticIPAnnotation := pod.StaticIPAnnotation( - externalNad.Definition.Name, []string{fmt.Sprintf("%s/%s", ipv4metalLbIPList[0], "24")}) + tsparams.ExternalMacVlanNADName, []string{fmt.Sprintf("%s/%s", ipv4metalLbIPList[0], "24")}) By("Creating MetalLb configMap") @@ -846,8 +871,8 @@ func createConfigMapWithStaticRoutes( enableMultiHop, enableBFD bool) *configmap.Builder { frrBFDConfig := frr.DefineBGPConfigWithStaticRouteAndNetwork( bgpAsn, tsparams.LocalBGPASN, hubIPAddresses, externalAdvertisedIPv4Routes, - externalAdvertisedIPv6Routes, removePrefixFromIPList(nodeAddrList), enableMultiHop, enableBFD) - configMapData := frr.DefineBaseConfig(tsparams.DaemonsFile, frrBFDConfig, "") + externalAdvertisedIPv6Routes, cmd.RemovePrefixFromIPList(nodeAddrList), enableMultiHop, enableBFD) + configMapData := frrconfig.DefineBaseConfig(frrconfig.DaemonsFile, frrBFDConfig, "") masterConfigMap, err := configmap.NewBuilder(APIClient, "frr-master-node-config", tsparams.TestNamespaceName). WithData(configMapData).Create() Expect(err).ToNot(HaveOccurred(), "Failed to create config map") @@ -857,7 +882,7 @@ func createConfigMapWithStaticRoutes( func verifyExternalAdvertisedRoutes(frrPod *pod.Builder, ipv4NodeAddrList, externalExpectedRoutes []string) { // Get advertised routes from FRR pod, now returned as a map of node IPs to their advertised routes - advertisedRoutesMap, err := frr.GetBGPAdvertisedRoutes(frrPod, removePrefixFromIPList(ipv4NodeAddrList)) + advertisedRoutesMap, err := frr.GetBGPAdvertisedRoutes(frrPod, cmd.RemovePrefixFromIPList(ipv4NodeAddrList)) Expect(err).ToNot(HaveOccurred(), "Failed to find advertised routes") // Iterate through each node in the advertised routes map @@ -875,26 +900,6 @@ func verifyExternalAdvertisedRoutes(frrPod *pod.Builder, ipv4NodeAddrList, exter } } -func buildRoutesMapWithSpecificRoutes(podList []*pod.Builder, nextHopList []string) map[string]string { - Expect(len(podList)).ToNot(BeZero(), "Pod list is empty") - Expect(len(nextHopList)).ToNot(BeZero(), "Nexthop IP addresses list is empty") - Expect(len(nextHopList)).To(BeNumerically(">=", len(podList)), - fmt.Sprintf("Number of speaker IP addresses[%d] is less than the number of pods[%d]", - len(nextHopList), len(podList))) - - routesMap := make(map[string]string) - - for _, frrPod := range podList { - if frrPod.Definition.Spec.NodeName == workerNodeList[0].Definition.Name { - routesMap[frrPod.Definition.Spec.NodeName] = nextHopList[1] - } else { - routesMap[frrPod.Definition.Spec.NodeName] = nextHopList[0] - } - } - - return routesMap -} - func createSecondaryInterfaceOnNode(policyName, nodeName, interfaceName, ipv4Address, ipv6Address string, vlanID uint16) { secondaryInterface := nmstate.NewPolicyBuilder(APIClient, policyName, map[string]string{ @@ -908,15 +913,6 @@ func createSecondaryInterfaceOnNode(policyName, nodeName, interfaceName, ipv4Add "fail to create secondary interface: %s.+%d", interfaceName, vlanID) } -func createStaticIPAnnotations(internalNADName, externalNADName string, internalIPAddresses, - externalIPAddresses []string) []*types.NetworkSelectionElement { - ipAnnotation := pod.StaticIPAnnotation(internalNADName, internalIPAddresses) - ipAnnotation = append(ipAnnotation, - pod.StaticIPAnnotation(externalNADName, externalIPAddresses)...) - - return ipAnnotation -} - func verifyReceivedRoutes(frrk8sPods []*pod.Builder, allowedPrefixes string) { By("Validate BGP received routes") diff --git a/tests/cnf/core/network/metallb/tests/layer2-test.go b/tests/cnf/core/network/metallb/tests/layer2-test.go index bade2d400..3fdd097e4 100644 --- a/tests/cnf/core/network/metallb/tests/layer2-test.go +++ b/tests/cnf/core/network/metallb/tests/layer2-test.go @@ -17,6 +17,7 @@ import ( "github.com/openshift-kni/eco-goinfra/pkg/pod" "github.com/openshift-kni/eco-goinfra/pkg/reportxml" "github.com/openshift-kni/eco-goinfra/pkg/service" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/define" . "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netinittools" "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netparam" "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/metallb/internal/cmd" @@ -79,7 +80,8 @@ var _ = Describe("Layer2", Ordered, Label(tsparams.LabelLayer2TestCases), Contin setupMetalLbService(netparam.IPV4Family, ipAddressPool, "Cluster") By("Creating external Network Attachment Definition") - createExternalNad(tsparams.ExternalMacVlanNADName) + err = define.CreateExternalNad(APIClient, tsparams.ExternalMacVlanNADName, tsparams.TestNamespaceName) + Expect(err).ToNot(HaveOccurred(), "Failed to create a network-attachment-definition") By("Creating client test pod") clientTestPod, err = pod.NewBuilder( diff --git a/tests/cnf/core/network/security/internal/tsparams/consts.go b/tests/cnf/core/network/security/internal/tsparams/consts.go new file mode 100644 index 000000000..6b715b4f5 --- /dev/null +++ b/tests/cnf/core/network/security/internal/tsparams/consts.go @@ -0,0 +1,27 @@ +package tsparams + +const ( + // LabelSuite represents nftables label that can be used for test cases selection. + LabelSuite = "nftables" + // LabelNftablesTestCases represents nftables custom firewall label that can be used for test cases selection. + LabelNftablesTestCases = "nftables-custom-rules" + // CustomFirewallDelete removes all the rules from the custom table. + // CustomFirewallDelete = ` + // table inet custom_table + // delete table inet custom_table + // table inet custom_table { + // }` + CustomFirewallDelete = `data:;base64, ` + + `H4sIAAAAAAAC/ypJTMpJVcjMSy1RSC4tLsnPjQeLcKWk5qSWpCrgksYhrlDNVcsFCAAA//9SII3uUwAAAA==` + // CustomFirewallIngressPort8888 adds an input rule blocking TCP port 8888. + // chain custom_chain_INPUT { + // type filter hook input priority 1; policy accept; + // # Drop TCP port 8888 and log + // tcp dport 8888 log prefix "[USERFIREWALL] PACKET DROP: " drop + CustomFirewallIngressPort8888 = `data:;base64,` + + `H4sIAAAAAAAC/3TMwUoDMRDG8XPyFB/1Cbwt9lTaFYpFl3WLB5ESk2k7GDNDnIKL` + + `9N2lBfG0x+//g8/CeyZwIUM8fZl87q7FJ8pkhCme6PjxLh4Dl796Hbv1Y7cdLuZs` + + `VMKes1HFUeQDXPRk0MpS2UbczqGSOY4IMZLa3Dt3g1UVxbDsoFINTdM0CCUhy+Fy` + + `GRXpH7IcoJX2/I3Z6/a57e/Xffuy2Gze0C2WD+2AVf/U3WGGVEW9O/uz/w0AAP//` + + `kU0CJQUBAAA=` +) diff --git a/tests/cnf/core/network/security/internal/tsparams/securityvars.go b/tests/cnf/core/network/security/internal/tsparams/securityvars.go new file mode 100644 index 000000000..97dcf8404 --- /dev/null +++ b/tests/cnf/core/network/security/internal/tsparams/securityvars.go @@ -0,0 +1,24 @@ +package tsparams + +import ( + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netparam" + "github.com/openshift-kni/k8sreporter" + mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" +) + +var ( + // Labels represents the range of labels that can be used for test cases selection. + Labels = append(netparam.Labels, LabelSuite) + + // ReporterNamespacesToDump tells to the reporter from where to collect logs. + ReporterNamespacesToDump = map[string]string{ + TestNamespaceName: TestNamespaceName, + } + + // TestNamespaceName security-tests namespace where all test cases are performed. + TestNamespaceName = "security-tests" + // ReporterCRDsToDump tells to the reporter what CRs to dump. + ReporterCRDsToDump = []k8sreporter.CRData{ + {Cr: &mcfgv1.MachineConfigList{}}, + } +) diff --git a/tests/cnf/core/network/security/security_suite_test.go b/tests/cnf/core/network/security/security_suite_test.go new file mode 100644 index 000000000..6295c1911 --- /dev/null +++ b/tests/cnf/core/network/security/security_suite_test.go @@ -0,0 +1,59 @@ +package security + +import ( + "runtime" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/openshift-kni/eco-goinfra/pkg/namespace" + "github.com/openshift-kni/eco-goinfra/pkg/reportxml" + . "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netinittools" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/security/internal/tsparams" + _ "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/security/tests" + "github.com/openshift-kni/eco-gotests/tests/internal/cluster" + "github.com/openshift-kni/eco-gotests/tests/internal/params" + "github.com/openshift-kni/eco-gotests/tests/internal/reporter" +) + +var ( + _, currentFile, _, _ = runtime.Caller(0) + testNS = namespace.NewBuilder(APIClient, tsparams.TestNamespaceName) +) + +func TestLB(t *testing.T) { + _, reporterConfig := GinkgoConfiguration() + reporterConfig.JUnitReport = NetConfig.GetJunitReportPath(currentFile) + + RegisterFailHandler(Fail) + RunSpecs(t, "security", Label(tsparams.Labels...), reporterConfig) +} + +var _ = BeforeSuite(func() { + By("Creating privileged test namespace") + for key, value := range params.PrivilegedNSLabels { + testNS.WithLabel(key, value) + } + + _, err := testNS.Create() + Expect(err).ToNot(HaveOccurred(), "error to create test namespace") + + By("Pulling test images on cluster before running test cases") + err = cluster.PullTestImageOnNodes(APIClient, NetConfig.WorkerLabel, NetConfig.CnfNetTestContainer, 300) + Expect(err).ToNot(HaveOccurred(), "Failed to pull test image on nodes") +}) + +var _ = AfterSuite(func() { + By("Deleting test namespace") + err := testNS.Delete() + Expect(err).ToNot(HaveOccurred(), "error to delete test namespace") +}) + +var _ = JustAfterEach(func() { + reporter.ReportIfFailed( + CurrentSpecReport(), currentFile, tsparams.ReporterNamespacesToDump, tsparams.ReporterCRDsToDump) +}) + +var _ = ReportAfterSuite("", func(report Report) { + reportxml.Create(report, NetConfig.GetReportPath(), NetConfig.TCPrefix) +}) diff --git a/tests/cnf/core/network/security/tests/nftables-custom-firewall-test.go b/tests/cnf/core/network/security/tests/nftables-custom-firewall-test.go new file mode 100644 index 000000000..b1af06cec --- /dev/null +++ b/tests/cnf/core/network/security/tests/nftables-custom-firewall-test.go @@ -0,0 +1,364 @@ +package tests + +import ( + "context" + "encoding/json" + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/openshift-kni/eco-goinfra/pkg/configmap" + "github.com/openshift-kni/eco-goinfra/pkg/mco" + "github.com/openshift-kni/eco-goinfra/pkg/nodes" + "github.com/openshift-kni/eco-goinfra/pkg/pod" + "github.com/openshift-kni/eco-goinfra/pkg/reportxml" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/cmd" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/define" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/frrconfig" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netenv" + . "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/internal/netinittools" + "github.com/openshift-kni/eco-gotests/tests/cnf/core/network/security/internal/tsparams" + ocpoperatorv1 "github.com/openshift/api/operator/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" + + ignition "github.com/coreos/ignition/v2/config/v3_4/types" + "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types" + apimachinerytype "k8s.io/apimachinery/pkg/types" +) + +var _ = Describe("nftables", Ordered, Label(tsparams.LabelNftablesTestCases), ContinueOnFailure, func() { + + var ( + hubIPv4ExternalAddresses = []string{"172.16.0.10", "172.16.0.11"} + hubIPv4Network = "172.16.0.0/24" + masterPodIPv4Address = "172.16.0.1" + portNum8888 = 8888 + cnfWorkerNodeList []*nodes.Builder + masterNodeList []*nodes.Builder + ipv4NodeAddrList []string + ipv4SecurityIPList []string + testPodWorker0 *pod.Builder + testPodWorker1 *pod.Builder + testPodList []*pod.Builder + routeMap map[string]string + mcNftablesName = "98-nftables-cnf-worker" + interfaceNameNet1 = "net1" + interfaceNameBrEx = "br-ex" + err error + ) + BeforeAll(func() { + By("List CNF worker nodes in cluster") + cnfWorkerNodeList, err = nodes.List(APIClient, + metav1.ListOptions{LabelSelector: labels.Set(NetConfig.WorkerLabelMap).String()}) + Expect(err).ToNot(HaveOccurred(), "Failed to discover worker nodes") + + By("Selecting worker node for Security tests") + ipv4NodeAddrList, err = nodes.ListExternalIPv4Networks( + APIClient, metav1.ListOptions{LabelSelector: labels.Set(NetConfig.WorkerLabelMap).String()}) + Expect(err).ToNot(HaveOccurred(), "Failed to collect external nodes ip addresses") + + By("Listing master nodes") + masterNodeList, err = nodes.List(APIClient, + metav1.ListOptions{LabelSelector: labels.Set(NetConfig.ControlPlaneLabelMap).String()}) + Expect(err).ToNot(HaveOccurred(), "Fail to list master nodes") + Expect(len(masterNodeList)).To(BeNumerically(">", 0), + "Failed to detect master nodes") + + By("Edit the machineconfiguration cluster to include NFTables") + updateMachineConfigurationNodeDisruptionPolicy() + + By("Create test pods on worker nodes") + testPodWorker0 = createTestPodOnWorkers(cnfWorkerNodeList[0].Definition.Name, portNum8888) + testPodWorker1 = createTestPodOnWorkers(cnfWorkerNodeList[1].Definition.Name, portNum8888) + testPodList = []*pod.Builder{testPodWorker0, testPodWorker1} + + By("Create a static route to the external Pod network on each worker node") + ipv4SecurityIPList, err = NetConfig.GetMetalLbVirIP() + Expect(err).ToNot(HaveOccurred(), "Failed to retrieve the ipv4SecurityIPList") + + routeMap, err = netenv.BuildRoutesMapWithSpecificRoutes(testPodList, cnfWorkerNodeList, ipv4SecurityIPList) + Expect(err).ToNot(HaveOccurred(), "Failed to create route map with specific routes") + + addDeleteStaticRouteOnWorkerNodes(testPodList, routeMap, "add", hubIPv4Network) + }) + + AfterAll(func() { + By("Remove the static route to the external Pod network on each worker node") + addDeleteStaticRouteOnWorkerNodes(testPodList, routeMap, "del", hubIPv4Network) + }) + + Context("custom firewall", func() { + AfterEach(func() { + By("Define and delete a NFTables custom rule") + createMCAndWaitforMCPStable(tsparams.CustomFirewallDelete, mcNftablesName) + err := mco.NewMCBuilder(APIClient, mcNftablesName).Delete() + Expect(err).ToNot(HaveOccurred(), "Failed to get the machineConfig") + }) + + It("Verify the creation of a new custom node firewall NFTables table with an ingress rule", + reportxml.ID("77412"), func() { + By("Setup test environment") + masterPod := setupRemoteMultiHopTest(ipv4SecurityIPList, hubIPv4ExternalAddresses, + ipv4NodeAddrList, cnfWorkerNodeList, masterNodeList) + + By("Verify ICMP connectivity between the master Pod and the test pods on the workers") + err := cmd.ICMPConnectivityCheck(masterPod, ipv4NodeAddrList, interfaceNameNet1) + Expect(err).ToNot(HaveOccurred(), "Failed to ping the worker nodes") + + By("Verify ingress TCP traffic over port 8888 between the master Pod and the test pods on the workers") + + err = cmd.ValidateTCPTraffic(masterPod, ipv4NodeAddrList, interfaceNameNet1, frrconfig.ContainerName, + portNum8888) + Expect(err).ToNot(HaveOccurred(), + "Failed to send ingress TCP traffic over port 8888 to the worker nodes") + + By("Verify egress TCP traffic over port 8888 between the test Pod on the worker and the master pod") + err = cmd.ValidateTCPTraffic(testPodWorker0, []string{masterPodIPv4Address}, + interfaceNameBrEx, "", portNum8888) + Expect(err).ToNot(HaveOccurred(), + "Failed to send egress TCP traffic over port 8888 to the pod on the master node") + + By("Define and create a NFTables custom rule blocking ingress TCP port 8888") + createMCAndWaitforMCPStable(tsparams.CustomFirewallIngressPort8888, mcNftablesName) + + By("Verify ingress TCP traffic is blocked over port 8888 between the master Pod and the test" + + " pods on the workers") + err = cmd.ValidateTCPTraffic(masterPod, ipv4NodeAddrList, interfaceNameNet1, frrconfig.ContainerName, + portNum8888) + Expect(err).To(HaveOccurred(), + "Successfully sent ingress TCP traffic over port 8888 to the worker nodes") + + By("Verify egress TCP traffic over port 8888 between the test Pod on the worker and the master pod") + err = cmd.ValidateTCPTraffic(testPodWorker0, []string{masterPodIPv4Address}, + interfaceNameBrEx, "", portNum8888) + Expect(err).ToNot(HaveOccurred(), + "Failed to send egress TCP traffic over port 8888 to the pod on the master node") + }) + }) +}) + +func updateMachineConfigurationNodeDisruptionPolicy() { + By("should update machineconfiguration cluster") + + jsonBytes := []byte(` + {"spec":{"nodeDisruptionPolicy": + {"files": [{"actions": + [{"restart": {"serviceName": "nftables.service"},"type": "Restart"}], + "path": "/etc/sysconfig/nftables.conf"}], + "units": + [{"actions": + [{"reload": {"serviceName":"nftables.service"},"type": "Reload"}, + {"type": "DaemonReload"}],"name": "nftables.service"}]}}}`) + + err := APIClient.Patch(context.TODO(), &ocpoperatorv1.MachineConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + }, + }, client.RawPatch(apimachinerytype.MergePatchType, jsonBytes)) + Expect(err).ToNot(HaveOccurred(), + "Failed to update the machineconfiguration cluster file") +} + +func setupRemoteMultiHopTest( + ipv4SecurityIPList []string, + hubIPv4ExternalAddresses []string, + ipv4NodeAddrList []string, + cnfWorkerNodeList []*nodes.Builder, + masterNodeList []*nodes.Builder) *pod.Builder { + var ( + externalMacVlanNADName = "external" + nadHubName = "nad-hub" + masterPodIPv4Prefix = "172.16.0.1/24" + ) + + By("Creating External NAD for master FRR pod") + + err := define.CreateExternalNad(APIClient, externalMacVlanNADName, tsparams.TestNamespaceName) + Expect(err).ToNot(HaveOccurred(), "Failed to create a network-attachment-definition") + + By("Creating External NAD for hub FRR pods") + + err = define.CreateExternalNad(APIClient, nadHubName, tsparams.TestNamespaceName) + Expect(err).ToNot(HaveOccurred(), "Failed to create a network-attachment-definition") + + By("Creating static ip annotation for hub0") + + hub0BRstaticIPAnnotation := frrconfig.CreateStaticIPAnnotations(externalMacVlanNADName, + nadHubName, + []string{fmt.Sprintf("%s/24", ipv4SecurityIPList[0])}, + []string{fmt.Sprintf("%s/24", hubIPv4ExternalAddresses[0])}) + + By("Creating static ip annotation for hub1") + + hub1BRstaticIPAnnotation := frrconfig.CreateStaticIPAnnotations(externalMacVlanNADName, nadHubName, + []string{fmt.Sprintf("%s/24", ipv4SecurityIPList[1])}, + []string{fmt.Sprintf("%s/24", hubIPv4ExternalAddresses[1])}) + + By("Creating Frr Hub pod configMap") + + hubConfigMap := createFrrConfigMap("hub-node-config", "") + + By("Creating FRR Hub pod on worker node 0") + + _ = createFrrPodTest("hub-pod-worker-0", + cnfWorkerNodeList[0].Object.Name, hubConfigMap.Definition.Name, hub0BRstaticIPAnnotation, false) + + By("Creating FRR Hub pod on worker node 1") + + _ = createFrrPodTest("hub-pod-worker-1", + cnfWorkerNodeList[1].Object.Name, hubConfigMap.Definition.Name, hub1BRstaticIPAnnotation, false) + + By("Creating configmap and Frr Master pod") + + frrConfigMapStaticRoutes := defineConfigMapWithStaticRouteAndNetwork(hubIPv4ExternalAddresses, + cmd.RemovePrefixFromIPList(ipv4NodeAddrList)) + masterConfigMap := createFrrConfigMap("master-configmap", frrConfigMapStaticRoutes) + + masterStaticIPAnnotation := frrconfig.CreateStaticIPAnnotations(externalMacVlanNADName, nadHubName, + []string{masterPodIPv4Prefix}, []string{""}) + + return createFrrPodTest("master-pod", masterNodeList[0].Definition.Name, + masterConfigMap.Definition.Name, masterStaticIPAnnotation, true) +} + +func createFrrConfigMap(name, configMapStaticRoutes string) *configmap.Builder { + configMapData := frrconfig.DefineBaseConfig(frrconfig.DaemonsFile, configMapStaticRoutes, "") + masterConfigMap, err := configmap.NewBuilder(APIClient, name, tsparams.TestNamespaceName). + WithData(configMapData).Create() + Expect(err).ToNot(HaveOccurred(), "Failed to create master config map") + + return masterConfigMap +} + +func createFrrPodTest( + name, nodeName, + configmapName string, + secondaryNetConfig []*types.NetworkSelectionElement, + masterPod bool) *pod.Builder { + var frrContainer *pod.ContainerBuilder + + By("Creating FRR master container in the test namespace") + + if masterPod { + frrContainer = pod.NewContainerBuilder( + "frr", NetConfig.CnfNetTestContainer, []string{"/bin/bash", "-c", + "testcmd -interface net1 -protocol tcp -port 8888 -listen"}) + } else { + frrContainer = pod.NewContainerBuilder( + "frr", NetConfig.CnfNetTestContainer, []string{"/bin/bash", "-c", "sleep INF"}) + } + + frrContainer.WithSecurityCapabilities([]string{"NET_ADMIN", "NET_RAW", "SYS_ADMIN"}, true) + frrCtr, err := frrContainer.GetContainerCfg() + Expect(err).ToNot(HaveOccurred(), "Failed to get container configuration") + + frrPod, err := pod.NewBuilder(APIClient, name, tsparams.TestNamespaceName, NetConfig.FrrImage). + DefineOnNode(nodeName). + WithTolerationToMaster(). + WithSecondaryNetwork(secondaryNetConfig). + RedefineDefaultCMD([]string{}). + WithAdditionalContainer(frrCtr). + WithLocalVolume(configmapName, "/etc/frr"). + WithPrivilegedFlag(). + CreateAndWaitUntilRunning(5 * time.Minute) + Expect(err).ToNot(HaveOccurred(), "Failed to create FRR test pod") + + return frrPod +} + +func defineConfigMapWithStaticRouteAndNetwork(hubPodIPs, nodeIPAddresses []string) string { + frrConfig := + fmt.Sprintf("ip route %s/32 %s\n", nodeIPAddresses[1], hubPodIPs[0]) + + fmt.Sprintf("ip route %s/32 %s\n!\n", nodeIPAddresses[0], hubPodIPs[1]) + + frrConfig += "!\nline vty\n!\nend\n" + + return frrConfig +} + +func createTestPodOnWorkers(nodeName string, portNum int) *pod.Builder { + testPod, err := pod.NewBuilder( + APIClient, "testpod-"+nodeName, tsparams.TestNamespaceName, NetConfig.CnfNetTestContainer). + DefineOnNode(nodeName).WithHostNetwork().WithHostPid(true). + RedefineDefaultCMD([]string{"/bin/bash", "-c", + fmt.Sprintf("testcmd -interface br-ex -protocol tcp -port %d -listen", portNum)}). + WithPrivilegedFlag().CreateAndWaitUntilRunning(180 * time.Second) + Expect(err).ToNot(HaveOccurred(), "Failed to create test pod") + + return testPod +} + +func createMCAndWaitforMCPStable(fileContentString, mcNftablesName string) { + truePointer := true + stringgzip := "gzip" + + mode := 384 + sysDContents := ` + [Unit] + Description=Netfilter Tables + Documentation=man:nft(8) + Wants=network-pre.target + Before=network-pre.target + [Service] + Type=oneshot + ProtectSystem=full + ProtectHome=true + ExecStart=/sbin/nft -f /etc/sysconfig/nftables.conf + ExecReload=/sbin/nft -f /etc/sysconfig/nftables.conf + ExecStop=/sbin/nft 'add table inet custom_table; delete table inet custom_table' + RemainAfterExit=yes + [Install] + WantedBy=multi-user.target` + ignitionConfig := ignition.Config{ + Ignition: ignition.Ignition{ + Version: "3.4.0", + }, + Systemd: ignition.Systemd{ + Units: []ignition.Unit{ + { + Enabled: &truePointer, + Name: "nftables.service", + Contents: &sysDContents, + }, + }, + }, + Storage: ignition.Storage{ + Files: []ignition.File{ + { + Node: ignition.Node{ + Overwrite: &truePointer, + Path: "/etc/sysconfig/nftables.conf", + }, + FileEmbedded1: ignition.FileEmbedded1{ + Contents: ignition.Resource{ + Compression: &stringgzip, + Source: &fileContentString, + }, + Mode: &mode, + }, + }, + }, + }, + } + finalIgnitionConfig, err := json.Marshal(ignitionConfig) + Expect(err).ToNot(HaveOccurred(), "Failed to serialize ignition config") + _, err = mco.NewMCBuilder(APIClient, mcNftablesName). + WithLabel("machineconfiguration.openshift.io/role", NetConfig.CnfMcpLabel). + WithRawConfig(finalIgnitionConfig). + Create() + Expect(err).ToNot(HaveOccurred(), "Failed to create nftables machine config") + + err = netenv.WaitForMcpStable(APIClient, 35*time.Minute, 30*time.Second, NetConfig.CnfMcpLabel) + Expect(err).ToNot(HaveOccurred(), "Failed to wait for MCP to be stable") +} + +func addDeleteStaticRouteOnWorkerNodes(testPodList []*pod.Builder, routeMap map[string]string, routeAction, + hubIPv4Network string) { + for _, testPod := range testPodList { + _, err := netenv.SetStaticRoute(testPod, routeAction, hubIPv4Network, "", routeMap) + Expect(err).ToNot(HaveOccurred(), "Failed to create or delete static route") + } +}