[chore] Use crosslink tidylist in make gotidy (attempt 2) (#37418)

#### Description

This is a second attempt at PR #37142, see that one for context.

The PR was reverted in #37173, because the result of `make tidylist` was
not a pure function of the repo state. The output of the tool depends on
all `go.mod` files present in the repo directory, including those that
may be `.gitignored`, such as the ones generated in `cmd/otelcontribcol`
and `cmd/oteltestbedcol`.

Since then, I [added a `--skip` flag to the
tool](open-telemetry/opentelemetry-go-build-tools#662),
which allows us to make the tool ignore these problematic `go.mod`
files.

Automatic detection of `.gitignored` files may be desirable if some
developers have other non-gitted Go modules inside their repo directory.
As this would require more advanced Git integration in the tool and
should be easily avoided, I've decided to keep this more manual
approach.

#### Link to tracking issue

Resolves #37336.

See previous PR for impacted issues in core.

#### Testing

See previous PR for initial testing.

I've tried running the tool locally, with and without the generated
files in `cmd/otelcontribcol`, and the output was identical, confirming
that the previous issue should be fixed.

---------

Co-authored-by: Pablo Baeyens <pbaeyens31+github@gmail.com>

.github/workflows/build-and-test.yml

@@ -221,6 +221,10 @@ jobs:
         run: |
           make crosslink
           git diff --exit-code || (echo 'Replace statements are out of date, please run "make crosslink" and commit the changes in this PR.' && exit 1)
+      - name: tidylist
+        run: |
+          make tidylist
+          git diff --exit-code || (echo 'Tidylist is out of date, please run "make tidylist" and commit the changes in this PR.' && exit 1)
       - name: Check for go mod dependency changes
         run: |
           make gotidy

Makefile

@@ -114,9 +114,23 @@ stability-tests: otelcontribcol
 gogci:
 	$(MAKE) $(FOR_GROUP_TARGET) TARGET="gci"
 
+.PHONY: tidylist
+tidylist: $(CROSSLINK)
+	cd internal/tidylist && \
+	$(CROSSLINK) tidylist \
+		--validate \
+		--allow-circular allow-circular.txt \
+		--skip cmd/otelcontribcol/go.mod \
+		--skip cmd/oteltestbedcol/go.mod \
+		tidylist.txt
+
+# internal/tidylist/tidylist.txt lists modules in topological order, to ensure `go mod tidy` converges.
 .PHONY: gotidy
 gotidy:
-	$(MAKE) $(FOR_GROUP_TARGET) TARGET="tidy"
+	@for mod in $$(cat internal/tidylist/tidylist.txt); do \
+		echo "Tidying $$mod"; \
+		(cd $$mod && rm -rf go.sum && $(GOCMD) mod tidy -compat=1.22.0) || exit $?; \
+	done
 
 .PHONY: remove-toolchain
 remove-toolchain:

internal/tidylist/allow-circular.txt

@@ -0,0 +1,12 @@
+# This file lists modules that are known to have intra-repository circular dependencies.
+# The `make tidylist` command will check against this list and error out if circular dependencies
+# are accidentally added or removed.
+
+# exporter/datadog <-> connector/datadog
+exporter/datadogexporter
+connector/datadogconnector
+
+# receiver/otelarrow <-> internal/otelarrow <-> exporter/otelarrow
+receiver/otelarrowreceiver
+exporter/otelarrowexporter
+internal/otelarrow

internal/tidylist/tidylist.txt

@@ -0,0 +1,305 @@
+cmd/githubgen
+cmd/opampsupervisor
+cmd/telemetrygen
+internal/common
+cmd/telemetrygen/internal/e2etest
+confmap/provider/aesprovider
+confmap/provider/s3provider
+confmap/provider/secretsmanagerprovider
+pkg/pdatautil
+pkg/golden
+pkg/pdatatest
+internal/coreinternal
+pkg/ottl
+internal/filter
+connector/countconnector
+internal/aws/ecsutil
+internal/k8sconfig
+internal/metadataproviders
+pkg/resourcetotelemetry
+pkg/xk8stest
+processor/k8sattributesprocessor
+pkg/core/xidutils
+pkg/sampling
+processor/probabilisticsamplerprocessor
+processor/resourcedetectionprocessor
+internal/pdatautil
+processor/transformprocessor
+internal/docker
+receiver/dockerstatsreceiver
+extension/storage
+pkg/stanza
+receiver/filelogreceiver
+pkg/experimentalmetricmetadata
+receiver/hostmetricsreceiver
+pkg/translator/prometheus
+pkg/translator/prometheusremotewrite
+exporter/prometheusremotewriteexporter
+receiver/prometheusreceiver
+pkg/datadog
+processor/tailsamplingprocessor
+exporter/datadogexporter
+connector/datadogconnector
+exporter/datadogexporter
+connector/exceptionsconnector
+connector/failoverconnector
+connector/grafanacloudconnector
+connector/otlpjsonconnector
+connector/roundrobinconnector
+connector/routingconnector
+connector/servicegraphconnector
+connector/signaltometricsconnector
+connector/spanmetricsconnector
+connector/sumconnector
+examples/demo/client
+examples/demo/server
+exporter/alertmanagerexporter
+exporter/alibabacloudlogserviceexporter
+internal/aws/awsutil
+internal/aws/cwlogs
+exporter/awscloudwatchlogsexporter
+internal/aws/metrics
+exporter/awsemfexporter
+pkg/translator/jaeger
+pkg/translator/zipkin
+exporter/awskinesisexporter
+exporter/awss3exporter
+internal/aws/xray
+exporter/awsxrayexporter
+exporter/azuredataexplorerexporter
+exporter/azuremonitorexporter
+exporter/bmchelixexporter
+exporter/carbonexporter
+exporter/cassandraexporter
+exporter/clickhouseexporter
+exporter/coralogixexporter
+exporter/datadogexporter/integrationtest
+exporter/datasetexporter
+exporter/dorisexporter
+exporter/elasticsearchexporter
+extension/storage/filestorage
+internal/sharedcomponent
+pkg/translator/opencensus
+receiver/opencensusreceiver
+exporter/opencensusexporter
+exporter/prometheusexporter
+internal/splunk
+pkg/batchperresourceattr
+exporter/sapmexporter
+pkg/translator/signalfx
+exporter/signalfxexporter
+exporter/syslogexporter
+receiver/zipkinreceiver
+exporter/zipkinexporter
+receiver/carbonreceiver
+internal/exp/metrics
+receiver/datadogreceiver
+receiver/jaegerreceiver
+receiver/sapmreceiver
+receiver/signalfxreceiver
+exporter/splunkhecexporter
+extension/ackextension
+receiver/splunkhecreceiver
+receiver/syslogreceiver
+testbed/mockdatasenders/mockdatadogagentexporter
+testbed
+exporter/elasticsearchexporter/integrationtest
+extension/encoding
+extension/encoding/otlpencodingextension
+exporter/fileexporter
+exporter/googlecloudexporter
+exporter/googlecloudpubsubexporter
+exporter/googlemanagedprometheusexporter
+exporter/honeycombmarkerexporter
+exporter/influxdbexporter
+internal/kafka
+pkg/batchpersignal
+pkg/kafka/topic
+exporter/kafkaexporter
+exporter/kineticaexporter
+exporter/loadbalancingexporter
+exporter/logicmonitorexporter
+exporter/logzioexporter
+pkg/translator/loki
+exporter/lokiexporter
+exporter/mezmoexporter
+exporter/opensearchexporter
+internal/grpcutil
+receiver/otelarrowreceiver
+internal/otelarrow
+exporter/otelarrowexporter
+receiver/otelarrowreceiver
+internal/otelarrow
+exporter/otelarrowexporter
+receiver/otelarrowreceiver
+exporter/pulsarexporter
+internal/rabbitmq
+exporter/rabbitmqexporter
+exporter/sematextexporter
+exporter/sentryexporter
+exporter/stefexporter
+extension/sumologicextension
+exporter/sumologicexporter
+exporter/tencentcloudlogserviceexporter
+extension/asapauthextension
+internal/aws/proxy
+extension/awsproxy
+extension/basicauthextension
+extension/bearertokenauthextension
+extension/cgroupruntimeextension
+extension/encoding/avrologencodingextension
+extension/encoding/jaegerencodingextension
+extension/encoding/jsonlogencodingextension
+pkg/translator/skywalking
+extension/encoding/skywalkingencodingextension
+extension/encoding/textencodingextension
+extension/encoding/zipkinencodingextension
+extension/googleclientauthextension
+extension/headerssetterextension
+extension/healthcheckextension
+pkg/status
+extension/healthcheckv2extension
+extension/httpforwarderextension
+extension/jaegerremotesampling
+extension/oauth2clientauthextension
+extension/observer
+extension/observer/cfgardenobserver
+extension/observer/dockerobserver
+extension/observer/ecsobserver
+extension/observer/ecstaskobserver
+extension/observer/hostobserver
+extension/observer/k8sobserver
+extension/oidcauthextension
+extension/opampcustommessages
+extension/opampextension
+extension/pprofextension
+extension/remotetapextension
+extension/sigv4authextension
+extension/solarwindsapmsettingsextension
+extension/storage/dbstorage
+extension/storage/redisstorageextension
+.
+internal/aws/containerinsight
+internal/aws/k8s
+internal/aws/xray/testdata/sampleapp
+internal/aws/xray/testdata/sampleserver
+internal/collectd
+internal/kubelet
+internal/sqlquery
+internal/tools
+pkg/translator/azure
+pkg/translator/azurelogs
+pkg/winperfcounters
+processor/attributesprocessor
+processor/coralogixprocessor
+processor/cumulativetodeltaprocessor
+processor/deltatocumulativeprocessor
+processor/deltatorateprocessor
+processor/filterprocessor
+processor/geoipprocessor
+processor/groupbyattrsprocessor
+processor/groupbytraceprocessor
+processor/intervalprocessor
+processor/logdedupprocessor
+processor/logstransformprocessor
+processor/metricsgenerationprocessor
+processor/metricstarttimeprocessor
+processor/metricstransformprocessor
+processor/redactionprocessor
+processor/remotetapprocessor
+processor/resourceprocessor
+processor/routingprocessor
+processor/schemaprocessor
+processor/spanprocessor
+processor/sumologicprocessor
+receiver/activedirectorydsreceiver
+receiver/aerospikereceiver
+receiver/apachereceiver
+receiver/apachesparkreceiver
+receiver/awscloudwatchmetricsreceiver
+receiver/awscloudwatchreceiver
+receiver/awscontainerinsightreceiver
+receiver/awsecscontainermetricsreceiver
+receiver/awsfirehosereceiver
+receiver/awss3receiver
+receiver/awsxrayreceiver
+receiver/azureblobreceiver
+receiver/azureeventhubreceiver
+receiver/azuremonitorreceiver
+receiver/bigipreceiver
+receiver/chronyreceiver
+receiver/cloudflarereceiver
+receiver/cloudfoundryreceiver
+receiver/collectdreceiver
+receiver/couchdbreceiver
+receiver/elasticsearchreceiver
+receiver/envoyalsreceiver
+receiver/expvarreceiver
+receiver/filestatsreceiver
+receiver/flinkmetricsreceiver
+receiver/fluentforwardreceiver
+receiver/githubreceiver
+receiver/gitlabreceiver
+receiver/googlecloudmonitoringreceiver
+receiver/googlecloudpubsubreceiver
+receiver/googlecloudspannerreceiver
+receiver/haproxyreceiver
+receiver/httpcheckreceiver
+receiver/huaweicloudcesreceiver
+receiver/iisreceiver
+receiver/influxdbreceiver
+receiver/jmxreceiver
+receiver/journaldreceiver
+receiver/k8sclusterreceiver
+receiver/k8seventsreceiver
+receiver/k8sobjectsreceiver
+receiver/kafkametricsreceiver
+receiver/kafkareceiver
+receiver/kubeletstatsreceiver
+receiver/libhoneyreceiver
+receiver/lokireceiver
+receiver/memcachedreceiver
+receiver/mongodbatlasreceiver
+receiver/mongodbreceiver
+receiver/mysqlreceiver
+receiver/namedpipereceiver
+receiver/netflowreceiver
+receiver/nginxreceiver
+receiver/nsxtreceiver
+receiver/ntpreceiver
+receiver/oracledbreceiver
+receiver/osqueryreceiver
+receiver/otlpjsonfilereceiver
+receiver/podmanreceiver
+receiver/postgresqlreceiver
+receiver/prometheusremotewritereceiver
+receiver/pulsarreceiver
+receiver/purefareceiver
+receiver/purefbreceiver
+receiver/rabbitmqreceiver
+receiver/receivercreator
+receiver/redisreceiver
+receiver/riakreceiver
+receiver/saphanareceiver
+receiver/simpleprometheusreceiver/examples/federation/prom-counter
+receiver/simpleprometheusreceiver
+receiver/skywalkingreceiver
+receiver/snmpreceiver
+receiver/snowflakereceiver
+receiver/solacereceiver
+receiver/splunkenterprisereceiver
+receiver/sqlqueryreceiver
+receiver/sqlserverreceiver
+receiver/sshcheckreceiver
+receiver/statsdreceiver
+receiver/systemdreceiver
+receiver/tcplogreceiver
+receiver/tlscheckreceiver
+receiver/udplogreceiver
+receiver/vcenterreceiver
+receiver/wavefrontreceiver
+receiver/webhookeventreceiver
+receiver/windowseventlogreceiver
+receiver/windowsperfcountersreceiver
+scraper/zookeeperscraper
+receiver/zookeeperreceiver