feat(internal/uploader): UDENG-5706 ubuntu insights uploader

cmd/insights/commands/upload.go

@@ -4,6 +4,7 @@ import (
 	"log/slog"
 
 	"github.com/spf13/cobra"
+	"github.com/ubuntu/ubuntu-insights/internal/constants"
 )
 
 type uploadConfig struct {
@@ -16,7 +17,7 @@ type uploadConfig struct {
 
 var defaultUploadConfig = uploadConfig{
 	sources: []string{""},
-	server:  "https://metrics.ubuntu.com",
+	server:  constants.DefaultServerURL,
 	minAge:  604800,
 	force:   false,
 	dryRun:  false,

internal/consent/consent.go

@@ -30,12 +30,12 @@ func New(path string) *Manager {
 	return &Manager{path: path}
 }
 
-// GetConsentState gets the consent state for the given source.
+// GetState gets the consent state for the given source.
 // If the source do not have a consent file, it will be considered as a false state.
 // If the source is an empty string, then the global consent state will be returned.
 // If the target consent file does not exist, it will not be created.
-func (cm Manager) GetConsentState(source string) (bool, error) {
-	sourceConsent, err := readConsentFile(cm.getConsentFile(source))
+func (cm Manager) GetState(source string) (bool, error) {
+	sourceConsent, err := readFile(cm.getFile(source))
 	if err != nil {
 		slog.Error("Error reading source consent file", "source", source, "error", err)
 		return false, err
@@ -46,20 +46,33 @@ func (cm Manager) GetConsentState(source string) (bool, error) {
 
 var consentSourceFilePattern = `%s` + constants.ConsentSourceBaseSeparator + constants.GlobalFileName
 
-// SetConsentState updates the consent state for the given source.
+// SetState updates the consent state for the given source.
 // If the source is an empty string, then the global consent state will be set.
 // If the target consent file does not exist, it will be created.
-func (cm Manager) SetConsentState(source string, state bool) (err error) {
+func (cm Manager) SetState(source string, state bool) (err error) {
 	defer decorate.OnError(&err, "could not set consent state")
 
 	consent := consentFile{ConsentState: state}
-	return consent.write(cm.getConsentFile(source))
+	return consent.write(cm.getFile(source))
 }
 
-// getConsentFile returns the expected path to the consent file for the given source.
+// HasConsent returns true if there is consent for the given source, based on the hierarchy rules.
+// If the source has a consent file, its value is returned.
+// Otherwise, the global consent state is returned.
+func (cm Manager) HasConsent(source string) (bool, error) {
+	consent, err := cm.GetState(source)
+	if err != nil {
+		slog.Warn("Could not get source specific consent state, falling back to global consent state", "source", source, "error", err)
+		return cm.GetState("")
+	}
+
+	return consent, nil
+}
+
+// getFile returns the expected path to the consent file for the given source.
 // If source is blank, it returns the path to the global consent file.
 // It does not check if the file exists, or if it is valid.
-func (cm Manager) getConsentFile(source string) string {
+func (cm Manager) getFile(source string) string {
 	p := filepath.Join(cm.path, constants.GlobalFileName)
 	if source != "" {
 		p = filepath.Join(cm.path, fmt.Sprintf(consentSourceFilePattern, source))
@@ -69,7 +82,7 @@ func (cm Manager) getConsentFile(source string) string {
 }
 
 // getSourceConsentFiles returns a map of all paths to validly named consent files in the folder, other than the global file.
-func (cm Manager) getConsentFiles() (map[string]string, error) {
+func (cm Manager) getFiles() (map[string]string, error) {
 	sourceFiles := make(map[string]string)
 
 	entries, err := os.ReadDir(cm.path)
@@ -94,7 +107,7 @@ func (cm Manager) getConsentFiles() (map[string]string, error) {
 	return sourceFiles, nil
 }
 
-func readConsentFile(path string) (consentFile, error) {
+func readFile(path string) (consentFile, error) {
 	var consent consentFile
 	_, err := toml.DecodeFile(path, &consent)
 	slog.Debug("Read consent file", "file", path, "consent", consent.ConsentState)

internal/consent/consent_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/ubuntu/ubuntu-insights/internal/testutils"
 )
 
-func TestGetConsentState(t *testing.T) {
+func TestGetState(t *testing.T) {
 	t.Parallel()
 
 	tests := map[string]struct {
@@ -55,10 +55,9 @@ func TestGetConsentState(t *testing.T) {
 			t.Parallel()
 			dir, err := setupTmpConsentFiles(t, tc.globalFile)
 			require.NoError(t, err, "Setup: failed to setup temporary consent files")
-			defer testutils.CleanupDir(t, dir)
 			cm := consent.New(dir)
 
-			got, err := cm.GetConsentState(tc.source)
+			got, err := cm.GetState(tc.source)
 			if tc.wantErr {
 				require.Error(t, err, "expected an error but got none")
 				return
@@ -71,7 +70,7 @@ func TestGetConsentState(t *testing.T) {
 	}
 }
 
-func TestSetConsentStates(t *testing.T) {
+func TestSetState(t *testing.T) {
 	t.Parallel()
 
 	tests := map[string]struct {
@@ -112,10 +111,9 @@ func TestSetConsentStates(t *testing.T) {
 			t.Parallel()
 			dir, err := setupTmpConsentFiles(t, tc.globalFile)
 			require.NoError(t, err, "Setup: failed to setup temporary consent files")
-			defer testutils.CleanupDir(t, dir)
 			cm := consent.New(dir)
 
-			err = cm.SetConsentState(tc.writeSource, tc.writeState)
+			err = cm.SetState(tc.writeSource, tc.writeState)
 			if tc.wantErr {
 				require.Error(t, err, "expected an error but got none")
 				return
@@ -135,23 +133,69 @@ func TestSetConsentStates(t *testing.T) {
 	}
 }
 
+func TestHasConsent(t *testing.T) {
+	t.Parallel()
+
+	tests := map[string]struct {
+		source string
+
+		globalFile string
+
+		want    bool
+		wantErr bool
+	}{
+		"True Global-True Source":          {source: "valid_true", globalFile: "valid_true-consent.toml", want: true},
+		"True Global-False Source":         {source: "valid_false", globalFile: "valid_true-consent.toml", want: false},
+		"True Global-Invalid Value Source": {source: "invalid_value", globalFile: "valid_true-consent.toml", want: true},
+		"True Global-Invalid File Source":  {source: "invalid_file", globalFile: "valid_true-consent.toml", want: true},
+		"True Global-Not A File Source":    {source: "not_a_file", globalFile: "valid_true-consent.toml", want: true},
+
+		"False Global-True Source":          {source: "valid_true", globalFile: "valid_false-consent.toml", want: true},
+		"False Global-False Source":         {source: "valid_false", globalFile: "valid_false-consent.toml", want: false},
+		"False Global-Invalid Value Source": {source: "invalid_value", globalFile: "valid_false-consent.toml", want: false},
+		"False Global-Invalid File Source":  {source: "invalid_file", globalFile: "valid_false-consent.toml", want: false},
+		"False Global-Not A File Source":    {source: "not_a_file", globalFile: "valid_false-consent.toml", want: false},
+
+		"No Global-True Source":          {source: "valid_true", want: true},
+		"No Global-False Source":         {source: "valid_false", want: false},
+		"No Global-Invalid Value Source": {source: "invalid_value", wantErr: true},
+		"No Global-Invalid File Source":  {source: "invalid_file", wantErr: true},
+		"No Global-Not A File Source":    {source: "not_a_file", wantErr: true},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			dir, err := setupTmpConsentFiles(t, tc.globalFile)
+			require.NoError(t, err, "Setup: failed to setup temporary consent files")
+			cm := consent.New(dir)
+
+			got, err := cm.HasConsent(tc.source)
+			if tc.wantErr {
+				require.Error(t, err, "expected an error but got none")
+				return
+			}
+			require.NoError(t, err, "got an unexpected error")
+
+			require.Equal(t, tc.want, got, "HasConsent should return expected consent state")
+		})
+	}
+}
+
 func setupTmpConsentFiles(t *testing.T, globalFile string) (string, error) {
 	t.Helper()
 
 	// Setup temporary directory
 	var err error
-	dir, err := os.MkdirTemp("", "consent-files")
-	if err != nil {
-		return dir, fmt.Errorf("failed to create temporary directory: %v", err)
-	}
+	dir := t.TempDir()
 
-	if err = testutils.CopyDir(filepath.Join("testdata", "consent_files"), dir); err != nil {
+	if err = testutils.CopyDir(t, filepath.Join("testdata", "consent_files"), dir); err != nil {
 		return dir, fmt.Errorf("failed to copy testdata directory to temporary directory: %v", err)
 	}
 
 	// Setup globalFile if provided
 	if globalFile != "" {
-		if err = testutils.CopyFile(filepath.Join(dir, globalFile), filepath.Join(dir, "consent.toml")); err != nil {
+		if err = testutils.CopyFile(t, filepath.Join(dir, globalFile), filepath.Join(dir, "consent.toml")); err != nil {
 			return dir, fmt.Errorf("failed to copy requested global consent file: %v", err)
 		}
 	}

internal/consent/export_test.go

@@ -4,14 +4,14 @@ package consent
 // It does not get the global consent state.
 // If continueOnErr is true, it will continue to the next source if an error occurs.
 func (cm Manager) GetAllSourceConsentStates(continueOnErr bool) (map[string]bool, error) {
-	p, err := cm.getConsentFiles()
+	p, err := cm.getFiles()
 	if err != nil {
 		return nil, err
 	}
 
 	consentStates := make(map[string]bool)
 	for source, path := range p {
-		consent, err := readConsentFile(path)
+		consent, err := readFile(path)
 		if err != nil && !continueOnErr {
 			return nil, err
 		}

internal/constants/constants.go

@@ -18,6 +18,15 @@ const (
 	// DefaultLogLevel is the default log level selected without any verbosity flags.
 	DefaultLogLevel = slog.LevelInfo
 
+	// DefaultServerURL is the default base URL for the server that reports are uploaded to.
+	DefaultServerURL = "https://metrics.ubuntu.com"
+
+	// LocalFolder is the default name of the local collected reports folder.
+	LocalFolder = "local"
+
+	// UploadedFolder is the default name of the uploaded reports folder.
+	UploadedFolder = "uploaded"
+
 	// GlobalFileName is the default base name of the consent state files.
 	GlobalFileName = "consent.toml"
 
@@ -28,6 +37,9 @@ const (
 	ReportExt = ".json"
 )
 
+// OptOutJSON is the data sent in case of Opt-Out choice.
+var OptOutJSON = struct{ OptOut bool }{true}
+
 type options struct {
 	baseDir func() (string, error)
 }

internal/fileutils/fileutils.go

@@ -0,0 +1,38 @@
+// Package fileutils provides utility functions for handling files.
+package fileutils
+
+import (
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+)
+
+// AtomicWrite writes data to a file atomically.
+// If the file already exists, then it will be overwritten.
+// Not atomic on Windows.
+func AtomicWrite(path string, data []byte) error {
+	tmp, err := os.CreateTemp(filepath.Dir(path), "tmp-*.tmp")
+	if err != nil {
+		return fmt.Errorf("could not create temporary file: %v", err)
+	}
+	defer func() {
+		_ = tmp.Close()
+		if err := os.Remove(tmp.Name()); err != nil && !os.IsNotExist(err) {
+			slog.Warn("Failed to remove temporary file", "file", tmp.Name(), "error", err)
+		}
+	}()
+
+	if _, err := tmp.Write(data); err != nil {
+		return fmt.Errorf("could not write to temporary file: %v", err)
+	}
+
+	if err := tmp.Close(); err != nil {
+		return fmt.Errorf("could not close temporary file: %v", err)
+	}
+
+	if err := os.Rename(tmp.Name(), path); err != nil {
+		return fmt.Errorf("could not rename temporary file: %v", err)
+	}
+	return nil
+}

internal/fileutils/fileutils_test.go

@@ -0,0 +1,81 @@
+package fileutils_test
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/ubuntu/ubuntu-insights/internal/fileutils"
+)
+
+func TestAtomicWrite(t *testing.T) {
+	t.Parallel()
+
+	tests := map[string]struct {
+		data            []byte
+		fileExists      bool
+		fileExistsPerms os.FileMode
+		invalidDir      bool
+
+		wantError bool
+	}{
+		"Empty file":          {data: []byte{}},
+		"Non-empty file":      {data: []byte("data")},
+		"Override file":       {data: []byte("data"), fileExistsPerms: 0600, fileExists: true},
+		"Override empty file": {data: []byte{}, fileExistsPerms: 0600, fileExists: true},
+
+		"Existing empty file":     {data: []byte{}, fileExistsPerms: 0600, fileExists: true},
+		"Existing non-empty file": {data: []byte("data"), fileExistsPerms: 0600, fileExists: true},
+
+		"Override read-only file": {data: []byte("data"), fileExistsPerms: 0400, fileExists: true, wantError: runtime.GOOS == "windows"},
+		"Override No Perms file":  {data: []byte("data"), fileExistsPerms: 0000, fileExists: true, wantError: runtime.GOOS == "windows"},
+		"Invalid Dir":             {data: []byte("data"), invalidDir: true, wantError: true},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			oldFile := []byte("Old File!")
+			tempDir := t.TempDir()
+			path := filepath.Join(tempDir, "file")
+			if tc.invalidDir {
+				path = filepath.Join(path, "fake_dir")
+			}
+
+			if tc.fileExists {
+				err := os.WriteFile(path, oldFile, tc.fileExistsPerms)
+				require.NoError(t, err, "Setup: WriteFile should not return an error")
+				t.Cleanup(func() { _ = os.Chmod(path, 0600) })
+			}
+
+			err := fileutils.AtomicWrite(path, tc.data)
+			if tc.wantError {
+				require.Error(t, err, "AtomicWrite should return an error")
+
+				// Check that the file was not overwritten
+				if !tc.fileExists {
+					return
+				}
+
+				if tc.invalidDir {
+					path = filepath.Dir(path)
+				}
+
+				data, err := os.ReadFile(path)
+				require.NoError(t, err, "ReadFile should not return an error")
+				require.Equal(t, oldFile, data, "AtomicWrite should not overwrite the file")
+
+				return
+			}
+			require.NoError(t, err, "AtomicWrite should not return an error")
+
+			// Check that the file was written
+			data, err := os.ReadFile(path)
+			require.NoError(t, err, "ReadFile should not return an error")
+			require.Equal(t, tc.data, data, "AtomicWrite should write the data to the file")
+		})
+	}
+}

internal/report/export_test.go

@@ -0,0 +1,10 @@
+package report
+
+type ReportStash = reportStash
+
+// ReportStash returns the reportStash of the report.
+//
+//nolint:revive // This is a false positive as we returned a typed alias and not the private type.
+func (r Report) ReportStash() ReportStash {
+	return r.reportStash
+}