Add command for running VCR in EAP

.ci/magician/cmd/test_eap_vcr.go

@@ -0,0 +1,245 @@
+package cmd
+
+import (
+	_ "embed"
+	"fmt"
+	"magician/exec"
+	"magician/provider"
+	"magician/vcr"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/spf13/cobra"
+)
+
+var tevEnvironmentVariables = [...]string{
+	"GENPATH",
+	"GOCACHE",
+	"GOPATH",
+	"GOOGLE_REGION",
+	"GOOGLE_ZONE",
+	"ORG_ID",
+	"GOOGLE_PROJECT",
+	"GOOGLE_BILLING_ACCOUNT",
+	"GOOGLE_ORG",
+	"GOOGLE_ORG_DOMAIN",
+	"GOOGLE_PROJECT_NUMBER",
+	"GOOGLE_USE_DEFAULT_CREDENTIALS",
+	"GOOGLE_IMPERSONATE_SERVICE_ACCOUNT",
+	"KOKORO_ARTIFACTS_DIR",
+	"HOME",
+	"MODIFIED_FILE_PATH",
+	"PATH",
+	"USER",
+}
+
+var testEAPVCRCmd = &cobra.Command{
+	Use:   "test-eap-vcr",
+	Short: "Run vcr tests for affected packages in EAP",
+	Long: `This command runs on new change lists to replay VCR cassettes and re-record failing cassettes.
+
+	The following environment variables are required:
+` + listTEVEnvironmentVariables(),
+	RunE: func(cmd *cobra.Command, args []string) error {
+		env := make(map[string]string, len(tevEnvironmentVariables))
+		for _, ev := range tevEnvironmentVariables {
+			val, ok := os.LookupEnv(ev)
+			if !ok {
+				return fmt.Errorf("did not provide %s environment variable", ev)
+			}
+			env[ev] = val
+		}
+		rnr, err := exec.NewRunner()
+		if err != nil {
+			return err
+		}
+		vt, err := vcr.NewTester(env, "ci-vcr-cassettes", "ci-vcr-logs", rnr)
+		if err != nil {
+			return err
+		}
+		return execTestEAPVCR(args[0], env["GEN_PATH"], env["KOKORO_ARTIFACTS_DIR"], env["MODIFIED_FILE_PATH"], rnr, vt)
+	},
+}
+
+func listTEVEnvironmentVariables() string {
+	var result string
+	for i, ev := range tevEnvironmentVariables {
+		result += fmt.Sprintf("\t%2d. %s\n", i+1, ev)
+	}
+	return result
+}
+
+func execTestEAPVCR(changeNumber, genPath, kokoroArtifactsDir, modifiedFilePath string, rnr ExecRunner, vt *vcr.Tester) error {
+	vt.SetRepoPath(provider.Alpha, genPath)
+	if err := rnr.PushDir(genPath); err != nil {
+		return fmt.Errorf("error changing to gen path: %w", err)
+	}
+
+	changedFiles, err := rnr.Run("git", []string{"diff", "--name-only"}, nil)
+	if err != nil {
+		return fmt.Errorf("error diffing gen path: %w", err)
+	}
+
+	services, runFullVCR := modifiedPackages(strings.Split(changedFiles, "\n"), provider.Alpha)
+	if len(services) == 0 && !runFullVCR {
+		fmt.Println("Skipping tests: No go files or test fixtures changed")
+		return nil
+	}
+	fmt.Println("Running tests: Go files or test fixtures changed")
+
+	head := "auto-cl-" + changeNumber
+	if err := vt.FetchCassettes(provider.Alpha, "main", head); err != nil {
+		return fmt.Errorf("error fetching cassettes: %w", err)
+	}
+	replayingResult, testDirs, replayingErr := runReplaying(runFullVCR, provider.Alpha, services, vt)
+	if err := vt.UploadLogs(vcr.UploadLogsOptions{
+		Head:    head,
+		Mode:    vcr.Replaying,
+		Version: provider.Alpha,
+	}); err != nil {
+		return fmt.Errorf("error uploading replaying logs: %w", err)
+	}
+
+	if hasPanics, err := handleEAPVCRPanics(head, kokoroArtifactsDir, modifiedFilePath, replayingResult, vcr.Replaying, rnr); err != nil {
+		return fmt.Errorf("error handling panics: %w", err)
+	} else if hasPanics {
+		return nil
+	}
+
+	var servicesArr []string
+	for s := range services {
+		if _, ok := allowedAlphaServices[s]; ok {
+			servicesArr = append(servicesArr, s)
+		}
+	}
+	analyticsData := analytics{
+		ReplayingResult:  replayingResult,
+		RunFullVCR:       runFullVCR,
+		AffectedServices: sort.StringSlice(servicesArr),
+	}
+	testsAnalyticsComment, err := formatTestsAnalytics(analyticsData)
+	if err != nil {
+		return fmt.Errorf("error formatting test_analytics comment: %w", err)
+	}
+	if len(replayingResult.FailedTests) > 0 {
+		withReplayFailedTestsData := withReplayFailedTests{
+			ReplayingResult: replayingResult,
+		}
+
+		withReplayFailedTestsComment, err := formatWithReplayFailedTests(withReplayFailedTestsData)
+		if err != nil {
+			return fmt.Errorf("error formatting action taken comment: %w", err)
+		}
+		comment := strings.Join([]string{testsAnalyticsComment, withReplayFailedTestsComment}, "\n")
+		if err := postGerritComment(kokoroArtifactsDir, modifiedFilePath, comment, rnr); err != nil {
+			return fmt.Errorf("error posting comment: %w", err)
+		}
+
+		recordingResult, recordingErr := vt.RunParallel(vcr.RunOptions{
+			Mode:     vcr.Recording,
+			Version:  provider.Beta,
+			TestDirs: testDirs,
+			Tests:    replayingResult.FailedTests,
+		})
+
+		if hasPanics, err := handleEAPVCRPanics(head, kokoroArtifactsDir, modifiedFilePath, recordingResult, vcr.Recording, rnr); err != nil {
+			return fmt.Errorf("error handling panics: %w", err)
+		} else if hasPanics {
+			return nil
+		}
+		replayingAfterRecordingResult := vcr.Result{}
+		if len(recordingResult.PassedTests) > 0 {
+			replayingAfterRecordingResult, _ = vt.RunParallel(vcr.RunOptions{
+				Mode:     vcr.Replaying,
+				Version:  provider.Alpha,
+				TestDirs: testDirs,
+				Tests:    recordingResult.PassedTests,
+			})
+			if err := vt.UploadLogs(vcr.UploadLogsOptions{
+				Head:           head,
+				Parallel:       true,
+				AfterRecording: true,
+				Mode:           vcr.Recording,
+				Version:        provider.Alpha,
+			}); err != nil {
+				return fmt.Errorf("error uploading recording logs: %w", err)
+			}
+		}
+		hasTerminatedTests := (len(recordingResult.PassedTests) + len(recordingResult.FailedTests)) < len(replayingResult.FailedTests)
+		allRecordingPassed := len(recordingResult.FailedTests) == 0 && !hasTerminatedTests && recordingErr == nil
+		recordReplayData := recordReplay{
+			RecordingResult:               recordingResult,
+			ReplayingAfterRecordingResult: replayingAfterRecordingResult,
+			RecordingErr:                  recordingErr,
+			HasTerminatedTests:            hasTerminatedTests,
+			AllRecordingPassed:            allRecordingPassed,
+		}
+		recordReplayComment, err := formatRecordReplay(recordReplayData)
+		if err != nil {
+			return fmt.Errorf("error formatting record replay comment: %w", err)
+		}
+		if err := postGerritComment(kokoroArtifactsDir, modifiedFilePath, recordReplayComment, rnr); err != nil {
+			return fmt.Errorf("error posting comment: %w", err)
+		}
+	} else { //  len(replayingResult.FailedTests) == 0
+		withoutReplayFailedTestsData := withoutReplayFailedTests{
+			ReplayingErr: replayingErr,
+		}
+		withoutReplayFailedTestsComment, err := formatWithoutReplayFailedTests(withoutReplayFailedTestsData)
+		if err != nil {
+			return fmt.Errorf("error formatting action taken comment: %w", err)
+		}
+		comment := strings.Join([]string{testsAnalyticsComment, withoutReplayFailedTestsComment}, "\n")
+		if err := postGerritComment(kokoroArtifactsDir, modifiedFilePath, comment, rnr); err != nil {
+			return fmt.Errorf("error posting comment: %w", err)
+		}
+	}
+	return nil
+}
+
+var allowedAlphaServices = map[string]struct{}{
+	"accesscontextmanager":      {},
+	"cloudbuild":                {},
+	"compute":                   {},
+	"dataprocgdc":               {},
+	"healthcare":                {},
+	"looker":                    {},
+	"osconfig":                  {},
+	"saasmanagement":            {},
+	"stackdriver":               {},
+	"tpuv2":                     {},
+	"workstations":              {},
+	"bigquery":                  {},
+	"cloudbuildv2":              {},
+	"contactcenteraiplatform":   {},
+	"gkeonprem":                 {},
+	"kms":                       {},
+	"netapp":                    {},
+	"remotebuildexecutionadmin": {},
+	"spanner":                   {},
+	"storageinsights":           {},
+	"vmwareengine":              {},
+}
+
+func handleEAPVCRPanics(head, kokoroArtifactsDir, modifiedFilePath string, result vcr.Result, mode vcr.Mode, rnr ExecRunner) (bool, error) {
+	if len(result.Panics) > 0 {
+		comment := fmt.Sprintf(`The provider crashed while running the VCR tests in %s mode.
+Please fix it to complete your CL
+View the [build log](https://storage.cloud.google.com/ci-vcr-logs/alpha/refs/heads/%s/build-log/%s_test.log)`, mode.Upper(), head, mode.Lower())
+		if err := postGerritComment(kokoroArtifactsDir, modifiedFilePath, comment, rnr); err != nil {
+			return true, fmt.Errorf("error posting comment: %v", err)
+		}
+		return true, nil
+	}
+	return false, nil
+}
+
+func postGerritComment(kokoroArtifactsDir, modifiedFilePath, comment string, rnr ExecRunner) error {
+	return rnr.AppendFile(filepath.Join(kokoroArtifactsDir, "gerrit_comments.json"), fmt.Sprintf("\n{path: \"%s\", message: \"%s\"}", modifiedFilePath, comment))
+}
+
+func init() {
+	rootCmd.AddCommand(testEAPVCRCmd)
+}

.ci/magician/cmd/test_terraform_vcr.go

@@ -173,7 +173,7 @@ func execTestTerraformVCR(prNumber, mmCommitSha, buildID, projectID, buildStep,
 		return fmt.Errorf("error changing to tpgbRepo dir: %w", err)
 	}
 
-	services, runFullVCR := modifiedPackages(tpgbRepo.ChangedFiles)
+	services, runFullVCR := modifiedPackages(tpgbRepo.ChangedFiles, provider.Beta)
 	if len(services) == 0 && !runFullVCR {
 		fmt.Println("Skipping tests: No go files or test fixtures changed")
 		return nil
@@ -189,7 +189,7 @@ func execTestTerraformVCR(prNumber, mmCommitSha, buildID, projectID, buildStep,
 		return fmt.Errorf("error posting pending status: %w", err)
 	}
 
-	replayingResult, testDirs, replayingErr := runReplaying(runFullVCR, services, vt)
+	replayingResult, testDirs, replayingErr := runReplaying(runFullVCR, provider.Beta, services, vt)
 	testState := "success"
 	if replayingErr != nil {
 		testState = "failure"
@@ -393,7 +393,7 @@ func notRunTests(gaDiff, betaDiff string, result vcr.Result) ([]string, []string
 	return notRunBeta, notRunGa
 }
 
-func modifiedPackages(changedFiles []string) (map[string]struct{}, bool) {
+func modifiedPackages(changedFiles []string, version provider.Version) (map[string]struct{}, bool) {
 	var goFiles []string
 	for _, line := range changedFiles {
 		if strings.HasSuffix(line, ".go") || strings.Contains(line, "test-fixtures") || strings.HasSuffix(line, "go.mod") || strings.HasSuffix(line, "go.sum") {
@@ -403,10 +403,10 @@ func modifiedPackages(changedFiles []string) (map[string]struct{}, bool) {
 	services := make(map[string]struct{})
 	runFullVCR := false
 	for _, file := range goFiles {
-		if strings.HasPrefix(file, "google-beta/services/") {
+		if strings.HasPrefix(file, version.ProviderName()+"/services/") {
 			fileParts := strings.Split(file, "/")
 			services[fileParts[2]] = struct{}{}
-		} else if file == "google-beta/provider/provider_mmv1_resources.go" || file == "google-beta/provider/provider_dcl_resources.go" {
+		} else if file == version.ProviderName()+"/provider/provider_mmv1_resources.go" || file == version.ProviderName()+"/provider/provider_dcl_resources.go" {
 			fmt.Println("ignore changes in ", file)
 		} else {
 			fmt.Println("run full tests ", file)
@@ -417,7 +417,7 @@ func modifiedPackages(changedFiles []string) (map[string]struct{}, bool) {
 	return services, runFullVCR
 }
 
-func runReplaying(runFullVCR bool, services map[string]struct{}, vt *vcr.Tester) (vcr.Result, []string, error) {
+func runReplaying(runFullVCR bool, version provider.Version, services map[string]struct{}, vt *vcr.Tester) (vcr.Result, []string, error) {
 	result := vcr.Result{}
 	var testDirs []string
 	var replayingErr error
@@ -430,7 +430,7 @@ func runReplaying(runFullVCR bool, services map[string]struct{}, vt *vcr.Tester)
 	} else if len(services) > 0 {
 		fmt.Printf("runReplaying: %d specific services: %v\n", len(services), services)
 		for service := range services {
-			servicePath := "./" + filepath.Join("google-beta", "services", service)
+			servicePath := "./" + filepath.Join(version.ProviderName(), "services", service)
 			testDirs = append(testDirs, servicePath)
 			fmt.Println("run VCR tests in ", service)
 			serviceResult, serviceReplayingErr := vt.Run(vcr.RunOptions{

.ci/magician/cmd/test_terraform_vcr_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/assert"
 
+	"magician/provider"
 	"magician/vcr"
 )
 
@@ -64,7 +65,7 @@ func TestModifiedPackagesFromDiffs(t *testing.T) {
 			all:      false,
 		},
 	} {
-		if packages, all := modifiedPackages(tc.diffs); !reflect.DeepEqual(packages, tc.packages) {
+		if packages, all := modifiedPackages(tc.diffs, provider.Beta); !reflect.DeepEqual(packages, tc.packages) {
 			t.Errorf("Unexpected packages found for test %s: %v, expected %v", tc.name, packages, tc.packages)
 		} else if all != tc.all {
 			t.Errorf("Unexpected value for all packages for test %s: %v, expected %v", tc.name, all, tc.all)

.ci/magician/provider/version.go

@@ -23,6 +23,18 @@ func (v Version) String() string {
 	return "unknown"
 }
 
+func (v Version) ProviderName() string {
+	switch v {
+	case GA:
+		return "google"
+	case Beta:
+		return "google-beta"
+	case Alpha:
+		return "google-private"
+	}
+	return "unknown"
+}
+
 func (v Version) BucketPath() string {
 	if v == GA {
 		return ""