diff --git a/.ci/magician/cmd/test_eap_vcr.go b/.ci/magician/cmd/test_eap_vcr.go new file mode 100644 index 000000000000..30f167352917 --- /dev/null +++ b/.ci/magician/cmd/test_eap_vcr.go @@ -0,0 +1,245 @@ +package cmd + +import ( + _ "embed" + "fmt" + "magician/exec" + "magician/provider" + "magician/vcr" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/spf13/cobra" +) + +var tevEnvironmentVariables = [...]string{ + "GENPATH", + "GOCACHE", + "GOPATH", + "GOOGLE_REGION", + "GOOGLE_ZONE", + "ORG_ID", + "GOOGLE_PROJECT", + "GOOGLE_BILLING_ACCOUNT", + "GOOGLE_ORG", + "GOOGLE_ORG_DOMAIN", + "GOOGLE_PROJECT_NUMBER", + "GOOGLE_USE_DEFAULT_CREDENTIALS", + "GOOGLE_IMPERSONATE_SERVICE_ACCOUNT", + "KOKORO_ARTIFACTS_DIR", + "HOME", + "MODIFIED_FILE_PATH", + "PATH", + "USER", +} + +var testEAPVCRCmd = &cobra.Command{ + Use: "test-eap-vcr", + Short: "Run vcr tests for affected packages in EAP", + Long: `This command runs on new change lists to replay VCR cassettes and re-record failing cassettes. + + The following environment variables are required: +` + listTEVEnvironmentVariables(), + RunE: func(cmd *cobra.Command, args []string) error { + env := make(map[string]string, len(tevEnvironmentVariables)) + for _, ev := range tevEnvironmentVariables { + val, ok := os.LookupEnv(ev) + if !ok { + return fmt.Errorf("did not provide %s environment variable", ev) + } + env[ev] = val + } + rnr, err := exec.NewRunner() + if err != nil { + return err + } + vt, err := vcr.NewTester(env, "ci-vcr-cassettes", "ci-vcr-logs", rnr) + if err != nil { + return err + } + return execTestEAPVCR(args[0], env["GEN_PATH"], env["KOKORO_ARTIFACTS_DIR"], env["MODIFIED_FILE_PATH"], rnr, vt) + }, +} + +func listTEVEnvironmentVariables() string { + var result string + for i, ev := range tevEnvironmentVariables { + result += fmt.Sprintf("\t%2d. %s\n", i+1, ev) + } + return result +} + +func execTestEAPVCR(changeNumber, genPath, kokoroArtifactsDir, modifiedFilePath string, rnr ExecRunner, vt *vcr.Tester) error { + vt.SetRepoPath(provider.Alpha, genPath) + if err := rnr.PushDir(genPath); err != nil { + return fmt.Errorf("error changing to gen path: %w", err) + } + + changedFiles, err := rnr.Run("git", []string{"diff", "--name-only"}, nil) + if err != nil { + return fmt.Errorf("error diffing gen path: %w", err) + } + + services, runFullVCR := modifiedPackages(strings.Split(changedFiles, "\n"), provider.Alpha) + if len(services) == 0 && !runFullVCR { + fmt.Println("Skipping tests: No go files or test fixtures changed") + return nil + } + fmt.Println("Running tests: Go files or test fixtures changed") + + head := "auto-cl-" + changeNumber + if err := vt.FetchCassettes(provider.Alpha, "main", head); err != nil { + return fmt.Errorf("error fetching cassettes: %w", err) + } + replayingResult, testDirs, replayingErr := runReplaying(runFullVCR, provider.Alpha, services, vt) + if err := vt.UploadLogs(vcr.UploadLogsOptions{ + Head: head, + Mode: vcr.Replaying, + Version: provider.Alpha, + }); err != nil { + return fmt.Errorf("error uploading replaying logs: %w", err) + } + + if hasPanics, err := handleEAPVCRPanics(head, kokoroArtifactsDir, modifiedFilePath, replayingResult, vcr.Replaying, rnr); err != nil { + return fmt.Errorf("error handling panics: %w", err) + } else if hasPanics { + return nil + } + + var servicesArr []string + for s := range services { + if _, ok := allowedAlphaServices[s]; ok { + servicesArr = append(servicesArr, s) + } + } + analyticsData := analytics{ + ReplayingResult: replayingResult, + RunFullVCR: runFullVCR, + AffectedServices: sort.StringSlice(servicesArr), + } + testsAnalyticsComment, err := formatTestsAnalytics(analyticsData) + if err != nil { + return fmt.Errorf("error formatting test_analytics comment: %w", err) + } + if len(replayingResult.FailedTests) > 0 { + withReplayFailedTestsData := withReplayFailedTests{ + ReplayingResult: replayingResult, + } + + withReplayFailedTestsComment, err := formatWithReplayFailedTests(withReplayFailedTestsData) + if err != nil { + return fmt.Errorf("error formatting action taken comment: %w", err) + } + comment := strings.Join([]string{testsAnalyticsComment, withReplayFailedTestsComment}, "\n") + if err := postGerritComment(kokoroArtifactsDir, modifiedFilePath, comment, rnr); err != nil { + return fmt.Errorf("error posting comment: %w", err) + } + + recordingResult, recordingErr := vt.RunParallel(vcr.RunOptions{ + Mode: vcr.Recording, + Version: provider.Beta, + TestDirs: testDirs, + Tests: replayingResult.FailedTests, + }) + + if hasPanics, err := handleEAPVCRPanics(head, kokoroArtifactsDir, modifiedFilePath, recordingResult, vcr.Recording, rnr); err != nil { + return fmt.Errorf("error handling panics: %w", err) + } else if hasPanics { + return nil + } + replayingAfterRecordingResult := vcr.Result{} + if len(recordingResult.PassedTests) > 0 { + replayingAfterRecordingResult, _ = vt.RunParallel(vcr.RunOptions{ + Mode: vcr.Replaying, + Version: provider.Alpha, + TestDirs: testDirs, + Tests: recordingResult.PassedTests, + }) + if err := vt.UploadLogs(vcr.UploadLogsOptions{ + Head: head, + Parallel: true, + AfterRecording: true, + Mode: vcr.Recording, + Version: provider.Alpha, + }); err != nil { + return fmt.Errorf("error uploading recording logs: %w", err) + } + } + hasTerminatedTests := (len(recordingResult.PassedTests) + len(recordingResult.FailedTests)) < len(replayingResult.FailedTests) + allRecordingPassed := len(recordingResult.FailedTests) == 0 && !hasTerminatedTests && recordingErr == nil + recordReplayData := recordReplay{ + RecordingResult: recordingResult, + ReplayingAfterRecordingResult: replayingAfterRecordingResult, + RecordingErr: recordingErr, + HasTerminatedTests: hasTerminatedTests, + AllRecordingPassed: allRecordingPassed, + } + recordReplayComment, err := formatRecordReplay(recordReplayData) + if err != nil { + return fmt.Errorf("error formatting record replay comment: %w", err) + } + if err := postGerritComment(kokoroArtifactsDir, modifiedFilePath, recordReplayComment, rnr); err != nil { + return fmt.Errorf("error posting comment: %w", err) + } + } else { // len(replayingResult.FailedTests) == 0 + withoutReplayFailedTestsData := withoutReplayFailedTests{ + ReplayingErr: replayingErr, + } + withoutReplayFailedTestsComment, err := formatWithoutReplayFailedTests(withoutReplayFailedTestsData) + if err != nil { + return fmt.Errorf("error formatting action taken comment: %w", err) + } + comment := strings.Join([]string{testsAnalyticsComment, withoutReplayFailedTestsComment}, "\n") + if err := postGerritComment(kokoroArtifactsDir, modifiedFilePath, comment, rnr); err != nil { + return fmt.Errorf("error posting comment: %w", err) + } + } + return nil +} + +var allowedAlphaServices = map[string]struct{}{ + "accesscontextmanager": {}, + "cloudbuild": {}, + "compute": {}, + "dataprocgdc": {}, + "healthcare": {}, + "looker": {}, + "osconfig": {}, + "saasmanagement": {}, + "stackdriver": {}, + "tpuv2": {}, + "workstations": {}, + "bigquery": {}, + "cloudbuildv2": {}, + "contactcenteraiplatform": {}, + "gkeonprem": {}, + "kms": {}, + "netapp": {}, + "remotebuildexecutionadmin": {}, + "spanner": {}, + "storageinsights": {}, + "vmwareengine": {}, +} + +func handleEAPVCRPanics(head, kokoroArtifactsDir, modifiedFilePath string, result vcr.Result, mode vcr.Mode, rnr ExecRunner) (bool, error) { + if len(result.Panics) > 0 { + comment := fmt.Sprintf(`The provider crashed while running the VCR tests in %s mode. +Please fix it to complete your CL +View the [build log](https://storage.cloud.google.com/ci-vcr-logs/alpha/refs/heads/%s/build-log/%s_test.log)`, mode.Upper(), head, mode.Lower()) + if err := postGerritComment(kokoroArtifactsDir, modifiedFilePath, comment, rnr); err != nil { + return true, fmt.Errorf("error posting comment: %v", err) + } + return true, nil + } + return false, nil +} + +func postGerritComment(kokoroArtifactsDir, modifiedFilePath, comment string, rnr ExecRunner) error { + return rnr.AppendFile(filepath.Join(kokoroArtifactsDir, "gerrit_comments.json"), fmt.Sprintf("\n{path: \"%s\", message: \"%s\"}", modifiedFilePath, comment)) +} + +func init() { + rootCmd.AddCommand(testEAPVCRCmd) +} diff --git a/.ci/magician/cmd/test_terraform_vcr.go b/.ci/magician/cmd/test_terraform_vcr.go index 519d1ef1a39e..f6e2ecb5e391 100644 --- a/.ci/magician/cmd/test_terraform_vcr.go +++ b/.ci/magician/cmd/test_terraform_vcr.go @@ -173,7 +173,7 @@ func execTestTerraformVCR(prNumber, mmCommitSha, buildID, projectID, buildStep, return fmt.Errorf("error changing to tpgbRepo dir: %w", err) } - services, runFullVCR := modifiedPackages(tpgbRepo.ChangedFiles) + services, runFullVCR := modifiedPackages(tpgbRepo.ChangedFiles, provider.Beta) if len(services) == 0 && !runFullVCR { fmt.Println("Skipping tests: No go files or test fixtures changed") return nil @@ -189,7 +189,7 @@ func execTestTerraformVCR(prNumber, mmCommitSha, buildID, projectID, buildStep, return fmt.Errorf("error posting pending status: %w", err) } - replayingResult, testDirs, replayingErr := runReplaying(runFullVCR, services, vt) + replayingResult, testDirs, replayingErr := runReplaying(runFullVCR, provider.Beta, services, vt) testState := "success" if replayingErr != nil { testState = "failure" @@ -393,7 +393,7 @@ func notRunTests(gaDiff, betaDiff string, result vcr.Result) ([]string, []string return notRunBeta, notRunGa } -func modifiedPackages(changedFiles []string) (map[string]struct{}, bool) { +func modifiedPackages(changedFiles []string, version provider.Version) (map[string]struct{}, bool) { var goFiles []string for _, line := range changedFiles { if strings.HasSuffix(line, ".go") || strings.Contains(line, "test-fixtures") || strings.HasSuffix(line, "go.mod") || strings.HasSuffix(line, "go.sum") { @@ -403,10 +403,10 @@ func modifiedPackages(changedFiles []string) (map[string]struct{}, bool) { services := make(map[string]struct{}) runFullVCR := false for _, file := range goFiles { - if strings.HasPrefix(file, "google-beta/services/") { + if strings.HasPrefix(file, version.ProviderName()+"/services/") { fileParts := strings.Split(file, "/") services[fileParts[2]] = struct{}{} - } else if file == "google-beta/provider/provider_mmv1_resources.go" || file == "google-beta/provider/provider_dcl_resources.go" { + } else if file == version.ProviderName()+"/provider/provider_mmv1_resources.go" || file == version.ProviderName()+"/provider/provider_dcl_resources.go" { fmt.Println("ignore changes in ", file) } else { fmt.Println("run full tests ", file) @@ -417,7 +417,7 @@ func modifiedPackages(changedFiles []string) (map[string]struct{}, bool) { return services, runFullVCR } -func runReplaying(runFullVCR bool, services map[string]struct{}, vt *vcr.Tester) (vcr.Result, []string, error) { +func runReplaying(runFullVCR bool, version provider.Version, services map[string]struct{}, vt *vcr.Tester) (vcr.Result, []string, error) { result := vcr.Result{} var testDirs []string var replayingErr error @@ -430,7 +430,7 @@ func runReplaying(runFullVCR bool, services map[string]struct{}, vt *vcr.Tester) } else if len(services) > 0 { fmt.Printf("runReplaying: %d specific services: %v\n", len(services), services) for service := range services { - servicePath := "./" + filepath.Join("google-beta", "services", service) + servicePath := "./" + filepath.Join(version.ProviderName(), "services", service) testDirs = append(testDirs, servicePath) fmt.Println("run VCR tests in ", service) serviceResult, serviceReplayingErr := vt.Run(vcr.RunOptions{ diff --git a/.ci/magician/cmd/test_terraform_vcr_test.go b/.ci/magician/cmd/test_terraform_vcr_test.go index 8c7c931c3333..8650643d482e 100644 --- a/.ci/magician/cmd/test_terraform_vcr_test.go +++ b/.ci/magician/cmd/test_terraform_vcr_test.go @@ -9,6 +9,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" + "magician/provider" "magician/vcr" ) @@ -64,7 +65,7 @@ func TestModifiedPackagesFromDiffs(t *testing.T) { all: false, }, } { - if packages, all := modifiedPackages(tc.diffs); !reflect.DeepEqual(packages, tc.packages) { + if packages, all := modifiedPackages(tc.diffs, provider.Beta); !reflect.DeepEqual(packages, tc.packages) { t.Errorf("Unexpected packages found for test %s: %v, expected %v", tc.name, packages, tc.packages) } else if all != tc.all { t.Errorf("Unexpected value for all packages for test %s: %v, expected %v", tc.name, all, tc.all) diff --git a/.ci/magician/provider/version.go b/.ci/magician/provider/version.go index 6372ff7fd297..224c0aeddd03 100644 --- a/.ci/magician/provider/version.go +++ b/.ci/magician/provider/version.go @@ -23,6 +23,18 @@ func (v Version) String() string { return "unknown" } +func (v Version) ProviderName() string { + switch v { + case GA: + return "google" + case Beta: + return "google-beta" + case Alpha: + return "google-private" + } + return "unknown" +} + func (v Version) BucketPath() string { if v == GA { return ""