openapi_generate initial product parsing

mmv1/api/product.go

@@ -34,12 +34,12 @@ type Product struct {
 
 	// original value of :name before the provider override happens
 	// same as :name if not overridden in provider
-	ApiName string `yaml:"api_name"`
+	ApiName string `yaml:"api_name,omitempty"`
 
 	// Display Name: The full name of the GCP product; eg "Cloud Bigtable"
-	DisplayName string `yaml:"display_name"`
+	DisplayName string `yaml:"display_name,omitempty"`
 
-	Objects []*Resource
+	Objects []*Resource `yaml:"objects,omitempty"`
 
 	// The list of permission scopes available for the service
 	// For example: `https://www.googleapis.com/auth/compute`
@@ -50,19 +50,19 @@ type Product struct {
 
 	// The base URL for the service API endpoint
 	// For example: `https://www.googleapis.com/compute/v1/`
-	BaseUrl string `yaml:"base_url"`
+	BaseUrl string `yaml:"base_url,omitempty"`
 
 	// A function reference designed for the rare case where you
 	// need to use retries in operation calls. Used for the service api
 	// as it enables itself (self referential) and can result in occasional
 	// failures on operation_get. see github.com/hashicorp/terraform-provider-google/issues/9489
-	OperationRetry string `yaml:"operation_retry"`
+	OperationRetry string `yaml:"operation_retry,omitempty"`
 
-	Async *Async
+	Async *Async `yaml:"async,omitempty"`
 
-	LegacyName string `yaml:"legacy_name"`
+	LegacyName string `yaml:"legacy_name,omitempty"`
 
-	ClientName string `yaml:"client_name"`
+	ClientName string `yaml:"client_name,omitempty"`
 }
 
 func (p *Product) UnmarshalYAML(unmarshal func(any) error) error {

mmv1/api/product/version.go

@@ -26,7 +26,7 @@ var ORDER = []string{"ga", "beta", "alpha", "private"}
 // a superset of beta, and beta a superset of GA. Each version will have a
 // different version url.
 type Version struct {
-	CaiBaseUrl string `yaml:"cai_base_url"`
+	CaiBaseUrl string `yaml:"cai_base_url,omitempty"`
 	BaseUrl    string `yaml:"base_url"`
 	Name       string
 }

mmv1/go.mod

@@ -11,4 +11,15 @@ require github.com/golang/glog v1.2.0
 
 require github.com/otiai10/copy v1.9.0
 
-require golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
+require (
+	github.com/getkin/kin-openapi v0.127.0 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/invopop/yaml v0.3.1 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
+	github.com/perimeterx/marshmallow v1.1.5 // indirect
+	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

mmv1/go.sum

@@ -1,13 +1,29 @@
+github.com/getkin/kin-openapi v0.127.0 h1:Mghqi3Dhryf3F8vR370nN67pAERW+3a95vomb3MAREY=
+github.com/getkin/kin-openapi v0.127.0/go.mod h1:OZrfXzUfGrNbsKj+xmFBx6E5c6yH3At/tAKSc2UszXM=
+github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
+github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
+github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
+github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
 github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68=
 github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/invopop/yaml v0.3.1 h1:f0+ZpmhfBSS4MhG+4HYseMdJhoeeopbSKbq5Rpeelso=
+github.com/invopop/yaml v0.3.1/go.mod h1:PMOp3nn4/12yEZUFfmOuNHJsZToEEOwoWsT+D81KkeA=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
 github.com/otiai10/copy v1.9.0 h1:7KFNiCgZ91Ru4qW4CWPf/7jqtxLagGRmIxWldPP9VY4=
 github.com/otiai10/copy v1.9.0/go.mod h1:hsfX19wcn0UWIHUQ3/4fHuehhk2UyArQ9dVFAn3FczI=
 github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
 github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
 github.com/otiai10/mint v1.4.0/go.mod h1:gifjb2MYOoULtKLqUAEILUG/9KONW6f7YsJ6vQLTlFI=
+github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s=
+github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw=
 golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ=
 golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
@@ -16,3 +32,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

mmv1/main.go

@@ -17,6 +17,7 @@ import (
 	"golang.org/x/exp/slices"
 
 	"github.com/GoogleCloudPlatform/magic-modules/mmv1/api"
+	"github.com/GoogleCloudPlatform/magic-modules/mmv1/openapi_generate"
 	"github.com/GoogleCloudPlatform/magic-modules/mmv1/provider"
 )
 
@@ -42,6 +43,8 @@ var doNotGenerateDocs = flag.Bool("no-docs", false, "do not generate docs")
 
 var forceProvider = flag.String("provider", "", "optional provider name. If specified, a non-default provider will be used.")
 
+var openapiGenerate = flag.Bool("openapi-generate", false, "Generate MMv1 YAML from openapi directory (Experimental)")
+
 // Example usage: --yaml
 var yamlMode = flag.Bool("yaml", false, "copy text over from ruby yaml to go yaml")
 
@@ -60,6 +63,12 @@ func main() {
 
 	flag.Parse()
 
+	if *openapiGenerate {
+		parser := openapi_generate.NewOpenapiParser("openapi_generate/openapi", "products")
+		parser.Run()
+		return
+	}
+
 	if *yamlMode || *yamlTempMode {
 		CopyAllDescriptions(*yamlTempMode)
 	}

mmv1/openapi_generate/parser.go

@@ -0,0 +1,190 @@
+// Copyright 2024 Google Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generator for a library converting terraform state to gcp objects.
+
+package openapi_generate
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"log"
+
+	"text/template"
+
+	"github.com/GoogleCloudPlatform/magic-modules/mmv1/api"
+	"github.com/GoogleCloudPlatform/magic-modules/mmv1/api/product"
+	"github.com/GoogleCloudPlatform/magic-modules/mmv1/google"
+	"github.com/getkin/kin-openapi/openapi3"
+	"github.com/golang/glog"
+	"gopkg.in/yaml.v2"
+)
+
+type Parser struct {
+	Folder string
+	Output string
+}
+
+func NewOpenapiParser(folder, output string) Parser {
+
+	wd, err := os.Getwd()
+	if err != nil {
+		log.Fatalf(err.Error())
+	}
+
+	parser := Parser{
+		Folder: path.Join(wd, folder),
+		Output: path.Join(wd, output),
+	}
+
+	return parser
+}
+
+func (parser Parser) Run() {
+
+	f, err := os.Open(parser.Folder)
+	if err != nil {
+		log.Fatalf(err.Error())
+		return
+	}
+	defer f.Close()
+	files, err := f.Readdirnames(0)
+	if err != nil {
+		log.Fatalf(err.Error())
+	}
+
+	// check if folder is empty
+	if len(files) == 0 {
+		log.Fatalf("No OpenAPI files found in %s", parser.Folder)
+	}
+
+	for _, file := range files {
+		parser.WriteYaml(path.Join(parser.Folder, file))
+	}
+}
+
+func (parser Parser) WriteYaml(filePath string) {
+	log.Printf("Reading from file path %s", filePath)
+
+	ctx := context.Background()
+	loader := &openapi3.Loader{Context: ctx, IsExternalRefsAllowed: true}
+	doc, _ := loader.LoadFromFile(filePath)
+	_ = doc.Validate(ctx)
+
+	resourcePaths := findResources(doc)
+	productPath := buildProduct(filePath, parser.Output, doc)
+
+	log.Printf("%+v", resourcePaths)
+	log.Printf("%+v", productPath)
+}
+
+func findResources(doc *openapi3.T) [][]string {
+	var resourcePaths [][]string
+
+	pathMap := doc.Paths.Map()
+	for key, pathValue := range pathMap {
+		if pathValue.Post == nil {
+			continue
+		}
+
+		// Not very clever way of identifying create resource methods
+		if strings.HasPrefix(pathValue.Post.OperationID, "Create") {
+			resourcePath := key
+			resourceName := strings.Replace(pathValue.Post.OperationID, "Create", "", 1)
+			resourcePaths = append(resourcePaths, []string{resourcePath, resourceName})
+		}
+	}
+
+	return resourcePaths
+}
+
+func buildProduct(filePath, output string, root *openapi3.T) string {
+
+	version := root.Info.Version
+	server := root.Servers[0].URL
+
+	productName := strings.Split(filepath.Base(filePath), "_")[0]
+	productPath := filepath.Join(output, productName)
+
+	if err := os.MkdirAll(productPath, os.ModePerm); err != nil {
+		log.Fatalf("error creating product output directory %v: %v", productPath, err)
+	}
+
+	apiProduct := &api.Product{}
+	apiVersion := &product.Version{}
+
+	apiVersion.BaseUrl = fmt.Sprintf("%s/%s/", server, version)
+	// TODO(slevenick) figure out how to tell the API version
+	apiVersion.Name = "ga"
+	apiProduct.Versions = []*product.Version{apiVersion}
+
+	// Standard titling is "Service Name API"
+	displayName := strings.Replace(root.Info.Title, " API", "", 1)
+	apiProduct.Name = strings.ReplaceAll(displayName, " ", "")
+	apiProduct.DisplayName = displayName
+
+	//Scopes should be added soon to OpenAPI, until then use global scope
+	apiProduct.Scopes = []string{"https://www.googleapis.com/auth/cloud-platform"}
+
+	// productOutPath := filepath.Join(output, fmt.Sprintf("/%s/product.yaml", productName))
+	templatePath := "openapi_generate/product_yaml.tmpl"
+
+	productOutPathTemplate := filepath.Join(output, fmt.Sprintf("/%s/product_template.yaml", productName))
+	WriteGoTemplate(templatePath, productOutPathTemplate, apiProduct)
+
+	productOutPathMarshal := filepath.Join(output, fmt.Sprintf("/%s/product_marshal.yaml", productName))
+
+	// Default yaml marshaller
+	bytes, err := yaml.Marshal(apiProduct)
+	if err != nil {
+		log.Fatalf("error marshalling yaml %v: %v", productOutPathMarshal, err)
+	}
+
+	err = os.WriteFile(productOutPathMarshal, bytes, 0644)
+	if err != nil {
+		log.Fatalf("error writing product to path %v: %v", productOutPathMarshal, err)
+	}
+
+	return productPath
+}
+
+func WriteGoTemplate(templatePath, filePath string, input any) {
+	contents := bytes.Buffer{}
+
+	templateFileName := filepath.Base(templatePath)
+	templates := []string{
+		templatePath,
+	}
+
+	tmpl, err := template.New(templateFileName).Funcs(google.TemplateFunctions).ParseFiles(templates...)
+	if err != nil {
+		glog.Exit(fmt.Sprintf("error parsing %s for filepath %s ", templatePath, filePath), err)
+	}
+	if err = tmpl.ExecuteTemplate(&contents, templateFileName, input); err != nil {
+		glog.Exit(fmt.Sprintf("error executing %s for filepath %s ", templatePath, filePath), err)
+	}
+
+	bytes := contents.Bytes()
+
+	err = os.WriteFile(filePath, bytes, 0644)
+	if err != nil {
+		log.Fatalf("error writing product to path %v: %v", filePath, err)
+	}
+
+}

mmv1/openapi_generate/product_yaml.tmpl

@@ -0,0 +1,25 @@
+# Copyright 2024 Google Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+name: '{{$.Name}}'
+display_name: '{{$.DisplayName}}'
+versions:
+{{- range $version := $.Versions }}
+  - name: '{{$version.Name}}'
+    base_url: {{$version.BaseUrl}}'
+{{- end }}
+scopes:
+{{- range $scope := $.Scopes }}
+  - '{{$scope}}'
+{{- end }}