Skip to content
This repository has been archived by the owner on Oct 9, 2023. It is now read-only.

Refactored type casting into flyteidl #548

Draft
wants to merge 3 commits into
base: master
Choose a base branch
from
Draft

Refactored type casting into flyteidl #548

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
aa3a5e3
refactored literal type casting into flyteidl
hamersaw Mar 31, 2023
2f5a30a
fixed unit tests
hamersaw Mar 31, 2023
0517c25
bumped flyteidl
hamersaw Mar 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ require (
github.com/spf13/cobra v1.4.0
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.7.2
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
golang.org/x/time v0.0.0-20220210224613-90d013bbcef8
google.golang.org/grpc v1.46.0
Expand Down Expand Up @@ -122,7 +121,8 @@ require (
github.com/stretchr/objx v0.3.0 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
go.opencensus.io v0.23.0 // indirect
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
golang.org/x/crypto v0.1.0 // indirect
golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
golang.org/x/net v0.7.0 // indirect
golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
golang.org/x/sys v0.5.0 // indirect
Expand All @@ -147,3 +147,5 @@ require (
)

replace github.com/aws/amazon-sagemaker-operator-for-k8s => github.com/aws/amazon-sagemaker-operator-for-k8s v1.0.1-0.20210303003444-0fb33b1fd49d

replace github.com/flyteorg/flyteidl => github.com/flyteorg/flyteidl v1.3.16-0.20230331180644-4649cf9c7cc7
16 changes: 8 additions & 8 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,8 +260,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/flyteorg/flyteidl v1.3.14 h1:o5M0g/r6pXTPu5PEurbYxbQmuOu3hqqsaI2M6uvK0N8=
github.com/flyteorg/flyteidl v1.3.14/go.mod h1:Pkt2skI1LiHs/2ZoekBnyPhuGOFMiuul6HHcKGZBsbM=
github.com/flyteorg/flyteidl v1.3.16-0.20230331180644-4649cf9c7cc7 h1:rzSpT+JTgpbc49Le3caq8Tt9sWWxdFjxrCTgr7tUVzU=
github.com/flyteorg/flyteidl v1.3.16-0.20230331180644-4649cf9c7cc7/go.mod h1:GdhmUeGpSBVf98mndMLgBsEM8X/emAaVY/LtyIMbMBo=
github.com/flyteorg/flyteplugins v1.0.44 h1:uKizng+i0vfXslyPBlrsfecInhvy71fTB4kRg7eiifE=
github.com/flyteorg/flyteplugins v1.0.44/go.mod h1:ztsonku5fKwyxcIg1k69PTiBVjRI6d3nK5DnC+iwx08=
github.com/flyteorg/flytestdlib v1.0.15 h1:kv9jDQmytbE84caY+pkZN8trJU2ouSAmESzpTEhfTt0=
Expand Down Expand Up @@ -846,8 +846,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM=
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU=
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
Expand All @@ -858,8 +858,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA=
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA=
golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug=
golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
Expand All @@ -886,7 +886,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I=
golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
Expand Down Expand Up @@ -1163,7 +1163,7 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down
6 changes: 3 additions & 3 deletions pkg/compiler/transformers/k8s/inputs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
package k8s

import (
"github.com/flyteorg/flyteidl/clients/go/coreutils"
"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core"
"github.com/flyteorg/flytepropeller/pkg/compiler/common"
"github.com/flyteorg/flytepropeller/pkg/compiler/errors"
"github.com/flyteorg/flytepropeller/pkg/compiler/validators"
"k8s.io/apimachinery/pkg/util/sets"
)

Expand Down Expand Up @@ -34,8 +34,8 @@ func validateInputs(nodeID common.NodeID, iface *core.TypedInterface, inputs cor
continue
}

inputType := validators.LiteralTypeForLiteral(inputVal)
if !validators.AreTypesCastable(inputType, v.Type) {
inputType := coreutils.LiteralTypeForLiteral(inputVal)
if !coreutils.AreTypesCastable(inputType, v.Type) {
errs.Collect(errors.NewMismatchingTypesErr(nodeID, inputVar, v.Type.String(), inputType.String()))
continue
}
Expand Down
7 changes: 4 additions & 3 deletions pkg/compiler/validators/bindings.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (

"github.com/flyteorg/flytepropeller/pkg/compiler/typing"

"github.com/flyteorg/flyteidl/clients/go/coreutils"
flyte "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core"
c "github.com/flyteorg/flytepropeller/pkg/compiler/common"
"github.com/flyteorg/flytepropeller/pkg/compiler/errors"
Expand Down Expand Up @@ -125,7 +126,7 @@ func validateBinding(w c.WorkflowBuilder, nodeID c.NodeID, nodeParam string, bin
}
}

if !validateParamTypes || AreTypesCastable(sourceType, expectedType) {
if !validateParamTypes || coreutils.AreTypesCastable(sourceType, expectedType) {
val.Promise.NodeId = upNode.GetId()
return param.GetType(), []c.NodeID{val.Promise.NodeId}, true
}
Expand All @@ -141,10 +142,10 @@ func validateBinding(w c.WorkflowBuilder, nodeID c.NodeID, nodeParam string, bin
return nil, nil, !errs.HasErrors()
}

literalType := literalTypeForScalar(val.Scalar)
literalType := coreutils.LiteralTypeForScalar(val.Scalar)
if literalType == nil {
errs.Collect(errors.NewUnrecognizedValueErr(nodeID, reflect.TypeOf(val.Scalar.GetValue()).String()))
} else if validateParamTypes && !AreTypesCastable(literalType, expectedType) {
} else if validateParamTypes && !coreutils.AreTypesCastable(literalType, expectedType) {
errs.Collect(errors.NewMismatchingTypesErr(nodeID, nodeParam, literalType.String(), expectedType.String()))
}

Expand Down
20 changes: 10 additions & 10 deletions pkg/compiler/validators/bindings_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ func TestValidateBindings(t *testing.T) {
vars := &core.VariableMap{
Variables: map[string]*core.Variable{
"x": {
Type: LiteralTypeForLiteral(coreutils.MustMakeLiteral(5)),
Type: coreutils.LiteralTypeForLiteral(coreutils.MustMakeLiteral(5)),
},
},
}
Expand Down Expand Up @@ -132,7 +132,7 @@ func TestValidateBindings(t *testing.T) {
vars := &core.VariableMap{
Variables: map[string]*core.Variable{
"x": {
Type: LiteralTypeForLiteral(coreutils.MustMakeLiteral([]interface{}{5})),
Type: coreutils.LiteralTypeForLiteral(coreutils.MustMakeLiteral([]interface{}{5})),
},
},
}
Expand Down Expand Up @@ -227,7 +227,7 @@ func TestValidateBindings(t *testing.T) {
vars := &core.VariableMap{
Variables: map[string]*core.Variable{
"x": {
Type: LiteralTypeForLiteral(coreutils.MustMakeLiteral(
Type: coreutils.LiteralTypeForLiteral(coreutils.MustMakeLiteral(
map[string]interface{}{
"xy": 5,
})),
Expand Down Expand Up @@ -265,7 +265,7 @@ func TestValidateBindings(t *testing.T) {
Outputs: &core.VariableMap{
Variables: map[string]*core.Variable{
"n2_out": {
Type: LiteralTypeForLiteral(coreutils.MustMakeLiteral(2)),
Type: coreutils.LiteralTypeForLiteral(coreutils.MustMakeLiteral(2)),
},
},
},
Expand All @@ -292,7 +292,7 @@ func TestValidateBindings(t *testing.T) {
vars := &core.VariableMap{
Variables: map[string]*core.Variable{
"x": {
Type: LiteralTypeForLiteral(coreutils.MustMakeLiteral(5)),
Type: coreutils.LiteralTypeForLiteral(coreutils.MustMakeLiteral(5)),
},
},
}
Expand Down Expand Up @@ -327,7 +327,7 @@ func TestValidateBindings(t *testing.T) {
Outputs: &core.VariableMap{
Variables: map[string]*core.Variable{
"n2_out": {
Type: LiteralTypeForLiteral(coreutils.MustMakeLiteral(2)),
Type: coreutils.LiteralTypeForLiteral(coreutils.MustMakeLiteral(2)),
},
},
},
Expand All @@ -349,7 +349,7 @@ func TestValidateBindings(t *testing.T) {
vars := &core.VariableMap{
Variables: map[string]*core.Variable{
"x": {
Type: LiteralTypeForLiteral(coreutils.MustMakeLiteral(5)),
Type: coreutils.LiteralTypeForLiteral(coreutils.MustMakeLiteral(5)),
},
},
}
Expand Down Expand Up @@ -906,7 +906,7 @@ func TestValidateBindings(t *testing.T) {
Outputs: &core.VariableMap{
Variables: map[string]*core.Variable{
"n2_out": {
Type: LiteralTypeForLiteral(coreutils.MustMakeLiteral(2)),
Type: coreutils.LiteralTypeForLiteral(coreutils.MustMakeLiteral(2)),
},
},
},
Expand Down Expand Up @@ -987,7 +987,7 @@ func TestValidateBindings(t *testing.T) {
Outputs: &core.VariableMap{
Variables: map[string]*core.Variable{
"n2_out": {
Type: LiteralTypeForLiteral(coreutils.MustMakeLiteral(2)),
Type: coreutils.LiteralTypeForLiteral(coreutils.MustMakeLiteral(2)),
},
},
},
Expand Down Expand Up @@ -1072,7 +1072,7 @@ func TestValidateBindings(t *testing.T) {
Outputs: &core.VariableMap{
Variables: map[string]*core.Variable{
"n2_out": {
Type: LiteralTypeForLiteral(&core.Literal{
Type: coreutils.LiteralTypeForLiteral(&core.Literal{
Value: &core.Literal_Scalar{
Scalar: &core.Scalar{
Value: &core.Scalar_Union{
Expand Down
3 changes: 2 additions & 1 deletion pkg/compiler/validators/condition.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package validators
import (
"fmt"

"github.com/flyteorg/flyteidl/clients/go/coreutils"
flyte "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core"
c "github.com/flyteorg/flytepropeller/pkg/compiler/common"
"github.com/flyteorg/flytepropeller/pkg/compiler/errors"
Expand All @@ -14,7 +15,7 @@ func validateOperand(node c.NodeBuilder, paramName string, operand *flyte.Operan
errs.Collect(errors.NewValueRequiredErr(node.GetId(), paramName))
} else if operand.GetPrimitive() != nil {
// no validation
literalType = literalTypeForPrimitive(operand.GetPrimitive())
literalType = coreutils.LiteralTypeForPrimitive(operand.GetPrimitive())
} else if len(operand.GetVar()) > 0 {
if node.GetInterface() != nil {
if param, paramOk := validateInputVar(node, operand.GetVar(), requireParamType, errs.NewScope()); paramOk {
Expand Down
Loading