From 44372d95ca5e01fc03ac60d4ac06c898387fba48 Mon Sep 17 00:00:00 2001 From: Keran Yang Date: Tue, 17 Sep 2024 18:06:23 -0400 Subject: [PATCH 01/13] fix: version compatibility check issue Signed-off-by: Keran Yang --- pkg/sdkclient/serverinfo/serverinfo_test.go | 131 +++++++++++++++----- pkg/sdkclient/serverinfo/types.go | 46 ++++++- 2 files changed, 145 insertions(+), 32 deletions(-) diff --git a/pkg/sdkclient/serverinfo/serverinfo_test.go b/pkg/sdkclient/serverinfo/serverinfo_test.go index e96919e243..de22f32be6 100644 --- a/pkg/sdkclient/serverinfo/serverinfo_test.go +++ b/pkg/sdkclient/serverinfo/serverinfo_test.go @@ -97,25 +97,64 @@ func Test_CheckNumaflowCompatibility(t *testing.T) { errMessage string }{ { - name: "Test with incompatible numaflow version", + name: "Test with incompatible numaflow version, min is a stable version 1.1.7", numaflowVersion: "v1.1.6", - minNumaflowVersion: "1.1.7", + minNumaflowVersion: "1.1.7-z", shouldErr: true, - errMessage: "numaflow version 1.1.6 must be upgraded to at least 1.1.7, in order to work with current SDK version", + errMessage: "numaflow version 1.1.6 must be upgraded to at least 1.1.7-z, in order to work with current SDK version", }, { - name: "Test with empty MinimumNumaflowVersion field", + name: "Test with compatible numaflow version - min is a stable version 1.1.6", numaflowVersion: "1.1.7", - minNumaflowVersion: "", + minNumaflowVersion: "1.1.6-z", + shouldErr: false, + }, + { + name: "Test with incompatible numaflow version - min is a stable version 1.1.7, numaflow version is a pre-release version", + numaflowVersion: "v1.1.7-rc1", + minNumaflowVersion: "1.1.7-z", shouldErr: true, - errMessage: "server info does not contain minimum numaflow version. Upgrade to newer SDK version", + errMessage: "numaflow version 1.1.7-rc1 must be upgraded to at least 1.1.7-z, in order to work with current SDK version", + }, + { + name: "Test with compatible numaflow version - min is a stable version 1.1.6, numaflow version is a pre-release version", + numaflowVersion: "1.1.7-rc1", + minNumaflowVersion: "1.1.6-z", + shouldErr: false, }, { - name: "Test with compatible numaflow version", + name: "Test with incompatible numaflow version, min is a rc version 1.1.7-rc1", + numaflowVersion: "v1.1.6", + minNumaflowVersion: "1.1.7-rc1", + shouldErr: true, + errMessage: "numaflow version 1.1.6 must be upgraded to at least 1.1.7-rc1, in order to work with current SDK version", + }, + { + name: "Test with compatible numaflow version - min is a rc version 1.1.6-rc1", numaflowVersion: "1.1.7", - minNumaflowVersion: "1.1.6", + minNumaflowVersion: "1.1.6-rc1", + shouldErr: false, + }, + { + name: "Test with incompatible numaflow version - min is a rc version 1.1.7-rc2, numaflow version is a pre-release version", + numaflowVersion: "v1.1.7-rc1", + minNumaflowVersion: "1.1.7-rc2", + shouldErr: true, + errMessage: "numaflow version 1.1.7-rc1 must be upgraded to at least 1.1.7-rc2, in order to work with current SDK version", + }, + { + name: "Test with compatible numaflow version - min is a rc version 1.1.6-rc2, numaflow version is a pre-release version", + numaflowVersion: "1.1.6-rc2", + minNumaflowVersion: "1.1.6-rc2", shouldErr: false, }, + { + name: "Test with empty MinimumNumaflowVersion field", + numaflowVersion: "1.1.7", + minNumaflowVersion: "", + shouldErr: true, + errMessage: "server info does not contain minimum numaflow version. Upgrade to newer SDK version", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -132,10 +171,10 @@ func Test_CheckNumaflowCompatibility(t *testing.T) { func Test_CheckSDKCompatibility(t *testing.T) { var testMinimumSupportedSDKVersions = sdkConstraints{ - Go: "0.6.0-0", - Python: "0.6.0a", - Java: "0.6.0-0", - Rust: "0.1.0", + Python: "0.6.0rc100", + Go: "0.6.0-z", + Java: "0.6.0-z", + Rust: "0.1.0-z", } tests := []struct { name string @@ -146,49 +185,85 @@ func Test_CheckSDKCompatibility(t *testing.T) { errMessage string }{ { - name: "Test with incompatible Python version", + name: "python pre-release version is lower than minimum supported version", sdkVersion: "v0.5.3a1", sdkLanguage: Python, minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, shouldErr: true, - errMessage: "SDK version 0.5.3a1 must be upgraded to at least 0.6.0a, in order to work with current numaflow version", + errMessage: "SDK version 0.5.3a1 must be upgraded to at least 0.6.0rc100, in order to work with current numaflow version", }, { - name: "Test with compatible Python version", - sdkVersion: "v0.6.0a2", + name: "python pre-release version is compatible with minimum supported version", + sdkVersion: "v0.6.3a1", sdkLanguage: Python, minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, shouldErr: false, }, { - name: "Test with incompatible Java version", - sdkVersion: "v0.4.3", - sdkLanguage: Java, + name: "python release version is compatible with minimum supported version", + sdkVersion: "v0.6.0", + sdkLanguage: Python, + minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, + shouldErr: false, + }, + { + name: "python pre-release version is lower than minimum supported version", + sdkVersion: "v0.5.3", + sdkLanguage: Python, minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, shouldErr: true, - errMessage: "SDK version 0.4.3 must be upgraded to at least 0.6.0-0, in order to work with current numaflow version", + errMessage: "SDK version 0.5.3 must be upgraded to at least 0.6.0rc100, in order to work with current numaflow version", + }, + { + name: "python release version is compatible with minimum supported version", + sdkVersion: "v0.6.0", + sdkLanguage: Python, + minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, + shouldErr: false, }, { - name: "Test with compatible Go version", - sdkVersion: "v0.6.0-rc2", + name: "java release version is compatible with minimum supported version", + sdkVersion: "v0.7.3", + sdkLanguage: Java, + minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, + shouldErr: false, + }, + { + name: "golang rc release version is compatible with minimum supported version", + sdkVersion: "v0.6.2-rc2", sdkLanguage: Go, minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, shouldErr: false, + }, { + name: "rust pre-release version is compatible with minimum supported version", + sdkVersion: "v0.1.2-0.20240913163521-4910018031a7", + sdkLanguage: Rust, + minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, + shouldErr: false, }, { - name: "Test with incompatible Rust version", + name: "rust release version is lower than minimum supported version", sdkVersion: "v0.0.3", sdkLanguage: Rust, minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, shouldErr: true, - errMessage: "SDK version 0.0.3 must be upgraded to at least 0.1.0, in order to work with current numaflow version", + errMessage: "SDK version 0.0.3 must be upgraded to at least 0.1.0-z, in order to work with current numaflow version", }, { - name: "Test with compatible Rust version", - sdkVersion: "v0.1.1", - sdkLanguage: Rust, + name: "java rc release version is lower than minimum supported version", + sdkVersion: "v0.6.0-rc1", + sdkLanguage: Java, minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, - shouldErr: false, + shouldErr: true, + errMessage: "SDK version 0.6.0-rc1 must be upgraded to at least 0.6.0-z, in order to work with current numaflow version", + }, + { + name: "golang pre-release version is lower than minimum supported version", + sdkVersion: "v0.6.0-0.20240913163521-4910018031a7", + sdkLanguage: Go, + minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, + shouldErr: true, + errMessage: "SDK version 0.6.0-0.20240913163521-4910018031a7 must be upgraded to at least 0.6.0-z, in order to work with current numaflow version", }, } for _, tt := range tests { diff --git a/pkg/sdkclient/serverinfo/types.go b/pkg/sdkclient/serverinfo/types.go index fc8fdd9b81..ed21721b84 100644 --- a/pkg/sdkclient/serverinfo/types.go +++ b/pkg/sdkclient/serverinfo/types.go @@ -27,11 +27,49 @@ const ( type sdkConstraints map[Language]string +/* +minimumSupportedSDKVersions is the minimum supported version of each SDK for the current numaflow version. +It is used to check if the SDK is compatible with the current numaflow version. + +Python SDK versioning follows PEP 440 (https://www.python.org/dev/peps/pep-0440/). +The other SDKs follow the semver versioning scheme (https://semver.org/). + +How to update this map: + +There are two types of releases, one is the stable release and the other is the pre-release. +Below are the typical formats of the versioning scheme: + + +------------------+-------------------------+-----------------------------+ + | | PEP 440 | semver | + +------------------+-------------------------+-----------------------------+ + | stable | 0.8.0 | 0.8.0 | + +------------------+-------------------------+-----------------------------+ + | pre-release | 0.8.0a1, | 0.8.0-rc1, | + | | 0.8.0b3, | 0.8.0-0.20240913163521, | + | | or 0.8.0rc1 | etc. | + +------------------+-------------------------+-----------------------------+ + +There are two cases to consider when updating the map: + +1. The minimum supported version is a pre-release version. +In this case, directly put the exact pre-release version in the map.E.g., +if the minimum supported version for java is "0.8.0-rc1", then put "0.8.0-rc1" for java. +"0.8.0b1", "0.8.0b1" for python. +2. The minimum supported version is a stable version. +In this case, put (almost) the largest available pre-release version of the stable version in the map. +E.g., if the minimum supported version is "0.8.0", then put "0.8.0-z" for java, go, rust, "0.8.0rc100" for python. + +More details about version comparison can be found in the PEP 440 and semver documentation. +*/ var minimumSupportedSDKVersions = sdkConstraints{ - Go: "0.8.0", - Python: "0.8.0", - Java: "0.8.0", - Rust: "0.1.0", + // meaning the minimum supported python SDK version is 0.8.0 + Python: "0.8.0rc100", + // meaning the minimum supported go SDK version is 0.8.0 + Go: "0.8.0-z", + // meaning the minimum supported java SDK version is 0.8.0 + Java: "0.8.0-z", + // meaning the minimum supported rust SDK version is 0.8.0 + Rust: "0.1.0-z", } type Protocol string From 92676025532c34e190ea4a27bc314b2aaa7f4275 Mon Sep 17 00:00:00 2001 From: Keran Yang Date: Tue, 17 Sep 2024 18:08:38 -0400 Subject: [PATCH 02/13] . Signed-off-by: Keran Yang --- pkg/sdkclient/serverinfo/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sdkclient/serverinfo/types.go b/pkg/sdkclient/serverinfo/types.go index ed21721b84..283e12fa87 100644 --- a/pkg/sdkclient/serverinfo/types.go +++ b/pkg/sdkclient/serverinfo/types.go @@ -68,7 +68,7 @@ var minimumSupportedSDKVersions = sdkConstraints{ Go: "0.8.0-z", // meaning the minimum supported java SDK version is 0.8.0 Java: "0.8.0-z", - // meaning the minimum supported rust SDK version is 0.8.0 + // meaning the minimum supported rust SDK version is 0.1.0 Rust: "0.1.0-z", } From 58e9170880d6041b1389346e18064d5ef3ff4cab Mon Sep 17 00:00:00 2001 From: Keran Yang Date: Tue, 17 Sep 2024 18:59:56 -0400 Subject: [PATCH 03/13] fix logs Signed-off-by: Keran Yang --- pkg/sdkclient/serverinfo/serverinfo.go | 6 +++--- pkg/sdkclient/serverinfo/serverinfo_test.go | 14 +++++++------- pkg/sdkclient/serverinfo/types.go | 21 +++++++++++++++++++++ 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/pkg/sdkclient/serverinfo/serverinfo.go b/pkg/sdkclient/serverinfo/serverinfo.go index 932ab2ff50..bb098744ef 100644 --- a/pkg/sdkclient/serverinfo/serverinfo.go +++ b/pkg/sdkclient/serverinfo/serverinfo.go @@ -170,7 +170,7 @@ func checkNumaflowCompatibility(numaflowVersion string, minNumaflowVersion strin numaflowConstraint := fmt.Sprintf(">= %s", minNumaflowVersion) if err = checkConstraint(numaflowVersionSemVer, numaflowConstraint); err != nil { return fmt.Errorf("numaflow version %s must be upgraded to at least %s, in order to work with current SDK version: %w", - numaflowVersionSemVer.String(), minNumaflowVersion, err) + numaflowVersionSemVer.String(), getRealMinimumVersion(minNumaflowVersion), err) } return nil } @@ -193,7 +193,7 @@ func checkSDKCompatibility(sdkVersion string, sdkLanguage Language, minSupported if !c.Check(sdkVersionPEP440) { return fmt.Errorf("SDK version %s must be upgraded to at least %s, in order to work with current numaflow version: %w", - sdkVersionPEP440.String(), sdkRequiredVersion, err) + sdkVersionPEP440.String(), getRealMinimumVersion(sdkRequiredVersion), err) } } else { sdkVersionSemVer, err := semver.NewVersion(sdkVersion) @@ -203,7 +203,7 @@ func checkSDKCompatibility(sdkVersion string, sdkLanguage Language, minSupported if err := checkConstraint(sdkVersionSemVer, sdkConstraint); err != nil { return fmt.Errorf("SDK version %s must be upgraded to at least %s, in order to work with current numaflow version: %w", - sdkVersionSemVer.String(), sdkRequiredVersion, err) + sdkVersionSemVer.String(), getRealMinimumVersion(sdkRequiredVersion), err) } } } diff --git a/pkg/sdkclient/serverinfo/serverinfo_test.go b/pkg/sdkclient/serverinfo/serverinfo_test.go index de22f32be6..0c2fe80b43 100644 --- a/pkg/sdkclient/serverinfo/serverinfo_test.go +++ b/pkg/sdkclient/serverinfo/serverinfo_test.go @@ -101,7 +101,7 @@ func Test_CheckNumaflowCompatibility(t *testing.T) { numaflowVersion: "v1.1.6", minNumaflowVersion: "1.1.7-z", shouldErr: true, - errMessage: "numaflow version 1.1.6 must be upgraded to at least 1.1.7-z, in order to work with current SDK version", + errMessage: "numaflow version 1.1.6 must be upgraded to at least 1.1.7, in order to work with current SDK version", }, { name: "Test with compatible numaflow version - min is a stable version 1.1.6", @@ -114,7 +114,7 @@ func Test_CheckNumaflowCompatibility(t *testing.T) { numaflowVersion: "v1.1.7-rc1", minNumaflowVersion: "1.1.7-z", shouldErr: true, - errMessage: "numaflow version 1.1.7-rc1 must be upgraded to at least 1.1.7-z, in order to work with current SDK version", + errMessage: "numaflow version 1.1.7-rc1 must be upgraded to at least 1.1.7, in order to work with current SDK version", }, { name: "Test with compatible numaflow version - min is a stable version 1.1.6, numaflow version is a pre-release version", @@ -190,7 +190,7 @@ func Test_CheckSDKCompatibility(t *testing.T) { sdkLanguage: Python, minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, shouldErr: true, - errMessage: "SDK version 0.5.3a1 must be upgraded to at least 0.6.0rc100, in order to work with current numaflow version", + errMessage: "SDK version 0.5.3a1 must be upgraded to at least 0.6.0, in order to work with current numaflow version", }, { name: "python pre-release version is compatible with minimum supported version", @@ -212,7 +212,7 @@ func Test_CheckSDKCompatibility(t *testing.T) { sdkLanguage: Python, minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, shouldErr: true, - errMessage: "SDK version 0.5.3 must be upgraded to at least 0.6.0rc100, in order to work with current numaflow version", + errMessage: "SDK version 0.5.3 must be upgraded to at least 0.6.0, in order to work with current numaflow version", }, { name: "python release version is compatible with minimum supported version", @@ -247,7 +247,7 @@ func Test_CheckSDKCompatibility(t *testing.T) { sdkLanguage: Rust, minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, shouldErr: true, - errMessage: "SDK version 0.0.3 must be upgraded to at least 0.1.0-z, in order to work with current numaflow version", + errMessage: "SDK version 0.0.3 must be upgraded to at least 0.1.0, in order to work with current numaflow version", }, { name: "java rc release version is lower than minimum supported version", @@ -255,7 +255,7 @@ func Test_CheckSDKCompatibility(t *testing.T) { sdkLanguage: Java, minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, shouldErr: true, - errMessage: "SDK version 0.6.0-rc1 must be upgraded to at least 0.6.0-z, in order to work with current numaflow version", + errMessage: "SDK version 0.6.0-rc1 must be upgraded to at least 0.6.0, in order to work with current numaflow version", }, { name: "golang pre-release version is lower than minimum supported version", @@ -263,7 +263,7 @@ func Test_CheckSDKCompatibility(t *testing.T) { sdkLanguage: Go, minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, shouldErr: true, - errMessage: "SDK version 0.6.0-0.20240913163521-4910018031a7 must be upgraded to at least 0.6.0-z, in order to work with current numaflow version", + errMessage: "SDK version 0.6.0-0.20240913163521-4910018031a7 must be upgraded to at least 0.6.0, in order to work with current numaflow version", }, } for _, tt := range tests { diff --git a/pkg/sdkclient/serverinfo/types.go b/pkg/sdkclient/serverinfo/types.go index 283e12fa87..46ac8ec363 100644 --- a/pkg/sdkclient/serverinfo/types.go +++ b/pkg/sdkclient/serverinfo/types.go @@ -16,6 +16,8 @@ limitations under the License. package serverinfo +import "strings" + type Language string const ( @@ -72,6 +74,25 @@ var minimumSupportedSDKVersions = sdkConstraints{ Rust: "0.1.0-z", } +// getRealMinimumVersion returns the real minimum supported version for the given version. +// it's used to help log the correct minimum supported version in the error message. +// it translates the version we used in the sdkConstraints map to the real minimum supported version. +// e.g., if the given version is "0.8.0rc100", the real minimum supported version is "0.8.0". +// if the given version is "0.8.0-z", the real minimum supported version is "0.8.0". +// if the given version is "0.8.0-rc1", the real minimum supported version is "0.8.0-rc1". +func getRealMinimumVersion(ver string) string { + if ver == "" { + return "" + } + if strings.HasSuffix(ver, "-z") { + return ver[:len(ver)-2] + } + if strings.HasSuffix(ver, "rc100") { + return ver[:len(ver)-5] + } + return ver +} + type Protocol string const ( From 02ec69a92e8d29d03c2faedf46ff631595897848 Mon Sep 17 00:00:00 2001 From: Keran Yang Date: Tue, 17 Sep 2024 19:07:01 -0400 Subject: [PATCH 04/13] . Signed-off-by: Keran Yang --- pkg/sdkclient/serverinfo/serverinfo.go | 6 +++--- pkg/sdkclient/serverinfo/types.go | 16 +++++++++------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/pkg/sdkclient/serverinfo/serverinfo.go b/pkg/sdkclient/serverinfo/serverinfo.go index bb098744ef..d01acd1cda 100644 --- a/pkg/sdkclient/serverinfo/serverinfo.go +++ b/pkg/sdkclient/serverinfo/serverinfo.go @@ -170,7 +170,7 @@ func checkNumaflowCompatibility(numaflowVersion string, minNumaflowVersion strin numaflowConstraint := fmt.Sprintf(">= %s", minNumaflowVersion) if err = checkConstraint(numaflowVersionSemVer, numaflowConstraint); err != nil { return fmt.Errorf("numaflow version %s must be upgraded to at least %s, in order to work with current SDK version: %w", - numaflowVersionSemVer.String(), getRealMinimumVersion(minNumaflowVersion), err) + numaflowVersionSemVer.String(), humanReadable(minNumaflowVersion), err) } return nil } @@ -193,7 +193,7 @@ func checkSDKCompatibility(sdkVersion string, sdkLanguage Language, minSupported if !c.Check(sdkVersionPEP440) { return fmt.Errorf("SDK version %s must be upgraded to at least %s, in order to work with current numaflow version: %w", - sdkVersionPEP440.String(), getRealMinimumVersion(sdkRequiredVersion), err) + sdkVersionPEP440.String(), humanReadable(sdkRequiredVersion), err) } } else { sdkVersionSemVer, err := semver.NewVersion(sdkVersion) @@ -203,7 +203,7 @@ func checkSDKCompatibility(sdkVersion string, sdkLanguage Language, minSupported if err := checkConstraint(sdkVersionSemVer, sdkConstraint); err != nil { return fmt.Errorf("SDK version %s must be upgraded to at least %s, in order to work with current numaflow version: %w", - sdkVersionSemVer.String(), getRealMinimumVersion(sdkRequiredVersion), err) + sdkVersionSemVer.String(), humanReadable(sdkRequiredVersion), err) } } } diff --git a/pkg/sdkclient/serverinfo/types.go b/pkg/sdkclient/serverinfo/types.go index 46ac8ec363..8dd107d12a 100644 --- a/pkg/sdkclient/serverinfo/types.go +++ b/pkg/sdkclient/serverinfo/types.go @@ -74,19 +74,21 @@ var minimumSupportedSDKVersions = sdkConstraints{ Rust: "0.1.0-z", } -// getRealMinimumVersion returns the real minimum supported version for the given version. -// it's used to help log the correct minimum supported version in the error message. -// it translates the version we used in the sdkConstraints map to the real minimum supported version. -// e.g., if the given version is "0.8.0rc100", the real minimum supported version is "0.8.0". -// if the given version is "0.8.0-z", the real minimum supported version is "0.8.0". -// if the given version is "0.8.0-rc1", the real minimum supported version is "0.8.0-rc1". -func getRealMinimumVersion(ver string) string { +// humanReadable returns the human-readable minimum supported version. +// it's used for logging purposes. +// it translates the version we used in the constraints to the real minimum supported version. +// e.g., if the given version is "0.8.0rc100", human-readable version is "0.8.0". +// if the given version is "0.8.0-z", "0.8.0". +// if "0.8.0-rc1", "0.8.0-rc1". +func humanReadable(ver string) string { if ver == "" { return "" } + // semver if strings.HasSuffix(ver, "-z") { return ver[:len(ver)-2] } + // PEP 440 if strings.HasSuffix(ver, "rc100") { return ver[:len(ver)-5] } From 66046d75bfe9d7c35eea6422d8bc50e83f374727 Mon Sep 17 00:00:00 2001 From: Keran Yang Date: Wed, 18 Sep 2024 14:27:45 -0400 Subject: [PATCH 05/13] todo - test for rc as minimum - rust better logging Signed-off-by: Keran Yang --- pkg/sdkclient/serverinfo/serverinfo_test.go | 11 +-- pkg/sdkclient/serverinfo/types.go | 2 + rust/monovertex/src/server_info.rs | 99 ++++++++++++++++----- 3 files changed, 79 insertions(+), 33 deletions(-) diff --git a/pkg/sdkclient/serverinfo/serverinfo_test.go b/pkg/sdkclient/serverinfo/serverinfo_test.go index 0c2fe80b43..0ea45eff60 100644 --- a/pkg/sdkclient/serverinfo/serverinfo_test.go +++ b/pkg/sdkclient/serverinfo/serverinfo_test.go @@ -200,27 +200,20 @@ func Test_CheckSDKCompatibility(t *testing.T) { shouldErr: false, }, { - name: "python release version is compatible with minimum supported version", + name: "python stable release version is compatible with minimum supported version", sdkVersion: "v0.6.0", sdkLanguage: Python, minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, shouldErr: false, }, { - name: "python pre-release version is lower than minimum supported version", + name: "python stable release version is lower than minimum supported version", sdkVersion: "v0.5.3", sdkLanguage: Python, minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, shouldErr: true, errMessage: "SDK version 0.5.3 must be upgraded to at least 0.6.0, in order to work with current numaflow version", }, - { - name: "python release version is compatible with minimum supported version", - sdkVersion: "v0.6.0", - sdkLanguage: Python, - minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, - shouldErr: false, - }, { name: "java release version is compatible with minimum supported version", sdkVersion: "v0.7.3", diff --git a/pkg/sdkclient/serverinfo/types.go b/pkg/sdkclient/serverinfo/types.go index 8dd107d12a..c981a84e9e 100644 --- a/pkg/sdkclient/serverinfo/types.go +++ b/pkg/sdkclient/serverinfo/types.go @@ -33,6 +33,8 @@ type sdkConstraints map[Language]string minimumSupportedSDKVersions is the minimum supported version of each SDK for the current numaflow version. It is used to check if the SDK is compatible with the current numaflow version. +NOTE: when updating it, please also update MINIMUM_SUPPORTED_SDK_VERSIONS for mono vertex at rust/monovertex/server_info.rs + Python SDK versioning follows PEP 440 (https://www.python.org/dev/peps/pep-0440/). The other SDKs follow the semver versioning scheme (https://semver.org/). diff --git a/rust/monovertex/src/server_info.rs b/rust/monovertex/src/server_info.rs index 225218b158..3423c1d49b 100644 --- a/rust/monovertex/src/server_info.rs +++ b/rust/monovertex/src/server_info.rs @@ -79,6 +79,32 @@ pub async fn check_for_server_compatibility( /// Checks if the given version meets the specified constraint. fn check_constraint(version: &Version, constraint: &str) -> error::Result<()> { + let binding = version.to_string(); + let mmp_version = binding.split('-').next().unwrap(); + println!("constraint: {}", constraint); + println!("mmp_version: {}", mmp_version); + println!("version: {}", version); + + // TODO - if constraint ends with -z, + // it means the minimum required version is a stable version, e.g. 0.8.0 + // update the constraint to be >=0.8.0 and check if the version satisfies the constraint + + // if a version is 0.8.0-rc1, it should not satisfy the constraint >=0.8.0 + // if a version is 0.8.0 or 0.9.0, it should satisfy the constraint >=0.8.0 + // if a version is 0.9.0-rc1, it should satisfy the constraint >=0.8.0 + if constraint.ends_with("-z") { + let stable_version = constraint.trim_end_matches("-z").trim_start_matches(">="); + let stable_constraint = format!(">={}", stable_version); + println!("stable_constraint: {}", stable_constraint); + // if the version is not prefixed with the stable_version, + // it's another mmp release, we trim to get that mmp version + if !version.to_string().starts_with(stable_version) { + println!("comparing mmp_version: {} with the stable constraint: {}", mmp_version, stable_constraint); + return check_constraint(&Version::parse(mmp_version).unwrap(), &stable_constraint); + } + return check_constraint(version, &stable_constraint); + } + // Parse the given constraint as a semantic version requirement let version_req = VersionReq::parse(constraint).map_err(|e| { Error::ServerInfoError(format!( @@ -257,11 +283,12 @@ mod version { static MINIMUM_SUPPORTED_SDK_VERSIONS: Lazy = Lazy::new(|| { // TODO: populate this from a static file and make it part of the release process // the value of the map matches `minimumSupportedSDKVersions` in pkg/sdkclient/serverinfo/types.go + // please follow the instruction there to update the value let mut m = HashMap::new(); - m.insert("go".to_string(), "0.8.0".to_string()); - m.insert("python".to_string(), "0.8.0".to_string()); - m.insert("java".to_string(), "0.8.0".to_string()); - m.insert("rust".to_string(), "0.1.0".to_string()); + m.insert("go".to_string(), "0.8.0-z".to_string()); + m.insert("python".to_string(), "0.8.0rc100".to_string()); + m.insert("java".to_string(), "0.8.0-z".to_string()); + m.insert("rust".to_string(), "0.1.0-z".to_string()); m }); @@ -398,18 +425,18 @@ mod tests { } // Helper function to create a SdkConstraints struct - fn create_sdk_constraints() -> version::SdkConstraints { + fn create_sdk_constraints() -> SdkConstraints { let mut constraints = HashMap::new(); - constraints.insert("python".to_string(), "1.2.0".to_string()); - constraints.insert("java".to_string(), "2.0.0".to_string()); - constraints.insert("go".to_string(), "0.10.0".to_string()); - constraints.insert("rust".to_string(), "0.1.0".to_string()); + constraints.insert("python".to_string(), "1.2.0rc100".to_string()); + constraints.insert("java".to_string(), "2.0.0-z".to_string()); + constraints.insert("go".to_string(), "0.10.0-z".to_string()); + constraints.insert("rust".to_string(), "0.1.0-z".to_string()); constraints } #[tokio::test] - async fn test_sdk_compatibility_python_valid() { - let sdk_version = "v1.3.0"; + async fn test_sdk_compatibility_python_stable_release_valid() { + let sdk_version = "1.3.0"; let sdk_language = "python"; let min_supported_sdk_versions = create_sdk_constraints(); @@ -420,7 +447,7 @@ mod tests { } #[tokio::test] - async fn test_sdk_compatibility_python_invalid() { + async fn test_sdk_compatibility_python_stable_release_invalid() { let sdk_version = "1.1.0"; let sdk_language = "python"; @@ -432,7 +459,31 @@ mod tests { } #[tokio::test] - async fn test_sdk_compatibility_java_valid() { + async fn test_sdk_compatibility_python_pre_release_valid() { + let sdk_version = "v1.3.0a1"; + let sdk_language = "python"; + + let min_supported_sdk_versions = create_sdk_constraints(); + let result = + check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); + + assert!(result.is_ok()); + } + + #[tokio::test] + async fn test_sdk_compatibility_python_pre_release_invalid() { + let sdk_version = "1.1.0a1"; + let sdk_language = "python"; + + let min_supported_sdk_versions = create_sdk_constraints(); + let result = + check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); + + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_sdk_compatibility_java_stable_release_valid() { let sdk_version = "v2.1.0"; let sdk_language = "java"; @@ -444,8 +495,8 @@ mod tests { } #[tokio::test] - async fn test_sdk_compatibility_java_invalid() { - let sdk_version = "1.5.0"; + async fn test_sdk_compatibility_java_rc_release_invalid() { + let sdk_version = "2.0.0-rc1"; let sdk_language = "java"; let min_supported_sdk_versions = create_sdk_constraints(); @@ -456,8 +507,8 @@ mod tests { } #[tokio::test] - async fn test_sdk_compatibility_go_valid() { - let sdk_version = "0.11.0"; + async fn test_sdk_compatibility_go_rc_release_valid() { + let sdk_version = "0.11.0-rc2"; let sdk_language = "go"; let min_supported_sdk_versions = create_sdk_constraints(); @@ -468,8 +519,8 @@ mod tests { } #[tokio::test] - async fn test_sdk_compatibility_go_invalid() { - let sdk_version = "0.9.0"; + async fn test_sdk_compatibility_go_pre_release_invalid() { + let sdk_version = "0.10.0-0.20240913163521-4910018031a7"; let sdk_language = "go"; let min_supported_sdk_versions = create_sdk_constraints(); @@ -480,8 +531,8 @@ mod tests { } #[tokio::test] - async fn test_sdk_compatibility_rust_valid() { - let sdk_version = "v0.1.0"; + async fn test_sdk_compatibility_rust_pre_release_valid() { + let sdk_version = "v0.1.1-0.20240913163521-4910018031a7"; let sdk_language = "rust"; let min_supported_sdk_versions = create_sdk_constraints(); @@ -492,7 +543,7 @@ mod tests { } #[tokio::test] - async fn test_sdk_compatibility_rust_invalid() { + async fn test_sdk_compatibility_rust_stable_release_invalid() { let sdk_version = "0.0.9"; let sdk_language = "rust"; @@ -591,7 +642,7 @@ mod tests { #[tokio::test] async fn test_read_server_info_success() { // Create a temporary directory - let dir = tempfile::tempdir().unwrap(); + let dir = tempdir().unwrap(); let file_path = dir.path().join("server_info.txt"); let cln_token = CancellationToken::new(); @@ -632,7 +683,7 @@ mod tests { #[tokio::test] async fn test_read_server_info_retry_limit() { // Create a temporary directory - let dir = tempfile::tempdir().unwrap(); + let dir = tempdir().unwrap(); let file_path = dir.path().join("server_info.txt"); // Write a partial test file not ending with END marker From 3f0cd5243f7a6474979619c8a62ca0e2703d7184 Mon Sep 17 00:00:00 2001 From: Keran Yang Date: Wed, 18 Sep 2024 16:32:22 -0400 Subject: [PATCH 06/13] human-readable log Signed-off-by: Keran Yang --- rust/monovertex/src/server_info.rs | 100 ++++++++++++----------------- 1 file changed, 41 insertions(+), 59 deletions(-) diff --git a/rust/monovertex/src/server_info.rs b/rust/monovertex/src/server_info.rs index 3423c1d49b..b3d3a31ea0 100644 --- a/rust/monovertex/src/server_info.rs +++ b/rust/monovertex/src/server_info.rs @@ -140,7 +140,7 @@ fn check_numaflow_compatibility( check_constraint(&numaflow_version_semver, &numaflow_constraint).map_err(|e| { Error::ServerInfoError(format!( "numaflow version {} must be upgraded to at least {}, in order to work with current SDK version {}", - numaflow_version_semver, min_numaflow_version, e + numaflow_version_semver, human_readable(min_numaflow_version.to_string()), e )) }) } @@ -167,7 +167,7 @@ fn check_sdk_compatibility( if !specifiers.contains(&sdk_version_pep440) { return Err(Error::ServerInfoError(format!( "SDK version {} must be upgraded to at least {}, in order to work with the current numaflow version", - sdk_version_pep440, sdk_required_version + sdk_version_pep440, human_readable(sdk_required_version.to_string()) ))); } } else { @@ -182,7 +182,7 @@ fn check_sdk_compatibility( check_constraint(&sdk_version_semver, &sdk_constraint).map_err(|_| { Error::ServerInfoError(format!( "SDK version {} must be upgraded to at least {}, in order to work with the current numaflow version", - sdk_version_semver, sdk_required_version + sdk_version_semver, human_readable(sdk_required_version.to_string()) )) })?; } @@ -202,6 +202,27 @@ fn check_sdk_compatibility( Ok(()) } +// human_readable returns the human-readable minimum supported version. +// it's used for logging purposes. +// it translates the version we used in the constraints to the real minimum supported version. +// e.g., if the given version is "0.8.0rc100", human-readable version is "0.8.0". +// if the given version is "0.8.0-z", "0.8.0". +// if "0.8.0-rc1", "0.8.0-rc1". +fn human_readable(ver: String) -> String { + if ver.is_empty() { + return String::new(); + } + // semver + if ver.ends_with("-z") { + return ver[..ver.len() - 2].to_string(); + } + // PEP 440 + if ver.ends_with("rc100") { + return ver[..ver.len() - 5].to_string(); + } + ver.to_string() +} + /// Reads the server info file and returns the parsed ServerInfo struct. /// The cancellation token is used to stop ready-check of server_info file in case it is missing. /// This cancellation token is closed via the global shutdown handler. @@ -456,6 +477,9 @@ mod tests { check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); assert!(result.is_err()); + assert!( + result.unwrap_err().to_string().contains( + "SDK version 1.1.0 must be upgraded to at least 1.2.0, in order to work with the current numaflow version")); } #[tokio::test] @@ -480,6 +504,9 @@ mod tests { check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); assert!(result.is_err()); + assert!( + result.unwrap_err().to_string().contains( + "SDK version 1.1.0a1 must be upgraded to at least 1.2.0, in order to work with the current numaflow version")); } #[tokio::test] @@ -504,6 +531,9 @@ mod tests { check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); assert!(result.is_err()); + assert!( + result.unwrap_err().to_string().contains( + "SDK version 2.0.0-rc1 must be upgraded to at least 2.0.0, in order to work with the current numaflow version")); } #[tokio::test] @@ -528,6 +558,9 @@ mod tests { check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); assert!(result.is_err()); + assert!( + result.unwrap_err().to_string().contains( + "SDK version 0.10.0-0.20240913163521-4910018031a7 must be upgraded to at least 0.10.0, in order to work with the current numaflow version")); } #[tokio::test] @@ -552,6 +585,9 @@ mod tests { check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); assert!(result.is_err()); + assert!( + result.unwrap_err().to_string().contains( + "ServerInfoError Error - SDK version 0.0.9 must be upgraded to at least 0.1.0, in order to work with the current numaflow version")); } #[tokio::test] @@ -572,6 +608,8 @@ mod tests { let result = check_numaflow_compatibility(numaflow_version, min_numaflow_version); assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains( + "numaflow version 1.2.0 must be upgraded to at least 1.3.0, in order to work with current SDK version")); } #[tokio::test] @@ -727,60 +765,4 @@ mod tests { let _parsed_server_info: ServerInfo = serde_json::from_str(&json_data).expect("Failed to parse JSON"); } - - #[test] - fn test_sdk_compatibility_go_version_with_v_prefix() { - let sdk_version = "v0.11.0"; - let sdk_language = "go"; - - let mut min_supported_sdk_versions = HashMap::new(); - min_supported_sdk_versions.insert("go".to_string(), "0.10.0".to_string()); - - let result = - check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); - - assert!(result.is_ok()); - } - - #[test] - fn test_sdk_compatibility_go_version_without_v_prefix() { - let sdk_version = "0.11.0"; - let sdk_language = "go"; - - let mut min_supported_sdk_versions = HashMap::new(); - min_supported_sdk_versions.insert("go".to_string(), "0.10.0".to_string()); - - let result = - check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); - - assert!(result.is_ok()); - } - - #[test] - fn test_sdk_compatibility_go_version_with_v_prefix_invalid() { - let sdk_version = "v0.9.0"; - let sdk_language = "go"; - - let mut min_supported_sdk_versions = HashMap::new(); - min_supported_sdk_versions.insert("go".to_string(), "0.10.0".to_string()); - - let result = - check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); - - assert!(result.is_err()); - } - - #[test] - fn test_sdk_compatibility_go_version_without_v_prefix_invalid() { - let sdk_version = "0.9.0"; - let sdk_language = "go"; - - let mut min_supported_sdk_versions = HashMap::new(); - min_supported_sdk_versions.insert("go".to_string(), "0.10.0".to_string()); - - let result = - check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); - - assert!(result.is_err()); - } } From cd0010808c3306689bb7b05bedb21ae4827651be Mon Sep 17 00:00:00 2001 From: Keran Yang Date: Wed, 18 Sep 2024 17:11:55 -0400 Subject: [PATCH 07/13] unit tests for pre-release Signed-off-by: Keran Yang --- pkg/sdkclient/serverinfo/serverinfo_test.go | 107 +++++++++- rust/monovertex/src/server_info.rs | 211 +++++++++++++++++--- 2 files changed, 294 insertions(+), 24 deletions(-) diff --git a/pkg/sdkclient/serverinfo/serverinfo_test.go b/pkg/sdkclient/serverinfo/serverinfo_test.go index 0ea45eff60..8473f0c3fe 100644 --- a/pkg/sdkclient/serverinfo/serverinfo_test.go +++ b/pkg/sdkclient/serverinfo/serverinfo_test.go @@ -169,7 +169,8 @@ func Test_CheckNumaflowCompatibility(t *testing.T) { } } -func Test_CheckSDKCompatibility(t *testing.T) { +// this test suite is to test SDK compatibility check when all the minimum-supported versions are stable releases +func Test_CheckSDKCompatibility_MinimumBeingStableReleases(t *testing.T) { var testMinimumSupportedSDKVersions = sdkConstraints{ Python: "0.6.0rc100", Go: "0.6.0-z", @@ -272,6 +273,110 @@ func Test_CheckSDKCompatibility(t *testing.T) { } } +// this test suite is to test SDK compatibility check when all the minimum-supported versions are pre-releases +func Test_CheckSDKCompatibility_MinimumBeingPreReleases(t *testing.T) { + var testMinimumSupportedSDKVersions = sdkConstraints{ + Python: "0.6.0b1", + Go: "0.6.0-rc2", + Java: "0.6.0-rc2", + Rust: "0.1.0-rc3", + } + tests := []struct { + name string + sdkVersion string + sdkLanguage Language + minimumSupportedSDKVersions sdkConstraints + shouldErr bool + errMessage string + }{ + { + name: "python pre-release version is lower than minimum supported version", + sdkVersion: "v0.5.3a1", + sdkLanguage: Python, + minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, + shouldErr: true, + errMessage: "SDK version 0.5.3a1 must be upgraded to at least 0.6.0b1, in order to work with current numaflow version", + }, + { + name: "python pre-release version is compatible with minimum supported version", + sdkVersion: "v0.6.3a1", + sdkLanguage: Python, + minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, + shouldErr: false, + }, + { + name: "python stable release version is compatible with minimum supported version", + sdkVersion: "v0.6.0", + sdkLanguage: Python, + minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, + shouldErr: false, + }, + { + name: "python stable release version is lower than minimum supported version", + sdkVersion: "v0.5.3", + sdkLanguage: Python, + minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, + shouldErr: true, + errMessage: "SDK version 0.5.3 must be upgraded to at least 0.6.0b1, in order to work with current numaflow version", + }, + { + name: "java release version is compatible with minimum supported version", + sdkVersion: "v0.7.3", + sdkLanguage: Java, + minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, + shouldErr: false, + }, + { + name: "golang rc release version is compatible with minimum supported version", + sdkVersion: "v0.6.2-rc2", + sdkLanguage: Go, + minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, + shouldErr: false, + }, { + name: "rust pre-release version is compatible with minimum supported version", + sdkVersion: "v0.1.2-0.20240913163521-4910018031a7", + sdkLanguage: Rust, + minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, + shouldErr: false, + }, + { + name: "rust release version is lower than minimum supported version", + sdkVersion: "v0.0.3", + sdkLanguage: Rust, + minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, + shouldErr: true, + errMessage: "SDK version 0.0.3 must be upgraded to at least 0.1.0-rc3, in order to work with current numaflow version", + }, + { + name: "java rc release version is lower than minimum supported version", + sdkVersion: "v0.6.0-rc1", + sdkLanguage: Java, + minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, + shouldErr: true, + errMessage: "SDK version 0.6.0-rc1 must be upgraded to at least 0.6.0-rc2, in order to work with current numaflow version", + }, + { + name: "golang pre-release version is lower than minimum supported version", + sdkVersion: "v0.6.0-0.20240913163521-4910018031a7", + sdkLanguage: Go, + minimumSupportedSDKVersions: testMinimumSupportedSDKVersions, + shouldErr: true, + errMessage: "SDK version 0.6.0-0.20240913163521-4910018031a7 must be upgraded to at least 0.6.0-rc2, in order to work with current numaflow version", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := checkSDKCompatibility(tt.sdkVersion, tt.sdkLanguage, tt.minimumSupportedSDKVersions) + if tt.shouldErr { + assert.Error(t, err, "Expected error") + assert.Contains(t, err.Error(), tt.errMessage) + } else { + assert.NoError(t, err, "Expected no error") + } + }) + } +} + // write is a test helper function to prepare server info file func write(svrInfo *ServerInfo, opts ...Option) error { b, err := json.Marshal(svrInfo) diff --git a/rust/monovertex/src/server_info.rs b/rust/monovertex/src/server_info.rs index b3d3a31ea0..ecf6cda7da 100644 --- a/rust/monovertex/src/server_info.rs +++ b/rust/monovertex/src/server_info.rs @@ -85,7 +85,9 @@ fn check_constraint(version: &Version, constraint: &str) -> error::Result<()> { println!("mmp_version: {}", mmp_version); println!("version: {}", version); - // TODO - if constraint ends with -z, + // TODO - if both rc, but not same mmp, compare the mmp version + + // DONE - if constraint ends with -z, // it means the minimum required version is a stable version, e.g. 0.8.0 // update the constraint to be >=0.8.0 and check if the version satisfies the constraint @@ -103,6 +105,16 @@ fn check_constraint(version: &Version, constraint: &str) -> error::Result<()> { return check_constraint(&Version::parse(mmp_version).unwrap(), &stable_constraint); } return check_constraint(version, &stable_constraint); + } else if constraint.contains("-") { + // if the constraint is not a stable version, + // we need + // to handle the case when the version to compare is also a pre-release version + // and it's mmp is different from the constraint's mmp + let stable_version = trim_after_dash(constraint.trim_start_matches(">=")); + let stable_constraint = format!(">={}", stable_version); + if !version.to_string().starts_with(stable_version) { + return check_constraint(&Version::parse(mmp_version).unwrap(), &stable_constraint); + } } // Parse the given constraint as a semantic version requirement @@ -121,6 +133,14 @@ fn check_constraint(version: &Version, constraint: &str) -> error::Result<()> { Ok(()) } +fn trim_after_dash(input: &str) -> &str { + if let Some(pos) = input.find('-') { + &input[..pos] + } else { + input + } +} + /// Checks if the current numaflow version is compatible with the given minimum numaflow version. fn check_numaflow_compatibility( numaflow_version: &str, @@ -445,8 +465,8 @@ mod tests { Ok(()) } - // Helper function to create a SdkConstraints struct - fn create_sdk_constraints() -> SdkConstraints { + // Helper function to create a SdkConstraints struct with minimum supported SDK versions all being stable releases + fn create_sdk_constraints_stable_versions() -> SdkConstraints { let mut constraints = HashMap::new(); constraints.insert("python".to_string(), "1.2.0rc100".to_string()); constraints.insert("java".to_string(), "2.0.0-z".to_string()); @@ -455,12 +475,22 @@ mod tests { constraints } + // Helper function to create a SdkConstraints struct with minimum supported SDK versions all being pre-releases + fn create_sdk_constraints_pre_release_versions() -> SdkConstraints { + let mut constraints = HashMap::new(); + constraints.insert("python".to_string(), "1.2.0b2".to_string()); + constraints.insert("java".to_string(), "2.0.0-rc2".to_string()); + constraints.insert("go".to_string(), "0.10.0-rc2".to_string()); + constraints.insert("rust".to_string(), "0.1.0-rc3".to_string()); + constraints + } + #[tokio::test] - async fn test_sdk_compatibility_python_stable_release_valid() { + async fn test_sdk_compatibility_min_stable_python_stable_release_valid() { let sdk_version = "1.3.0"; let sdk_language = "python"; - let min_supported_sdk_versions = create_sdk_constraints(); + let min_supported_sdk_versions = create_sdk_constraints_stable_versions(); let result = check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); @@ -468,11 +498,11 @@ mod tests { } #[tokio::test] - async fn test_sdk_compatibility_python_stable_release_invalid() { + async fn test_sdk_compatibility_min_stable_python_stable_release_invalid() { let sdk_version = "1.1.0"; let sdk_language = "python"; - let min_supported_sdk_versions = create_sdk_constraints(); + let min_supported_sdk_versions = create_sdk_constraints_stable_versions(); let result = check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); @@ -483,11 +513,11 @@ mod tests { } #[tokio::test] - async fn test_sdk_compatibility_python_pre_release_valid() { + async fn test_sdk_compatibility_min_stable_python_pre_release_valid() { let sdk_version = "v1.3.0a1"; let sdk_language = "python"; - let min_supported_sdk_versions = create_sdk_constraints(); + let min_supported_sdk_versions = create_sdk_constraints_stable_versions(); let result = check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); @@ -495,11 +525,11 @@ mod tests { } #[tokio::test] - async fn test_sdk_compatibility_python_pre_release_invalid() { + async fn test_sdk_compatibility_min_stable_python_pre_release_invalid() { let sdk_version = "1.1.0a1"; let sdk_language = "python"; - let min_supported_sdk_versions = create_sdk_constraints(); + let min_supported_sdk_versions = create_sdk_constraints_stable_versions(); let result = check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); @@ -510,11 +540,11 @@ mod tests { } #[tokio::test] - async fn test_sdk_compatibility_java_stable_release_valid() { + async fn test_sdk_compatibility_min_stable_java_stable_release_valid() { let sdk_version = "v2.1.0"; let sdk_language = "java"; - let min_supported_sdk_versions = create_sdk_constraints(); + let min_supported_sdk_versions = create_sdk_constraints_stable_versions(); let result = check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); @@ -522,11 +552,11 @@ mod tests { } #[tokio::test] - async fn test_sdk_compatibility_java_rc_release_invalid() { + async fn test_sdk_compatibility_min_stable_java_rc_release_invalid() { let sdk_version = "2.0.0-rc1"; let sdk_language = "java"; - let min_supported_sdk_versions = create_sdk_constraints(); + let min_supported_sdk_versions = create_sdk_constraints_stable_versions(); let result = check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); @@ -537,11 +567,11 @@ mod tests { } #[tokio::test] - async fn test_sdk_compatibility_go_rc_release_valid() { + async fn test_sdk_compatibility_min_stable_go_rc_release_valid() { let sdk_version = "0.11.0-rc2"; let sdk_language = "go"; - let min_supported_sdk_versions = create_sdk_constraints(); + let min_supported_sdk_versions = create_sdk_constraints_stable_versions(); let result = check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); @@ -549,11 +579,11 @@ mod tests { } #[tokio::test] - async fn test_sdk_compatibility_go_pre_release_invalid() { + async fn test_sdk_compatibility_min_stable_go_pre_release_invalid() { let sdk_version = "0.10.0-0.20240913163521-4910018031a7"; let sdk_language = "go"; - let min_supported_sdk_versions = create_sdk_constraints(); + let min_supported_sdk_versions = create_sdk_constraints_stable_versions(); let result = check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); @@ -564,11 +594,11 @@ mod tests { } #[tokio::test] - async fn test_sdk_compatibility_rust_pre_release_valid() { + async fn test_sdk_compatibility_min_stable_rust_pre_release_valid() { let sdk_version = "v0.1.1-0.20240913163521-4910018031a7"; let sdk_language = "rust"; - let min_supported_sdk_versions = create_sdk_constraints(); + let min_supported_sdk_versions = create_sdk_constraints_stable_versions(); let result = check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); @@ -576,11 +606,11 @@ mod tests { } #[tokio::test] - async fn test_sdk_compatibility_rust_stable_release_invalid() { + async fn test_sdk_compatibility_min_stable_rust_stable_release_invalid() { let sdk_version = "0.0.9"; let sdk_language = "rust"; - let min_supported_sdk_versions = create_sdk_constraints(); + let min_supported_sdk_versions = create_sdk_constraints_stable_versions(); let result = check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); @@ -590,6 +620,141 @@ mod tests { "ServerInfoError Error - SDK version 0.0.9 must be upgraded to at least 0.1.0, in order to work with the current numaflow version")); } + #[tokio::test] + async fn test_sdk_compatibility_min_pre_release_python_stable_release_valid() { + let sdk_version = "1.3.0"; + let sdk_language = "python"; + + let min_supported_sdk_versions = create_sdk_constraints_pre_release_versions(); + let result = + check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); + + assert!(result.is_ok()); + } + + #[tokio::test] + async fn test_sdk_compatibility_min_pre_release_python_stable_release_invalid() { + let sdk_version = "1.1.0"; + let sdk_language = "python"; + + let min_supported_sdk_versions = create_sdk_constraints_pre_release_versions(); + let result = + check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); + + assert!(result.is_err()); + assert!( + result.unwrap_err().to_string().contains( + "SDK version 1.1.0 must be upgraded to at least 1.2.0b2, in order to work with the current numaflow version")); + } + + #[tokio::test] + async fn test_sdk_compatibility_min_pre_release_python_pre_release_valid() { + let sdk_version = "v1.3.0a1"; + let sdk_language = "python"; + + let min_supported_sdk_versions = create_sdk_constraints_pre_release_versions(); + let result = + check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); + + assert!(result.is_ok()); + } + + #[tokio::test] + async fn test_sdk_compatibility_min_pre_release_python_pre_release_invalid() { + let sdk_version = "1.2.0a1"; + let sdk_language = "python"; + + let min_supported_sdk_versions = create_sdk_constraints_pre_release_versions(); + let result = + check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); + + assert!(result.is_err()); + assert!( + result.unwrap_err().to_string().contains( + "SDK version 1.2.0a1 must be upgraded to at least 1.2.0b2, in order to work with the current numaflow version")); + } + + #[tokio::test] + async fn test_sdk_compatibility_min_pre_release_java_stable_release_valid() { + let sdk_version = "v2.1.0"; + let sdk_language = "java"; + + let min_supported_sdk_versions = create_sdk_constraints_pre_release_versions(); + let result = + check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); + + assert!(result.is_ok()); + } + + #[tokio::test] + async fn test_sdk_compatibility_min_pre_release_java_rc_release_invalid() { + let sdk_version = "2.0.0-rc1"; + let sdk_language = "java"; + + let min_supported_sdk_versions = create_sdk_constraints_pre_release_versions(); + let result = + check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); + + assert!(result.is_err()); + assert!( + result.unwrap_err().to_string().contains( + "SDK version 2.0.0-rc1 must be upgraded to at least 2.0.0-rc2, in order to work with the current numaflow version")); + } + + #[tokio::test] + async fn test_sdk_compatibility_min_pre_release_go_rc_release_valid() { + let sdk_version = "0.11.0-rc2"; + let sdk_language = "go"; + + let min_supported_sdk_versions = create_sdk_constraints_pre_release_versions(); + let result = + check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); + + assert!(result.is_ok()); + } + + #[tokio::test] + async fn test_sdk_compatibility_min_pre_release_go_pre_release_invalid() { + let sdk_version = "0.10.0-0.20240913163521-4910018031a7"; + let sdk_language = "go"; + + let min_supported_sdk_versions = create_sdk_constraints_pre_release_versions(); + let result = + check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); + + assert!(result.is_err()); + assert!( + result.unwrap_err().to_string().contains( + "SDK version 0.10.0-0.20240913163521-4910018031a7 must be upgraded to at least 0.10.0-rc2, in order to work with the current numaflow version")); + } + + #[tokio::test] + async fn test_sdk_compatibility_min_pre_release_rust_pre_release_valid() { + let sdk_version = "v0.1.1-0.20240913163521-4910018031a7"; + let sdk_language = "rust"; + + let min_supported_sdk_versions = create_sdk_constraints_pre_release_versions(); + let result = + check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); + + assert!(result.is_ok()); + } + + #[tokio::test] + async fn test_sdk_compatibility_min_pre_release_rust_stable_release_invalid() { + let sdk_version = "0.0.9"; + let sdk_language = "rust"; + + let min_supported_sdk_versions = create_sdk_constraints_pre_release_versions(); + let result = + check_sdk_compatibility(sdk_version, sdk_language, &min_supported_sdk_versions); + + assert!(result.is_err()); + assert!( + result.unwrap_err().to_string().contains( + "ServerInfoError Error - SDK version 0.0.9 must be upgraded to at least 0.1.0-rc3, in order to work with the current numaflow version")); + } + #[tokio::test] async fn test_numaflow_compatibility_valid() { let numaflow_version = "1.4.0"; From 057af8ed3204cc8f1877fbf5d6eef4b39f8b3cd7 Mon Sep 17 00:00:00 2001 From: Keran Yang Date: Wed, 18 Sep 2024 18:38:55 -0400 Subject: [PATCH 08/13] clean up Signed-off-by: Keran Yang --- rust/monovertex/src/server_info.rs | 120 ++++++++++++++++++++++------- 1 file changed, 92 insertions(+), 28 deletions(-) diff --git a/rust/monovertex/src/server_info.rs b/rust/monovertex/src/server_info.rs index ecf6cda7da..da3896399c 100644 --- a/rust/monovertex/src/server_info.rs +++ b/rust/monovertex/src/server_info.rs @@ -80,39 +80,34 @@ pub async fn check_for_server_compatibility( /// Checks if the given version meets the specified constraint. fn check_constraint(version: &Version, constraint: &str) -> error::Result<()> { let binding = version.to_string(); + // extract the major.minor.patch version let mmp_version = binding.split('-').next().unwrap(); - println!("constraint: {}", constraint); - println!("mmp_version: {}", mmp_version); - println!("version: {}", version); - // TODO - if both rc, but not same mmp, compare the mmp version - - // DONE - if constraint ends with -z, - // it means the minimum required version is a stable version, e.g. 0.8.0 - // update the constraint to be >=0.8.0 and check if the version satisfies the constraint - - // if a version is 0.8.0-rc1, it should not satisfy the constraint >=0.8.0 - // if a version is 0.8.0 or 0.9.0, it should satisfy the constraint >=0.8.0 - // if a version is 0.9.0-rc1, it should satisfy the constraint >=0.8.0 if constraint.ends_with("-z") { + // the minimum supported version is a stable version let stable_version = constraint.trim_end_matches("-z").trim_start_matches(">="); let stable_constraint = format!(">={}", stable_version); - println!("stable_constraint: {}", stable_constraint); - // if the version is not prefixed with the stable_version, - // it's another mmp release, we trim to get that mmp version if !version.to_string().starts_with(stable_version) { - println!("comparing mmp_version: {} with the stable constraint: {}", mmp_version, stable_constraint); + // if the version is prefixed with a different mmp version, + // rust semver lib doesn't handle the comparison the same as golang. + // e.g., rust semver doesn't treat 0.9.0-rc* as larger than 0.8.0. + // this is because to rust semver, both are smaller than 0.9.0. + // so we need to handle this case manually by only comparing the mmp version. + // TODO - remove this once rust semver handles pre-release comparison the same as golang. + // https://github.com/dtolnay/semver/issues/323 return check_constraint(&Version::parse(mmp_version).unwrap(), &stable_constraint); } return check_constraint(version, &stable_constraint); } else if constraint.contains("-") { - // if the constraint is not a stable version, - // we need - // to handle the case when the version to compare is also a pre-release version - // and it's mmp is different from the constraint's mmp + // the minimum supported version is a pre-release version. let stable_version = trim_after_dash(constraint.trim_start_matches(">=")); let stable_constraint = format!(">={}", stable_version); if !version.to_string().starts_with(stable_version) { + // if the version is prefixed with a different mmp version, + // rust semver lib doesn't handle the comparison the same as golang. + // e.g., rust semver doesn't treat 0.9.0-rc* as larger than 0.8.0-rc*. + // TODO - remove this once rust semver handles pre-release comparison the same as golang. + // https://github.com/dtolnay/semver/issues/323 return check_constraint(&Version::parse(mmp_version).unwrap(), &stable_constraint); } } @@ -151,8 +146,11 @@ fn check_numaflow_compatibility( return Err(Error::ServerInfoError("invalid version".to_string())); } + // Strip the 'v' prefix if present. + let numaflow_version_stripped = numaflow_version.trim_start_matches('v'); + // Parse the provided numaflow version as a semantic version - let numaflow_version_semver = Version::parse(numaflow_version) + let numaflow_version_semver = Version::parse(numaflow_version_stripped) .map_err(|e| Error::ServerInfoError(format!("Error parsing Numaflow version: {}", e)))?; // Create a version constraint based on the minimum numaflow version @@ -756,9 +754,75 @@ mod tests { } #[tokio::test] - async fn test_numaflow_compatibility_valid() { - let numaflow_version = "1.4.0"; - let min_numaflow_version = "1.3.0"; + async fn test_numaflow_compatibility_min_stable_version_stable_valid() { + let numaflow_version = "v1.1.7"; + let min_numaflow_version = "1.1.6-z"; + + let result = check_numaflow_compatibility(numaflow_version, min_numaflow_version); + + assert!(result.is_ok()); + } + + #[tokio::test] + async fn test_numaflow_compatibility_min_stable_version_stable_invalid() { + let numaflow_version = "v1.1.6"; + let min_numaflow_version = "1.1.7-z"; + + let result = check_numaflow_compatibility(numaflow_version, min_numaflow_version); + + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains( + "numaflow version 1.1.6 must be upgraded to at least 1.1.7, in order to work with current SDK version")); + } + + #[tokio::test] + async fn test_numaflow_compatibility_min_stable_version_pre_release_valid() { + let numaflow_version = "1.1.7-rc1"; + let min_numaflow_version = "1.1.6-z"; + + let result = check_numaflow_compatibility(numaflow_version, min_numaflow_version); + + assert!(result.is_ok()); + } + + #[tokio::test] + async fn test_numaflow_compatibility_min_stable_version_pre_release_invalid() { + let numaflow_version = "v1.1.6-rc1"; + let min_numaflow_version = "1.1.6-z"; + + let result = check_numaflow_compatibility(numaflow_version, min_numaflow_version); + + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains( + "numaflow version 1.1.6-rc1 must be upgraded to at least 1.1.6, in order to work with current SDK version")); + } + + #[tokio::test] + async fn test_numaflow_compatibility_min_rc_version_stable_invalid() { + let numaflow_version = "v1.1.6"; + let min_numaflow_version = "1.1.7-rc1"; + + let result = check_numaflow_compatibility(numaflow_version, min_numaflow_version); + + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains( + "numaflow version 1.1.6 must be upgraded to at least 1.1.7-rc1, in order to work with current SDK version")); + } + + #[tokio::test] + async fn test_numaflow_compatibility_min_rc_version_stable_valid() { + let numaflow_version = "1.1.7"; + let min_numaflow_version = "1.1.6-rc1"; + + let result = check_numaflow_compatibility(numaflow_version, min_numaflow_version); + + assert!(result.is_ok()); + } + + #[tokio::test] + async fn test_numaflow_compatibility_min_rc_version_pre_release_valid() { + let numaflow_version = "1.1.7-rc3"; + let min_numaflow_version = "1.1.7-rc2"; let result = check_numaflow_compatibility(numaflow_version, min_numaflow_version); @@ -766,15 +830,15 @@ mod tests { } #[tokio::test] - async fn test_numaflow_compatibility_invalid() { - let numaflow_version = "1.2.0"; - let min_numaflow_version = "1.3.0"; + async fn test_numaflow_compatibility_min_rc_version_pre_release_invalid() { + let numaflow_version = "v1.1.6-rc1"; + let min_numaflow_version = "1.1.6-rc2"; let result = check_numaflow_compatibility(numaflow_version, min_numaflow_version); assert!(result.is_err()); assert!(result.unwrap_err().to_string().contains( - "numaflow version 1.2.0 must be upgraded to at least 1.3.0, in order to work with current SDK version")); + "numaflow version 1.1.6-rc1 must be upgraded to at least 1.1.6-rc2, in order to work with current SDK version")); } #[tokio::test] From 5101e72bffbbf677179b589f1eef6129a489ad92 Mon Sep 17 00:00:00 2001 From: Keran Yang Date: Wed, 18 Sep 2024 20:07:54 -0400 Subject: [PATCH 09/13] . Signed-off-by: Keran Yang --- pkg/sdkclient/serverinfo/types.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/sdkclient/serverinfo/types.go b/pkg/sdkclient/serverinfo/types.go index c981a84e9e..02c4611794 100644 --- a/pkg/sdkclient/serverinfo/types.go +++ b/pkg/sdkclient/serverinfo/types.go @@ -56,8 +56,8 @@ Below are the typical formats of the versioning scheme: There are two cases to consider when updating the map: 1. The minimum supported version is a pre-release version. -In this case, directly put the exact pre-release version in the map.E.g., -if the minimum supported version for java is "0.8.0-rc1", then put "0.8.0-rc1" for java. +In this case, directly put the exact pre-release version in the map. +E.g., if the minimum supported version is "0.8.0-rc1", then put "0.8.0-rc1" for java, go, rust. "0.8.0b1", "0.8.0b1" for python. 2. The minimum supported version is a stable version. In this case, put (almost) the largest available pre-release version of the stable version in the map. From fa302baf1962ba671f05ff666d727e5ccd68f43b Mon Sep 17 00:00:00 2001 From: Keran Yang Date: Wed, 18 Sep 2024 21:49:07 -0400 Subject: [PATCH 10/13] address comments Signed-off-by: Keran Yang --- pkg/sdkclient/serverinfo/serverinfo_test.go | 13 ++++++ rust/monovertex/src/server_info.rs | 51 ++++++++++++--------- 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/pkg/sdkclient/serverinfo/serverinfo_test.go b/pkg/sdkclient/serverinfo/serverinfo_test.go index 8473f0c3fe..ad7f06e690 100644 --- a/pkg/sdkclient/serverinfo/serverinfo_test.go +++ b/pkg/sdkclient/serverinfo/serverinfo_test.go @@ -155,6 +155,19 @@ func Test_CheckNumaflowCompatibility(t *testing.T) { shouldErr: true, errMessage: "server info does not contain minimum numaflow version. Upgrade to newer SDK version", }, + { + name: "Test with invalid numaflow version", + numaflowVersion: "", + minNumaflowVersion: "1.1.7", + shouldErr: true, + errMessage: "error parsing numaflow version: Invalid Semantic Version", + }, + { + name: "Test with empty min numaflow version", + numaflowVersion: "1.1.7", + shouldErr: true, + errMessage: "server info does not contain minimum numaflow version. Upgrade to newer SDK version", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/rust/monovertex/src/server_info.rs b/rust/monovertex/src/server_info.rs index da3896399c..0e53bb207a 100644 --- a/rust/monovertex/src/server_info.rs +++ b/rust/monovertex/src/server_info.rs @@ -81,37 +81,34 @@ pub async fn check_for_server_compatibility( fn check_constraint(version: &Version, constraint: &str) -> error::Result<()> { let binding = version.to_string(); // extract the major.minor.patch version - let mmp_version = binding.split('-').next().unwrap(); + let mmp_version = Version::parse(binding.split('-').next().unwrap_or_default()).map_err(|e| { + Error::ServerInfoError(format!("Error parsing version: {}, version string: {}", e, binding)) + })?; + let mmp_ver_str_constraint = trim_after_dash(constraint.trim_start_matches(">=")); + let mmp_ver_constraint = format!(">={}", mmp_ver_str_constraint); - if constraint.ends_with("-z") { + if constraint.contains("-z") { // the minimum supported version is a stable version - let stable_version = constraint.trim_end_matches("-z").trim_start_matches(">="); - let stable_constraint = format!(">={}", stable_version); - if !version.to_string().starts_with(stable_version) { + if !version.to_string().starts_with(mmp_ver_str_constraint) { // if the version is prefixed with a different mmp version, - // rust semver lib doesn't handle the comparison the same as golang. + // rust semver lib doesn't handle the comparison the same way as golang. // e.g., rust semver doesn't treat 0.9.0-rc* as larger than 0.8.0. - // this is because to rust semver, both are smaller than 0.9.0. + // this is because to rust semver, it only knows that both are smaller than 0.9.0. // so we need to handle this case manually by only comparing the mmp version. - // TODO - remove this once rust semver handles pre-release comparison the same as golang. - // https://github.com/dtolnay/semver/issues/323 - return check_constraint(&Version::parse(mmp_version).unwrap(), &stable_constraint); + return check_constraint(&mmp_version, &mmp_ver_constraint); } - return check_constraint(version, &stable_constraint); + return check_constraint(version, &mmp_ver_constraint); } else if constraint.contains("-") { - // the minimum supported version is a pre-release version. - let stable_version = trim_after_dash(constraint.trim_start_matches(">=")); - let stable_constraint = format!(">={}", stable_version); - if !version.to_string().starts_with(stable_version) { - // if the version is prefixed with a different mmp version, - // rust semver lib doesn't handle the comparison the same as golang. - // e.g., rust semver doesn't treat 0.9.0-rc* as larger than 0.8.0-rc*. - // TODO - remove this once rust semver handles pre-release comparison the same as golang. - // https://github.com/dtolnay/semver/issues/323 - return check_constraint(&Version::parse(mmp_version).unwrap(), &stable_constraint); + // the minimum supported version is a pre-release version + if !version.to_string().starts_with(mmp_ver_str_constraint) { + // similar reason as above, we compare the mmp version only. + return check_constraint(&mmp_version, &mmp_ver_constraint); } } + // TODO - remove all the extra check above once rust semver handles pre-release comparison the same way as golang. + // https://github.com/dtolnay/semver/issues/323 + // Parse the given constraint as a semantic version requirement let version_req = VersionReq::parse(constraint).map_err(|e| { Error::ServerInfoError(format!( @@ -753,6 +750,18 @@ mod tests { "ServerInfoError Error - SDK version 0.0.9 must be upgraded to at least 0.1.0-rc3, in order to work with the current numaflow version")); } + #[tokio::test] + async fn test_numaflow_compatibility_invalid_version_string() { + let numaflow_version = "v1.abc.7"; + let min_numaflow_version = "1.1.6-z"; + + let result = check_numaflow_compatibility(numaflow_version, min_numaflow_version); + + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains( + "Error parsing Numaflow version: unexpected character 'a' while parsing minor version number")); + } + #[tokio::test] async fn test_numaflow_compatibility_min_stable_version_stable_valid() { let numaflow_version = "v1.1.7"; From 46b0188905e38307d31f6131cc06e24a6cb8ec51 Mon Sep 17 00:00:00 2001 From: Keran Yang Date: Thu, 19 Sep 2024 09:56:19 -0400 Subject: [PATCH 11/13] nats Signed-off-by: Keran Yang --- pkg/shared/clients/nats/test/server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/shared/clients/nats/test/server.go b/pkg/shared/clients/nats/test/server.go index 556a7390c2..b64db98645 100644 --- a/pkg/shared/clients/nats/test/server.go +++ b/pkg/shared/clients/nats/test/server.go @@ -27,7 +27,7 @@ import ( func RunNatsServer(t *testing.T) *server.Server { t.Helper() opts := natstestserver.DefaultTestOptions - opts.Port = 4223 + opts.Port = -1 // Use random port to avoid conflicts return natstestserver.RunServer(&opts) } @@ -35,7 +35,7 @@ func RunNatsServer(t *testing.T) *server.Server { func RunJetStreamServer(t *testing.T) *server.Server { t.Helper() opts := natstestserver.DefaultTestOptions - opts.Port = -1 // Random port + opts.Port = -1 // Use random port to avoid conflicts opts.JetStream = true storeDir, err := os.MkdirTemp("", "") if err != nil { From 558dc0af4a1c33c5839234aef294236f96927085 Mon Sep 17 00:00:00 2001 From: Keran Yang Date: Thu, 19 Sep 2024 15:25:54 -0400 Subject: [PATCH 12/13] . Signed-off-by: Keran Yang --- pkg/sdkclient/serverinfo/types.go | 4 ++++ rust/monovertex/src/server_info.rs | 13 ++++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/pkg/sdkclient/serverinfo/types.go b/pkg/sdkclient/serverinfo/types.go index 02c4611794..9e4a152d03 100644 --- a/pkg/sdkclient/serverinfo/types.go +++ b/pkg/sdkclient/serverinfo/types.go @@ -61,7 +61,11 @@ E.g., if the minimum supported version is "0.8.0-rc1", then put "0.8.0-rc1" for "0.8.0b1", "0.8.0b1" for python. 2. The minimum supported version is a stable version. In this case, put (almost) the largest available pre-release version of the stable version in the map. +This is because the go semver library considers pre-releases to be invalid if the constraint range does not include pre-releases. +Therefore, we have to put a pre-release version of the stable version in the map and choose the largest one. +For python, we use "rc100" as the largest pre-release version. For go, rust, we use "-z" as the largest pre-release version. E.g., if the minimum supported version is "0.8.0", then put "0.8.0-z" for java, go, rust, "0.8.0rc100" for python. +A constraint ">=0.8.0-z" will match any pre-release version of 0.8.0, including "0.8.0-rc1", "0.8.0-rc2", etc. More details about version comparison can be found in the PEP 440 and semver documentation. */ diff --git a/rust/monovertex/src/server_info.rs b/rust/monovertex/src/server_info.rs index 0e53bb207a..7719c9b77c 100644 --- a/rust/monovertex/src/server_info.rs +++ b/rust/monovertex/src/server_info.rs @@ -87,19 +87,22 @@ fn check_constraint(version: &Version, constraint: &str) -> error::Result<()> { let mmp_ver_str_constraint = trim_after_dash(constraint.trim_start_matches(">=")); let mmp_ver_constraint = format!(">={}", mmp_ver_str_constraint); + // "-z" is used to indicate the minimum supported version is a stable version + // the reason why we choose the letter z is that it can represent the largest pre-release version. + // e.g., 0.8.0-z means the minimum supported version is 0.8.0. if constraint.contains("-z") { - // the minimum supported version is a stable version if !version.to_string().starts_with(mmp_ver_str_constraint) { // if the version is prefixed with a different mmp version, - // rust semver lib doesn't handle the comparison the same way as golang. + // rust semver lib can't figure out the correct order. + // to work around, we compare the mmp version only. // e.g., rust semver doesn't treat 0.9.0-rc* as larger than 0.8.0. - // this is because to rust semver, it only knows that both are smaller than 0.9.0. - // so we need to handle this case manually by only comparing the mmp version. + // to work around, instead of comparing 0.9.0-rc* with 0.8.0, + // we compare 0.9.0 with 0.8.0. return check_constraint(&mmp_version, &mmp_ver_constraint); } return check_constraint(version, &mmp_ver_constraint); } else if constraint.contains("-") { - // the minimum supported version is a pre-release version + // if the constraint doesn't contain "-z", but contains "-", it's a pre-release version. if !version.to_string().starts_with(mmp_ver_str_constraint) { // similar reason as above, we compare the mmp version only. return check_constraint(&mmp_version, &mmp_ver_constraint); From a7cd0079a6285ef82b07e35d1484a80d0e1b1149 Mon Sep 17 00:00:00 2001 From: Keran Yang Date: Thu, 19 Sep 2024 15:43:10 -0400 Subject: [PATCH 13/13] . Signed-off-by: Keran Yang --- rust/monovertex/src/server_info.rs | 126 ++++++++++++++--------------- 1 file changed, 63 insertions(+), 63 deletions(-) diff --git a/rust/monovertex/src/server_info.rs b/rust/monovertex/src/server_info.rs index 7719c9b77c..98484869fd 100644 --- a/rust/monovertex/src/server_info.rs +++ b/rust/monovertex/src/server_info.rs @@ -77,65 +77,6 @@ pub async fn check_for_server_compatibility( Ok(()) } -/// Checks if the given version meets the specified constraint. -fn check_constraint(version: &Version, constraint: &str) -> error::Result<()> { - let binding = version.to_string(); - // extract the major.minor.patch version - let mmp_version = Version::parse(binding.split('-').next().unwrap_or_default()).map_err(|e| { - Error::ServerInfoError(format!("Error parsing version: {}, version string: {}", e, binding)) - })?; - let mmp_ver_str_constraint = trim_after_dash(constraint.trim_start_matches(">=")); - let mmp_ver_constraint = format!(">={}", mmp_ver_str_constraint); - - // "-z" is used to indicate the minimum supported version is a stable version - // the reason why we choose the letter z is that it can represent the largest pre-release version. - // e.g., 0.8.0-z means the minimum supported version is 0.8.0. - if constraint.contains("-z") { - if !version.to_string().starts_with(mmp_ver_str_constraint) { - // if the version is prefixed with a different mmp version, - // rust semver lib can't figure out the correct order. - // to work around, we compare the mmp version only. - // e.g., rust semver doesn't treat 0.9.0-rc* as larger than 0.8.0. - // to work around, instead of comparing 0.9.0-rc* with 0.8.0, - // we compare 0.9.0 with 0.8.0. - return check_constraint(&mmp_version, &mmp_ver_constraint); - } - return check_constraint(version, &mmp_ver_constraint); - } else if constraint.contains("-") { - // if the constraint doesn't contain "-z", but contains "-", it's a pre-release version. - if !version.to_string().starts_with(mmp_ver_str_constraint) { - // similar reason as above, we compare the mmp version only. - return check_constraint(&mmp_version, &mmp_ver_constraint); - } - } - - // TODO - remove all the extra check above once rust semver handles pre-release comparison the same way as golang. - // https://github.com/dtolnay/semver/issues/323 - - // Parse the given constraint as a semantic version requirement - let version_req = VersionReq::parse(constraint).map_err(|e| { - Error::ServerInfoError(format!( - "Error parsing constraint: {}, constraint string: {}", - e, constraint - )) - })?; - - // Check if the provided version satisfies the parsed constraint - if !version_req.matches(version) { - return Err(Error::ServerInfoError("invalid version".to_string())); - } - - Ok(()) -} - -fn trim_after_dash(input: &str) -> &str { - if let Some(pos) = input.find('-') { - &input[..pos] - } else { - input - } -} - /// Checks if the current numaflow version is compatible with the given minimum numaflow version. fn check_numaflow_compatibility( numaflow_version: &str, @@ -158,7 +99,7 @@ fn check_numaflow_compatibility( check_constraint(&numaflow_version_semver, &numaflow_constraint).map_err(|e| { Error::ServerInfoError(format!( "numaflow version {} must be upgraded to at least {}, in order to work with current SDK version {}", - numaflow_version_semver, human_readable(min_numaflow_version.to_string()), e + numaflow_version_semver, human_readable(min_numaflow_version), e )) }) } @@ -185,7 +126,7 @@ fn check_sdk_compatibility( if !specifiers.contains(&sdk_version_pep440) { return Err(Error::ServerInfoError(format!( "SDK version {} must be upgraded to at least {}, in order to work with the current numaflow version", - sdk_version_pep440, human_readable(sdk_required_version.to_string()) + sdk_version_pep440, human_readable(sdk_required_version) ))); } } else { @@ -200,7 +141,7 @@ fn check_sdk_compatibility( check_constraint(&sdk_version_semver, &sdk_constraint).map_err(|_| { Error::ServerInfoError(format!( "SDK version {} must be upgraded to at least {}, in order to work with the current numaflow version", - sdk_version_semver, human_readable(sdk_required_version.to_string()) + sdk_version_semver, human_readable(sdk_required_version) )) })?; } @@ -226,7 +167,7 @@ fn check_sdk_compatibility( // e.g., if the given version is "0.8.0rc100", human-readable version is "0.8.0". // if the given version is "0.8.0-z", "0.8.0". // if "0.8.0-rc1", "0.8.0-rc1". -fn human_readable(ver: String) -> String { +fn human_readable(ver: &str) -> String { if ver.is_empty() { return String::new(); } @@ -241,6 +182,65 @@ fn human_readable(ver: String) -> String { ver.to_string() } +/// Checks if the given version meets the specified constraint. +fn check_constraint(version: &Version, constraint: &str) -> error::Result<()> { + let binding = version.to_string(); + // extract the major.minor.patch version + let mmp_version = Version::parse(binding.split('-').next().unwrap_or_default()).map_err(|e| { + Error::ServerInfoError(format!("Error parsing version: {}, version string: {}", e, binding)) + })?; + let mmp_ver_str_constraint = trim_after_dash(constraint.trim_start_matches(">=")); + let mmp_ver_constraint = format!(">={}", mmp_ver_str_constraint); + + // "-z" is used to indicate the minimum supported version is a stable version + // the reason why we choose the letter z is that it can represent the largest pre-release version. + // e.g., 0.8.0-z means the minimum supported version is 0.8.0. + if constraint.contains("-z") { + if !version.to_string().starts_with(mmp_ver_str_constraint) { + // if the version is prefixed with a different mmp version, + // rust semver lib can't figure out the correct order. + // to work around, we compare the mmp version only. + // e.g., rust semver doesn't treat 0.9.0-rc* as larger than 0.8.0. + // to work around, instead of comparing 0.9.0-rc* with 0.8.0, + // we compare 0.9.0 with 0.8.0. + return check_constraint(&mmp_version, &mmp_ver_constraint); + } + return check_constraint(version, &mmp_ver_constraint); + } else if constraint.contains("-") { + // if the constraint doesn't contain "-z", but contains "-", it's a pre-release version. + if !version.to_string().starts_with(mmp_ver_str_constraint) { + // similar reason as above, we compare the mmp version only. + return check_constraint(&mmp_version, &mmp_ver_constraint); + } + } + + // TODO - remove all the extra check above once rust semver handles pre-release comparison the same way as golang. + // https://github.com/dtolnay/semver/issues/323 + + // Parse the given constraint as a semantic version requirement + let version_req = VersionReq::parse(constraint).map_err(|e| { + Error::ServerInfoError(format!( + "Error parsing constraint: {}, constraint string: {}", + e, constraint + )) + })?; + + // Check if the provided version satisfies the parsed constraint + if !version_req.matches(version) { + return Err(Error::ServerInfoError("invalid version".to_string())); + } + + Ok(()) +} + +fn trim_after_dash(input: &str) -> &str { + if let Some(pos) = input.find('-') { + &input[..pos] + } else { + input + } +} + /// Reads the server info file and returns the parsed ServerInfo struct. /// The cancellation token is used to stop ready-check of server_info file in case it is missing. /// This cancellation token is closed via the global shutdown handler.