AVRO-3906: Add .NET 8.0 support (#2589)

* Add NEt8.0 support * Add liblzma to ARM 64 C# * Restore XZ library version * Add comment for Joveler.Compression.XZ ARM64 issue --------- Co-authored-by: Zoltan Csizmadia <[email protected]>
apache · Nov 22, 2023 · 5a279a9 · 5a279a9
1 parent c23c68b
commit 5a279a9
Show file tree

Hide file tree

Showing 10 changed files with 61 additions and 41 deletions.
diff --git a/.github/workflows/codeql-csharp-analysis.yml b/.github/workflows/codeql-csharp-analysis.yml
@@ -68,6 +68,7 @@ jobs:
           5.0.x
           6.0.x
           7.0.x
+          8.0.x
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

diff --git a/.github/workflows/test-lang-csharp.yml b/.github/workflows/test-lang-csharp.yml
@@ -50,6 +50,7 @@ jobs:
             5.0.x
             6.0.x
             7.0.x
+            8.0.x
 
       - uses: actions/cache@v3
         with:
@@ -81,6 +82,7 @@ jobs:
             5.0.x
             6.0.x
             7.0.x
+            8.0.x
 
       - name: Cache Local Maven Repository
         uses: actions/cache@v3
@@ -133,10 +135,11 @@ jobs:
           bash ./dotnet-install.sh --channel "5.0" --install-dir "$HOME/.dotnet" # 5.0
           bash ./dotnet-install.sh --channel "6.0" --install-dir "$HOME/.dotnet" # 6.0
           bash ./dotnet-install.sh --channel "7.0" --install-dir "$HOME/.dotnet" # 7.0
+          bash ./dotnet-install.sh --channel "8.0" --install-dir "$HOME/.dotnet" # 8.0
           
       - name: Build
         run: |
           set -x
           export PATH=$HOME/.dotnet:$PATH
           dotnet --list-sdks
-          ./build.sh clean test
+          ./build.sh clean test
diff --git a/.github/workflows/test-lang-java.yml b/.github/workflows/test-lang-java.yml
@@ -119,6 +119,7 @@ jobs:
             5.0.x
             6.0.x
             7.0.x
+            8.0.x
 
       - name: Install Java Avro for Interop Test
         working-directory: .
@@ -181,4 +182,4 @@ jobs:
           java -version
           mvn -version
           #MAVEN_OPTS="-Dsurefire.excludes=*TestCustomCodec*,*TestAllCodecs*,*TestNettyServer*" ./build.sh clean test
-          ./build.sh clean test
+          ./build.sh clean test
diff --git a/lang/csharp/README.md b/lang/csharp/README.md
@@ -12,25 +12,25 @@ Install-Package Apache.Avro
 
 ## Build & Test
 
-1. Install [.NET SDK 5.0+](https://dotnet.microsoft.com/download/dotnet-core)
+1. Install [.NET SDK 8.0+](https://dotnet.microsoft.com/download/dotnet-core)
 2. `dotnet test`
 
 ## Project Target Frameworks
 
-| Project             | Published to nuget.org     | Type       | .NET Standard 2.0  | .NET Standard 2.1 | .NET Core 3.1 | .NET 5.0  | .NET 6.0  | .NET 7.0  |
-|:-------------------:|:--------------------------:|:----------:|:------------------:|:-----------------:|:-------------:|:---------:|:---------:|:---------:|
-| Avro.main           | Apache.Avro                | Library    | ✔️                 | ✔️               |               |           |           |           |
-| Avro.File.Snappy    | Apache.Avro.File.Snappy    | Library    | ✔️                 | ✔️               |               |           |           |           |
-| Avro.File.BZip2     | Apache.Avro.File.BZip2     | Library    | ✔️                 | ✔️               |               |           |           |           |
-| Avro.File.XZ        | Apache.Avro.File.XZ        | Library    | ✔️                 | ✔️               |               |           |           |           |
-| Avro.File.Zstandard | Apache.Avro.File.Zstandard | Library    | ✔️                 | ✔️               |               |           |           |           |
-| Avro.codegen        | Apache.Avro.Tools          |  Exe        |                    |                   | ✔️            |✔️        |✔️        |✔️        |
-| Avro.ipc            |                            | Library    | ✔️                 | ✔️               |               |           |           |           |
-| Avro.ipc.test       |                            | Unit Tests |                    |                   | ✔️            |✔️        |✔️        |✔️        |
-| Avro.msbuild        |                            | Library    | ✔️                 | ✔️               |               |           |           |           |
-| Avro.perf           |                            | Exe        |                    |                   | ✔️            |✔️        |✔️        |✔️        |
-| Avro.test           |                            | Unit Tests |                    |                   | ✔️            |✔️        |✔️        |✔️        |
-| Avro.benchmark      |                            | Exe        |                    |                   | ✔️            |✔️        |✔️        |✔️        |
+| Project             | Published to nuget.org     | Type       | .NET Standard 2.0  | .NET Standard 2.1 | .NET Core 3.1 | .NET 5.0  | .NET 6.0  | .NET 7.0  | .NET 8.0  |
+|:-------------------:|:--------------------------:|:----------:|:------------------:|:-----------------:|:-------------:|:---------:|:---------:|:---------:|:---------:|
+| Avro.main           | Apache.Avro                | Library    | ✔️                 | ✔️               |               |           |           |           |           |
+| Avro.File.Snappy    | Apache.Avro.File.Snappy    | Library    | ✔️                 | ✔️               |               |           |           |           |           |
+| Avro.File.BZip2     | Apache.Avro.File.BZip2     | Library    | ✔️                 | ✔️               |               |           |           |           |           |
+| Avro.File.XZ        | Apache.Avro.File.XZ        | Library    | ✔️                 | ✔️               |               |           |           |           |           |
+| Avro.File.Zstandard | Apache.Avro.File.Zstandard | Library    | ✔️                 | ✔️               |               |           |           |           |           |
+| Avro.codegen        | Apache.Avro.Tools          |  Exe        |                    |                   | ✔️            |✔️        |✔️        |✔️        |✔️        |
+| Avro.ipc            |                            | Library    | ✔️                 | ✔️               |               |           |           |           |           |
+| Avro.ipc.test       |                            | Unit Tests |                    |                   | ✔️            |✔️        |✔️        |✔️        |✔️        |
+| Avro.msbuild        |                            | Library    | ✔️                 | ✔️               |               |           |           |           |           |
+| Avro.perf           |                            | Exe        |                    |                   | ✔️            |✔️        |✔️        |✔️        |✔️        |
+| Avro.test           |                            | Unit Tests |                    |                   | ✔️            |✔️        |✔️        |✔️        |✔️        |
+| Avro.benchmark      |                            | Exe        |                    |                   | ✔️            |✔️        |✔️        |✔️        |✔️        |
 
 ## Dependency package version strategy
 

diff --git a/lang/csharp/build.sh b/lang/csharp/build.sh
@@ -42,7 +42,7 @@ do
 
     perf)
       pushd ./src/apache/perf/
-      dotnet run --configuration Release --framework net7.0
+      dotnet run --configuration Release --framework net8.0
       ;;
 
     dist)
@@ -77,7 +77,7 @@ do
       ;;
 
     interop-data-generate)
-      dotnet run --project src/apache/test/Avro.test.csproj --framework net7.0 ../../share/test/schemas/interop.avsc ../../build/interop/data
+      dotnet run --project src/apache/test/Avro.test.csproj --framework net8.0 ../../share/test/schemas/interop.avsc ../../build/interop/data
       ;;
 
     interop-data-test)

diff --git a/lang/csharp/common.props b/lang/csharp/common.props
@@ -37,7 +37,7 @@
 
   <PropertyGroup Label="Target Frameworks">
     <!-- Exe -->
-    <DefaultExeTargetFrameworks>netcoreapp3.1;net5.0;net6.0;net7.0</DefaultExeTargetFrameworks>
+    <DefaultExeTargetFrameworks>netcoreapp3.1;net5.0;net6.0;net7.0;net8.0</DefaultExeTargetFrameworks>
     <!-- Library -->
     <DefaultLibraryTargetFrameworks>netstandard2.0;netstandard2.1</DefaultLibraryTargetFrameworks>
     <!-- Unit Tests -->

diff --git a/lang/csharp/src/apache/benchmark/Program.cs b/lang/csharp/src/apache/benchmark/Program.cs
@@ -21,8 +21,8 @@ namespace Avro.Benchmark
 {
     public class Program
     {
-        // dotnet run -c Release -f net7.0
-        // dotnet run -c Release -f net7.0 --runtimes netcoreapp3.1 net5.0 net6.0 net7.0
+        // dotnet run -c Release -f net8.0
+        // dotnet run -c Release -f net8.0 --runtimes netcoreapp3.1 net5.0 net6.0 net7.0 net8.0
         public static void Main(string[] args)
         {
             BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);

diff --git a/lang/csharp/src/apache/test/AvroGen/AvroGenHelper.cs b/lang/csharp/src/apache/test/AvroGen/AvroGenHelper.cs
@@ -181,7 +181,18 @@ public static Assembly CompileCSharpFilesAndCheckTypes(
             if (typeNamesToCheck != null)
             {
                 // Check if the compiled code has the same number of types defined as the check list
-                Assert.That(typeNamesToCheck.Count(), Is.EqualTo(assembly.DefinedTypes.Count()));
+                // Note: Ignore types which are injected by the compiler (System.* and Microsoft.*), e.g. Microsoft.CodeAnalysis.EmbeddedAttribute
+                Assert.That(
+                    typeNamesToCheck.Count(),
+                    Is.EqualTo(
+                        assembly
+                        .DefinedTypes
+                        .Where(t =>
+                        {
+                            return !t.Namespace.StartsWith("Microsoft.", StringComparison.OrdinalIgnoreCase) &&
+                                   !t.Namespace.StartsWith("System.", StringComparison.OrdinalIgnoreCase);
+                        })
+                        .Count()));
 
                 // Check if types available in compiled assembly
                 foreach (string typeName in typeNamesToCheck)

diff --git a/lang/csharp/versions.props b/lang/csharp/versions.props
@@ -27,14 +27,16 @@
   -->
   <PropertyGroup Label="Latest Package Versions">
     <NewtonsoftJsonVersion>13.0.1</NewtonsoftJsonVersion>
-    <SystemCodeDomVersion>7.0.0</SystemCodeDomVersion>
+    <SystemCodeDomVersion>8.0.0</SystemCodeDomVersion>
     <SystemReflectionVersion>4.3.0</SystemReflectionVersion>
     <SystemReflectionEmitILGenerationVersion>4.7.0</SystemReflectionEmitILGenerationVersion>
     <SystemReflectionEmitLightweightVersion>4.7.0</SystemReflectionEmitLightweightVersion>
 
     <!-- The following packages are required for the extra codec libraries. These are not direct dependencies of the Avro.main library. -->
-    <SharpZipLibVersion>1.4.1</SharpZipLibVersion>
-    <IronSnappyVersion>1.3.0</IronSnappyVersion>
+    <SharpZipLibVersion>1.4.2</SharpZipLibVersion>
+    <IronSnappyVersion>1.3.1</IronSnappyVersion>
+    <!-- As of 11/19/2023, Joveler.Compression.XZ 4.3+ is not compatible with the Linux ARM64 C# github workflow.
+    Keep it at 4.1.0 until the issue is resolved. Maybe the shipped liblzma.so is incompatible? -->
     <JovelerCompressionXZVersion>4.1.0</JovelerCompressionXZVersion>
     <ZstandardNetVersion>1.1.7</ZstandardNetVersion>
   </PropertyGroup>
@@ -47,27 +49,28 @@
     See https://github.com/apache/avro/pull/1126 & https://github.com/apache/avro/pull/981 for more details
   -->
   <PropertyGroup Label="Minimum Package Versions">
-    <NewtonsoftJsonMinimumVersion>10.0.3</NewtonsoftJsonMinimumVersion>
+    <!-- Newtonsoft.Json prior to version 13.0.1 is vulnerable. See https://github.com/advisories/GHSA-5crp-9r3c-p9vr -->
+    <NewtonsoftJsonMinimumVersion>13.0.1</NewtonsoftJsonMinimumVersion>
   </PropertyGroup>
 
   <!--
     These packages are used for building, testing and code analysis. Most likely they are safe to upgrade
 	Please sort the packages alphabetically
   -->
   <PropertyGroup Label="Build, Test, Code Analysis, Benchmark Package Versions">
-    <BenchmarkDotNetVersion>0.13.2</BenchmarkDotNetVersion>
-    <CoverletCollectorVersion>3.2.0</CoverletCollectorVersion>
-    <CoverletMSBuildVersion>3.2.0</CoverletMSBuildVersion>
-    <MicrosoftBuildFrameworkVersion>17.4.0</MicrosoftBuildFrameworkVersion>
-    <MicrosoftBuildUtilitiesCoreVersion>17.4.0</MicrosoftBuildUtilitiesCoreVersion>
-    <MicrosoftCodeAnalysisVersion>4.3.1</MicrosoftCodeAnalysisVersion>
-    <MicrosoftCodeAnalysisCSharpVersion>4.3.1</MicrosoftCodeAnalysisCSharpVersion>
-    <MicrosoftCodeAnalysisCSharpCodeStyleVersion>4.3.1</MicrosoftCodeAnalysisCSharpCodeStyleVersion>
-    <MicrosoftCodeAnalysisNetAnalyzersVersion>7.0.1</MicrosoftCodeAnalysisNetAnalyzersVersion>
-    <MicrosoftNETTestSdkVersion>17.4.0</MicrosoftNETTestSdkVersion>
-    <NUnitVersion>3.13.3</NUnitVersion>
-    <NUnitConsoleRunnerVersion>3.15.2</NUnitConsoleRunnerVersion>
-    <NUnit3TestAdapterVersion>4.3.0</NUnit3TestAdapterVersion>
+    <BenchmarkDotNetVersion>0.13.10</BenchmarkDotNetVersion>
+    <CoverletCollectorVersion>6.0.0</CoverletCollectorVersion>
+    <CoverletMSBuildVersion>6.0.0</CoverletMSBuildVersion>
+    <MicrosoftBuildFrameworkVersion>17.8.3</MicrosoftBuildFrameworkVersion>
+    <MicrosoftBuildUtilitiesCoreVersion>17.8.3</MicrosoftBuildUtilitiesCoreVersion>
+    <MicrosoftCodeAnalysisVersion>4.7.0</MicrosoftCodeAnalysisVersion>
+    <MicrosoftCodeAnalysisCSharpVersion>4.7.0</MicrosoftCodeAnalysisCSharpVersion>
+    <MicrosoftCodeAnalysisCSharpCodeStyleVersion>4.7.0</MicrosoftCodeAnalysisCSharpCodeStyleVersion>
+    <MicrosoftCodeAnalysisNetAnalyzersVersion>8.0.0</MicrosoftCodeAnalysisNetAnalyzersVersion>
+    <MicrosoftNETTestSdkVersion>17.8.0</MicrosoftNETTestSdkVersion>
+    <NUnitVersion>3.14.0</NUnitVersion>
+    <NUnitConsoleRunnerVersion>3.16.3</NUnitConsoleRunnerVersion>
+    <NUnit3TestAdapterVersion>4.5.0</NUnit3TestAdapterVersion>
     <StyleCopAnalyzersVersion>1.1.118</StyleCopAnalyzersVersion>
   </PropertyGroup>
 </Project>
diff --git a/share/docker/Dockerfile b/share/docker/Dockerfile
@@ -227,7 +227,8 @@ RUN cd /opt ; \
     bash ./dotnet-install.sh --channel "3.1" --install-dir "/opt/dotnet" ; \
     bash ./dotnet-install.sh --channel "5.0" --install-dir "/opt/dotnet" ; \
     bash ./dotnet-install.sh --channel "6.0" --install-dir "/opt/dotnet" ; \
-    bash ./dotnet-install.sh --channel "7.0" --install-dir "/opt/dotnet" ;
+    bash ./dotnet-install.sh --channel "7.0" --install-dir "/opt/dotnet" ; \
+    bash ./dotnet-install.sh --channel "8.0" --install-dir "/opt/dotnet" ;
 
 ENV PATH $PATH:/opt/dotnet
-Original file line number
+Diff line change
@@ Expand Up / @@ -68,6 +68,7 @@ jobs: @@
 .0.x
 .0.x
 .0.x
+.0.x
         # Initializes the CodeQL tools for scanning.
         - name: Initialize CodeQL
@@ Expand Down @@