From b095acb3afc40b0df423307f6fa394dc29f5a205 Mon Sep 17 00:00:00 2001 From: Kashif Khan Date: Thu, 11 Aug 2022 08:34:12 -0500 Subject: [PATCH 1/4] linux only jobs --- .azure-pipelines/client.test.live.yml | 268 +------------------------- 1 file changed, 1 insertion(+), 267 deletions(-) diff --git a/.azure-pipelines/client.test.live.yml b/.azure-pipelines/client.test.live.yml index 1bdf159b..c6fe3975 100644 --- a/.azure-pipelines/client.test.live.yml +++ b/.azure-pipelines/client.test.live.yml @@ -10,218 +10,7 @@ variables: GetPip: 'https://bootstrap.pypa.io/get-pip.py' jobs: - - job: 'Windows' - - pool: - vmImage: 'windows-2019' - - timeoutInMinutes: 120 - - strategy: - maxParallel: 1 - matrix: - x64 Python 3.6: - PythonArchitecture: 'x64' - PythonVersion: '$(PythonVersion36)' - x64 Python 3.7: - PythonArchitecture: 'x64' - PythonVersion: '$(PythonVersion37)' - x64 Python 3.8: - PythonArchitecture: 'x64' - PythonVersion: '$(PythonVersion38)' - x64 Python 3.9: - PythonArchitecture: 'x64' - PythonVersion: '$(PythonVersion39)' - x64 Python 3.10: - PythonArchitecture: 'x64' - PythonVersion: '$(PythonVersion310)' - x86 Python 3.6: - PythonArchitecture: 'x86' - PythonVersion: '$(PythonVersion36)' - x86 Python 3.7: - PythonArchitecture: 'x86' - PythonVersion: '$(PythonVersion37)' - x86 Python 3.8: - PythonArchitecture: 'x86' - PythonVersion: '$(PythonVersion38)' - x86 Python 3.9: - PythonArchitecture: 'x86' - PythonVersion: '$(PythonVersion39)' - x86 Python 3.10: - PythonArchitecture: 'x86' - PythonVersion: '$(PythonVersion310)' - - steps: - - template: /.azure-pipelines/use-python-version.yml - parameters: - architecture: '$(PythonArchitecture)' - versionSpec: '$(PythonVersion)' - - - powershell: | - python --version - Invoke-WebRequest -UseBasicParsing -Uri $(GetPip) | Select-Object -ExpandProperty Content | python - python -m pip --version - python -m pip install -r dev_requirements.txt - displayName: 'Install dependencies' - env: - PYTHONWARNINGS: ignore:DEPRECATION - - - script: python setup.py bdist_wheel - displayName: 'Build uAMQP Wheel' - env: - UAMQP_REBUILD_PYX: 'True' - - - script: dir dist - displayName: 'Check output' - - - powershell: | - $whlfile = Get-ChildItem -Filter *.whl dist | Select-Object -First 1 -ExpandProperty Name - python -m pip install --ignore-installed ./dist/$whlfile - pytest tests --doctest-modules --junitxml=junit/test-results-c.xml - pytest samples --doctest-modules --junitxml=junit/test-results-live.xml - displayName: 'Run tests' - env: - EVENT_HUB_HOSTNAME: $(python-eh-livetest-event-hub-hostname) - EVENT_HUB_NAME: $(python-eh-livetest-event-hub-name) - EVENT_HUB_SAS_POLICY: $(python-eh-livetest-event-hub-sas-policy) - EVENT_HUB_SAS_KEY: $(python-eh-livetest-event-hub-sas-key) - IOTHUB_HOSTNAME: $(python-iothub-livetest-host-name) - IOTHUB_HUB_NAME: $(python-iothub-livetest-hub-name) - IOTHUB_DEVICE: $(python-eh-livetest-event-hub-iothub-device) - IOTHUB_ENDPOINT: $(python-iothub-livetest-endpoint) - IOTHUB_SAS_POLICY: $(python-iothub-livetest-sas-policy) - IOTHUB_SAS_KEY: $(python-iothub-livetest-sas-key) - - - task: PublishTestResults@2 - displayName: 'Publish test results' - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-results-*.xml' - testResultsFormat: 'JUnit' - testRunTitle: 'Windows $(PythonArchitecture) Python $(PythonVersion)' - - - task: PublishBuildArtifacts@1 - displayName: 'Publish wheel artifact' - inputs: - artifactName: uamqp-win$(PythonArchitecture)-$(PythonVersion)-whl - pathToPublish: 'dist' - - - job: 'MacOS' - - dependsOn: 'Windows' - - timeoutInMinutes: 120 - - pool: - vmImage: 'macOS-10.15' - - strategy: - maxParallel: 1 - matrix: - Python 3.6: - PythonBin: 'python3' - PythonVersion: '$(PythonVersion36)' - Python 3.7: - PythonBin: 'python3' - PythonVersion: '$(PythonVersion37)' - Python 3.8: - PythonBin: 'python3' - PythonVersion: '$(PythonVersion38)' - Python 3.9: - PythonBin: 'python3' - PythonVersion: '$(PythonVersion39)' - Python 3.10: - PythonBin: 'python3' - PythonVersion: '$(PythonVersion310)' - - variables: - MacOSXDeploymentTarget: '10.9' - OpenSSLDir: $(Agent.BuildDirectory)/openssl-macosx$(MacOSXDeploymentTarget) - PythonVersion36: '3.6.8' - PythonVersion37: '3.7.6' - PythonVersion38: '3.8.2' - PythonVersion39: '3.9.0' - PythonVersion310: '3.10.0' - - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download OpenSSL artifact' - inputs: - artifactName: openssl-macosx$(MacOSXDeploymentTarget) - buildType: specific - buildVersionToDownload: latest - downloadPath: $(Agent.BuildDirectory) - pipeline: 119 # azure-uamqp-python - openssl - project: '29ec6040-b234-4e31-b139-33dc4287b756' # public - - - script: sudo xcode-select --switch /Applications/Xcode_11.3.1.app - displayName: 'Select Xcode 11.3.1' - - - script: source ./install_python_osx.sh - displayName: 'Install Official Python' - - - script: | - echo "Prepending PATH environment variable with directory: /usr/local/bin" - echo "##vso[task.prependpath]/usr/local/bin" - export PYTHON_SHORT_VER=`echo $(PythonVersion) | cut -d . -f 1,2` - echo "Prepending PATH environment variable with directory: $HOME/Library/Python/$PYTHON_SHORT_VER/bin" - echo "##vso[task.prependpath]$HOME/Library/Python/$PYTHON_SHORT_VER/bin" - displayName: 'Prepare PATH' - - - script: | - $(PythonBin) --version - curl -sS $(GetPip) | $(PythonBin) - --user - python -m pip --version - $(PythonBin) -m pip install --user -r dev_requirements.txt - displayName: 'Install dependencies' - - - script: $(PythonBin) setup.py bdist_wheel - displayName: 'Build uAMQP Wheel' - env: - CFLAGS: -mmacosx-version-min=$(MacOSXDeploymentTarget) -I$(OpenSSLDir)/include - CMAKE_OSX_ARCHITECTURES: 'x86_64' - CMAKE_OSX_DEPLOYMENT_TARGET: $(MacOSXDeploymentTarget) - LDFLAGS: -mmacosx-version-min=$(MacOSXDeploymentTarget) -L$(OpenSSLDir)/lib - MACOSX_DEPLOYMENT_TARGET: $(MacOSXDeploymentTarget) - OPENSSL_INCLUDE_DIR: $(OpenSSLDir)/include - OPENSSL_ROOT_DIR: $(OpenSSLDir) - UAMQP_REBUILD_PYX: 'True' - UAMQP_USE_OPENSSL: 'True' - - - script: ls ./dist - displayName: 'Check output' - - - script: | - $(PythonBin) -m pip install --user --ignore-installed ./dist/*.whl - pytest tests --doctest-modules --junitxml=junit/test-results-c.xml - pytest samples --doctest-modules --junitxml=junit/test-results-live.xml - displayName: 'Run tests' - env: - EVENT_HUB_HOSTNAME: $(python-eh-livetest-event-hub-hostname) - EVENT_HUB_NAME: $(python-eh-livetest-event-hub-name) - EVENT_HUB_SAS_POLICY: $(python-eh-livetest-event-hub-sas-policy) - EVENT_HUB_SAS_KEY: $(python-eh-livetest-event-hub-sas-key) - IOTHUB_HOSTNAME: $(python-iothub-livetest-host-name) - IOTHUB_HUB_NAME: $(python-iothub-livetest-hub-name) - IOTHUB_DEVICE: $(python-eh-livetest-event-hub-iothub-device) - IOTHUB_ENDPOINT: $(python-iothub-livetest-endpoint) - IOTHUB_SAS_POLICY: $(python-iothub-livetest-sas-policy) - IOTHUB_SAS_KEY: $(python-iothub-livetest-sas-key) - - - task: PublishTestResults@2 - displayName: 'Publish test results' - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-results-*.xml' - testResultsFormat: 'JUnit' - testRunTitle: 'MacOS Python $(PythonVersion)' - - - task: PublishBuildArtifacts@1 - displayName: 'Publish wheel artifact' - inputs: - artifactName: uamqp-macosx$(MacOSXDeploymentTarget)-$(PythonVersion)-whl - pathToPublish: 'dist' - + - job: 'Linux' timeoutInMinutes: 120 @@ -301,61 +90,6 @@ jobs: artifactName: uamqp-linux-$(PythonVersion)-whl pathToPublish: 'dist' - - job: 'SDK_LiveTest_windows2019_37' - timeoutInMinutes: 300 - dependsOn: 'Windows' - pool: - name: 'azsdk-pool-mms-win-2019-general' - vmImage: 'MMS2019' - strategy: - matrix: - EventHub x64 Python 3.7: - PythonVersion: '$(PythonVersion37)' - SDK: 'EventHub' - EventHub x64 Python 3.10: - PythonVersion: '$(PythonVersion310)' - SDK: 'EventHub' - ServiceBus x64 Python 3.7: - PythonVersion: '$(PythonVersion37)' - SDK: 'ServiceBus' - ServiceBus x64 Python 3.10: - PythonVersion: '$(PythonVersion310)' - SDK: 'ServiceBus' - variables: - PythonArchitecture: 'x64' - OSArch: 'Windows' - DownloadArtifactFolder: 'uamqp-win$(PythonArchitecture)-$(PythonVersion)-whl' - steps: - - template: /.azure-pipelines/use-python-version.yml - parameters: - architecture: '$(PythonArchitecture)' - versionSpec: '$(PythonVersion)' - - template: /.azure-pipelines/test-eh-sb-sdk.yml - - - job: 'SDK_LiveTest_macOS1015_36' - timeoutInMinutes: 300 - dependsOn: 'MacOS' - pool: - name: 'Azure Pipelines' - vmImage: 'macOS-10.15' - strategy: - matrix: - EventHub Python 3.6: - SDK: 'EventHub' - ServiceBus Python 3.6: - SDK: 'ServiceBus' - variables: - PythonBin: 'python3' - PythonVersion: '$(PythonVersion36)' - MacOSXDeploymentTarget: '10.9' - PythonVersion368: '3.6.8' - OSArch: 'MacOS' - DownloadArtifactFolder: 'uamqp-macosx$(MacOSXDeploymentTarget)-$(PythonVersion368)-whl' - steps: - - template: /.azure-pipelines/use-python-version.yml - parameters: - versionSpec: '$(PythonVersion)' - - template: /.azure-pipelines/test-eh-sb-sdk.yml - job: 'SDK_LiveTest_ubuntu1804_39' timeoutInMinutes: 300 From 20fdc21fcd15cfe7e5a3d70609dfdddb2406f3b1 Mon Sep 17 00:00:00 2001 From: Kashif Khan Date: Thu, 11 Aug 2022 08:35:50 -0500 Subject: [PATCH 2/4] changes for ipv6 support --- .../adapters/socketio_berkeley.c | 21 +++++++++---------- .../src/dns_resolver_sync.c | 16 +++++++------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/vendor/azure-uamqp-c/deps/azure-c-shared-utility/adapters/socketio_berkeley.c b/src/vendor/azure-uamqp-c/deps/azure-c-shared-utility/adapters/socketio_berkeley.c index 1b2532a4..8231b61a 100755 --- a/src/vendor/azure-uamqp-c/deps/azure-c-shared-utility/adapters/socketio_berkeley.c +++ b/src/vendor/azure-uamqp-c/deps/azure-c-shared-utility/adapters/socketio_berkeley.c @@ -140,7 +140,7 @@ static void* socketio_CloneOption(const char* name, const void* value) } else { - LogError("Cannot clone option %s (not supported)", name); + LogError("Cannot clone option %s (not suppported)", name); } } else @@ -288,7 +288,7 @@ static int initiate_socket_connection(SOCKET_IO_INSTANCE* socket_io_instance) int result; int flags; struct addrinfo* addr = NULL; - struct sockaddr* connect_addr = NULL; + struct sockaddr_in6* connect_addr = NULL; struct sockaddr_un addrInfoUn; socklen_t connect_addr_len; @@ -310,8 +310,8 @@ static int initiate_socket_connection(SOCKET_IO_INSTANCE* socket_io_instance) } else { - connect_addr = addr->ai_addr; - connect_addr_len = sizeof(*addr->ai_addr); + connect_addr = (struct sockaddr_in6*) addr->ai_addr; + connect_addr_len = sizeof(struct sockaddr_in6); result = 0; } } @@ -331,7 +331,7 @@ static int initiate_socket_connection(SOCKET_IO_INSTANCE* socket_io_instance) // No need to add NULL terminator due to the above memset (void)memcpy(addrInfoUn.sun_path, socket_io_instance->hostname, hostname_len); - connect_addr = (struct sockaddr*)&addrInfoUn; + connect_addr = (struct sockaddr_in6*)&addrInfoUn; connect_addr_len = sizeof(addrInfoUn); result = 0; } @@ -347,7 +347,7 @@ static int initiate_socket_connection(SOCKET_IO_INSTANCE* socket_io_instance) } else { - result = connect(socket_io_instance->socket, connect_addr, connect_addr_len); + result = connect(socket_io_instance->socket, (struct sockaddr *) connect_addr, connect_addr_len); if ((result != 0) && (errno != EINPROGRESS)) { LogError("Failure: connect failure %d.", errno); @@ -810,7 +810,7 @@ int socketio_open(CONCRETE_IO_HANDLE socket_io, ON_IO_OPEN_COMPLETE on_io_open_c } else { - socket_io_instance->socket = socket (socket_io_instance->address_type == ADDRESS_TYPE_IP ? AF_INET : AF_UNIX, SOCK_STREAM, 0); + socket_io_instance->socket = socket (socket_io_instance->address_type == ADDRESS_TYPE_IP ? AF_INET6 : AF_UNIX, SOCK_STREAM, 0); if (socket_io_instance->socket < SOCKET_SUCCESS) { LogError("Failure: socket create failure %d.", socket_io_instance->socket); @@ -939,14 +939,14 @@ int socketio_send(CONCRETE_IO_HANDLE socket_io, const void* buffer, size_t size, ssize_t send_result = send(socket_io_instance->socket, buffer, size, MSG_NOSIGNAL); if ((size_t)send_result != size) { - if (send_result == SOCKET_SEND_FAILURE && errno != EAGAIN) + if (send_result == SOCKET_SEND_FAILURE && errno != EAGAIN && errno != ENOBUFS) { LogError("Failure: sending socket failed. errno=%d (%s).", errno, strerror(errno)); result = MU_FAILURE; } else { - /*send says "come back later" with EAGAIN - likely the socket buffer cannot accept more data*/ + /*send says "come back later" with EAGAIN, ENOBUFS - likely the socket buffer cannot accept more data*/ /* queue data */ size_t bytes_sent = (send_result < 0 ? 0 : send_result); @@ -1002,7 +1002,7 @@ void socketio_dowork(CONCRETE_IO_HANDLE socket_io) { if (send_result == INVALID_SOCKET) { - if (errno == EAGAIN) /*send says "come back later" with EAGAIN - likely the socket buffer cannot accept more data*/ + if (errno == EAGAIN || errno == ENOBUFS) /*send says "come back later" with EAGAIN, ENOBUFS - likely the socket buffer cannot accept more data*/ { /*do nothing until next dowork */ break; @@ -1224,4 +1224,3 @@ const IO_INTERFACE_DESCRIPTION* socketio_get_interface_description(void) { return &socket_io_interface_description; } - diff --git a/src/vendor/azure-uamqp-c/deps/azure-c-shared-utility/src/dns_resolver_sync.c b/src/vendor/azure-uamqp-c/deps/azure-c-shared-utility/src/dns_resolver_sync.c index 3ef60194..6703fec4 100644 --- a/src/vendor/azure-uamqp-c/deps/azure-c-shared-utility/src/dns_resolver_sync.c +++ b/src/vendor/azure-uamqp-c/deps/azure-c-shared-utility/src/dns_resolver_sync.c @@ -19,7 +19,7 @@ // EXTRACT_IPV4 pulls the uint32_t IPv4 address out of an addrinfo struct // This will not be needed for the asynchronous design // The default definition handles lwIP. Please add comments for other systems tested. -#define EXTRACT_IPV4(ptr) ((struct sockaddr_in *) ptr->ai_addr)->sin_addr.s_addr +#define EXTRACT_IPV4(ptr) ((struct sockaddr_in6 *) ptr->ai_addr)->sin6_addr.s_addr typedef struct { @@ -102,9 +102,10 @@ bool dns_resolver_is_lookup_complete(DNSRESOLVER_HANDLE dns_in) // Setup the hints address info structure // which is passed to the getaddrinfo() function memset(&hints, 0, sizeof(hints)); - hints.ai_family = AF_INET; + hints.ai_family = AF_INET6; hints.ai_socktype = SOCK_STREAM; - hints.ai_protocol = 0; + // hints.ai_protocol = 0; + // hints.ai_flags = AI_DEFAULT; //-------------------------------- // Call getaddrinfo(). If the call succeeds, @@ -127,14 +128,15 @@ bool dns_resolver_is_lookup_complete(DNSRESOLVER_HANDLE dns_in) { switch (ptr->ai_family) { - case AF_INET: + case AF_INET6: /* Codes_SRS_dns_resolver_30_032: [ If dns_resolver_is_create_complete has returned true and the lookup process has succeeded, dns_resolver_get_ipv4 shall return the discovered IPv4 address. ]*/ - dns->ip_v4 = EXTRACT_IPV4(ptr); + // dns->ip_v4 = EXTRACT_IPV4(ptr); + dns->is_failed = false; break; } } /* Codes_SRS_dns_resolver_30_033: [ If dns_resolver_is_create_complete has returned true and the lookup process has failed, dns_resolver_get_ipv4 shall return 0. ]*/ - dns->is_failed = (dns->ip_v4 == 0); + // dns->is_failed = (dns->ip_v4 == 0); } else { @@ -252,4 +254,4 @@ struct addrinfo* dns_resolver_get_addrInfo(DNSRESOLVER_HANDLE dns_in) } } return result; -} +} \ No newline at end of file From 17840fe10f0dae2a00c9338aed1c71355bc8e41c Mon Sep 17 00:00:00 2001 From: Kashif Khan Date: Thu, 11 Aug 2022 08:41:07 -0500 Subject: [PATCH 3/4] yaml --- .azure-pipelines/client.test.live.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.azure-pipelines/client.test.live.yml b/.azure-pipelines/client.test.live.yml index c6fe3975..af774c95 100644 --- a/.azure-pipelines/client.test.live.yml +++ b/.azure-pipelines/client.test.live.yml @@ -15,8 +15,6 @@ jobs: timeoutInMinutes: 120 - dependsOn: 'MacOS' - pool: vmImage: 'ubuntu-18.04' From ceb11c04d72c13cce9bbed7702c0e939e5953e62 Mon Sep 17 00:00:00 2001 From: Kashif Khan Date: Thu, 11 Aug 2022 10:20:38 -0500 Subject: [PATCH 4/4] whl build always --- .azure-pipelines/client.test.live.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure-pipelines/client.test.live.yml b/.azure-pipelines/client.test.live.yml index af774c95..c09b63b3 100644 --- a/.azure-pipelines/client.test.live.yml +++ b/.azure-pipelines/client.test.live.yml @@ -84,6 +84,7 @@ jobs: - task: PublishBuildArtifacts@1 displayName: 'Publish wheel artifact' + condition: succeededOrFailed() inputs: artifactName: uamqp-linux-$(PythonVersion)-whl pathToPublish: 'dist'