Merge branch 'release/9.x' of github.com:ec-europa/toolkit into relea…

…se/10.x

.drone.yml

@@ -71,7 +71,7 @@ pipeline:
   phpunit:
     group: test
     image: registry.fpfis.eu/fpfis/httpd-php:${PHP_VERSION=8.1}-ci
-    secrets: [ qa_api_basic_auth, qa_api_auth_token, github_api_token, gitlab_api_token, composer_auth, asda_user, asda_password, nextcloud_user, nextcloud_pass ]
+    secrets: [ qa_api_basic_auth, qa_api_auth_token, github_api_token, gitlab_api_token, composer_auth, asda_user, asda_password, nextcloud_user, nextcloud_pass, toolkit_mock_repo ]
     environment: *env
     commands:
       - ./run toolkit:test-phpunit --execution=parallel

.gitignore

@@ -7,7 +7,8 @@
 /runner.yml
 /phpunit.xml
 /docker-compose.*.yml
-.phpunit.result.cache
+/.phpunit.result.cache
+/.phpunit.cache/
 *.sql
 phpda.*
 .phpdoc*
@@ -16,6 +17,7 @@ node_modules/
 package.json
 package-lock.json
 /docs_tmp/
+/mock
 
 # Documentation folder exclusions.
 !docs/

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Toolkit change log
 
+## Version 9.14.0
+- DQA-7830: Switch array_pop to array_shift in Toolkit secure check.
+- DQA-7713: Add support to phpunit/phpunit 10.
+- DQA-7674: Update qa-automation for Drupal 10.
+- DQA-7414: Ignore metapackage packages in component-check.
+
 ## Version 9.13.1 | 10.3.1
 - DQA-7826: Hotfix for components security check.
 

composer.json

@@ -22,7 +22,7 @@
         "consolidation/robo": "^3.0 || ^4.0",
         "cweagans/composer-patches": "^1.4 || ^1.7",
         "drush/drush": "^10.0.0 || ^11.0.4 || ^12.0",
-        "ec-europa/qa-automation": "^9.0",
+        "ec-europa/qa-automation": "^9.1",
         "guzzlehttp/guzzle": "^6.3 || ^7.0",
         "jakeasmith/http_build_url": "^1.0",
         "league/container": "^4.1.1",
@@ -32,7 +32,7 @@
         "phpmd/phpmd": "^2.12",
         "phpstan/phpstan": "^1.10",
         "phpstan/phpstan-deprecation-rules": "^1.0",
-        "phpunit/phpunit": "^9.5",
+        "phpunit/phpunit": "^9.5 || ^10.0",
         "squizlabs/php_codesniffer": "^3.7"
     },
     "suggest": {

config/runner/toolkit.yml

@@ -1,4 +1,5 @@
 toolkit:
+  project_id: toolkit
   tmp_folder: '/cache'
   clean:
     config_file: 'config/sync/core.extension.yml'

docker-compose.yml

@@ -19,6 +19,9 @@ services:
       ASDA_PASSWORD:
       QA_API_BASIC_AUTH:
       QA_API_AUTH_TOKEN:
+      QA_WEBSITE_URL:
+      TOOLKIT_MOCK_REPO:
+      TOOLKIT_MOCK_BRANCH: 0.0.2
       GITHUB_API_TOKEN:
       GITLAB_API_TOKEN:
       XDEBUG_CONFIG: client_host=host.docker.internal

phpunit.xml.dist

@@ -1,16 +1,17 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <phpunit
-    backupGlobals               = "true"
-    backupStaticAttributes      = "false"
-    colors                      = "true"
-    convertErrorsToExceptions   = "true"
-    convertNoticesToExceptions  = "true"
-    convertWarningsToExceptions = "true"
-    processIsolation            = "true"
-    stopOnFailure               = "false"
-    bootstrap                   = "vendor/autoload.php">
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:noNamespaceSchemaLocation="https://schema.phpunit.de/10.3/phpunit.xsd"
+    backupGlobals="true"
+    colors="true"
+    processIsolation="true"
+    stopOnFailure="false"
+    bootstrap="vendor/autoload.php">
     <php>
         <env name="CI" value="true"/>
+        <env name="QA_WEBSITE_URL" value="http://web:8080" force="true"/>
+        <env name="TOOLKIT_MOCK_BRANCH" value="0.0.2"/>
+        <env name="TOOLKIT_DEBUG_EXPECTATIONS" value="false"/>
         <env name="DRUPAL_DATABASE_NAME" value="drupal"/>
         <env name="DRUPAL_DATABASE_USERNAME" value="root"/>
         <env name="DRUPAL_DATABASE_HOST" value="mysql"/>
@@ -19,9 +20,6 @@
         <testsuite name="Unit GitHooks">
             <file>tests/Unit/GitHooksCommandsTest.php</file>
         </testsuite>
-        <testsuite name="Unit Mock">
-            <file>tests/Unit/MockTest.php</file>
-        </testsuite>
         <testsuite name="Unit ReplaceBlock">
             <file>tests/Unit/ReplaceBlockTest.php</file>
         </testsuite>

src/Mock.php

@@ -0,0 +1,68 @@
+<?php
+
+declare(strict_types=1);
+
+namespace EcEuropa\Toolkit;
+
+use Symfony\Component\Process\Process;
+
+/**
+ * Toolkit mock class.
+ */
+final class Mock
+{
+
+    /**
+     * Downloads the mock from the repo.
+     *
+     * @throws \Exception
+     *   If missing env var TOOLKIT_MOCK_REPO.
+     */
+    public static function download(): bool
+    {
+        if (!Toolkit::isCiCd()) {
+            return false;
+        }
+        $mockDir = getenv('TOOLKIT_MOCK_DIR') ?: 'mock';
+        if (file_exists($mockDir)) {
+            return true;
+        }
+        if (empty($repo = getenv('TOOLKIT_MOCK_REPO'))) {
+            throw new \Exception('Missing env var TOOLKIT_MOCK_REPO.');
+        }
+        $branch = getenv('TOOLKIT_MOCK_BRANCH') ?: '0.0.2';
+        $command = "git clone --depth 1 --branch $branch $repo $mockDir";
+        $process = Process::fromShellCommandline($command);
+        $process->run();
+        if ($process->getExitCode()) {
+            throw new \Exception($process->getErrorOutput());
+        }
+        return file_exists($mockDir);
+    }
+
+    /**
+     * Returns the content of the endpoint from the mock.
+     *
+     * @param string $endpoint
+     *   The endpoint to return the content from.
+     *
+     * @throws \Exception
+     *   If the mock or endpoint file is not found.
+     */
+    public static function getEndpointContent(string $endpoint)
+    {
+        if (!Toolkit::isCiCd()) {
+            return false;
+        }
+        $mockDir = getenv('TOOLKIT_MOCK_DIR') ?: 'mock';
+        if (!file_exists($mockDir)) {
+            throw new \Exception("Mock not found at '$mockDir'.");
+        }
+        $endpointFile = "$mockDir/$endpoint.json";
+        if (!file_exists($endpointFile)) {
+            throw new \Exception("No file found for endpoint '$endpoint'.");
+        }
+        return file_get_contents($endpointFile);
+    }
+
+}

src/TaskRunner/Commands/ComponentCheckCommands.php

@@ -57,7 +57,7 @@ public function componentCheck(ConsoleIO $io, array $options = [
         if (!empty($options['endpoint'])) {
             Website::setUrl($options['endpoint']);
         }
-        if (empty($auth = Website::apiAuth())) {
+        if (empty(Website::apiAuth())) {
             return 1;
         }
         $this->io = $io;
@@ -79,9 +79,11 @@ public function componentCheck(ConsoleIO $io, array $options = [
         }
 
         $status = 0;
-        $endpoint = Website::url();
-        $result = Website::get($endpoint . '/api/v1/package-reviews?version=8.x', $auth);
-        $data = json_decode($result, true);
+        $data = Website::packages();
+        if (empty($data)) {
+            $io->error('Failed to connect to the endpoint ' . Website::url() . '/api/v1/package-reviews');
+            return 1;
+        }
         $modules = array_filter(array_combine(array_column($data, 'name'), $data));
 
         // To test this command execute it with the --test-command option:
@@ -250,6 +252,10 @@ protected function printComponentResults(ConsoleIO $io)
      */
     protected function validateComponent(array $package, array $modules)
     {
+        // Ignore if the package is a metapackage.
+        if ($package['type'] === 'metapackage') {
+            return;
+        }
         $config = $this->getConfig();
         $packageName = $package['name'];
         $hasBeenQaEd = isset($modules[$packageName]);

src/TaskRunner/Commands/TestsCommands.php

@@ -169,7 +169,6 @@ public function toolkitTestPhpmd(array $options = [
         }
 
         if (!empty($options['ignore_patterns'])) {
-            Toolkit::filterFolders($options['ignore_patterns']);
             $execOptions['exclude'] = implode(',', $options['ignore_patterns']);
         }
         if (!empty($options['triggered_by'])) {

src/TaskRunner/Commands/ToolCommands.php

@@ -99,16 +99,6 @@ public function optsReview(ConsoleIO $io, array $options = [
             $io->say("The file '.opts.yml' was not found, skipping.");
             return ResultData::EXITCODE_OK;
         }
-        $project_id = $this->getConfig()->get('toolkit.project_id');
-        if (empty($project_id)) {
-            $io->say('The configuration toolkit.project_id value is not valid.');
-            return ResultData::EXITCODE_ERROR;
-        }
-        $forbiddenCommands = Website::projectConstraints($project_id);
-        if (empty($forbiddenCommands)) {
-            $io->error('Failed to get constraints from the endpoint.');
-            return ResultData::EXITCODE_ERROR;
-        }
 
         // Check for invalid php_version value, if given version is 8.0 as float when converted to string will be 8
         // and will cause issues like in docker images.
@@ -127,6 +117,18 @@ public function optsReview(ConsoleIO $io, array $options = [
             $io->say("Your structure for the 'upgrade_commands' is invalid.\nSee the documentation at https://webgate.ec.europa.eu/fpfis/wikis/display/MULTISITE/Pipeline+configuration+and+override");
             return ResultData::EXITCODE_ERROR;
         }
+
+        $project_id = $this->getConfig()->get('toolkit.project_id');
+        if (empty($project_id)) {
+            $io->say('The configuration toolkit.project_id value is not valid.');
+            return ResultData::EXITCODE_ERROR;
+        }
+
+        $forbiddenCommands = Website::projectConstraints($project_id);
+        if (empty($forbiddenCommands)) {
+            $io->error('Failed to get constraints from the endpoint.');
+            return ResultData::EXITCODE_ERROR;
+        }
         // Gather all the commands, ignore the 'ephemeral' commands.
         $commands = [];
         if (!empty($parseOptsFile['upgrade_commands']['append']['acceptance'])) {
@@ -180,7 +182,7 @@ public function toolkitRequirements(ConsoleIO $io, array $options = [
         }
         $data = Website::requirements();
         if (empty($data)) {
-            $this->writeln('Failed to connect to the endpoint. Required env var QA_API_AUTH_TOKEN.');
+            $io->error('Failed to connect to the endpoint ' . Website::url() . '/api/v1/toolkit-requirements');
             return 1;
         }
         if (!isset($data['toolkit'])) {

src/Website.php

@@ -224,6 +224,7 @@ public static function post(array $fields, AuthorizationInterface $auth): string
      */
     public static function projectInformation(string $project_id)
     {
+        $project_id = "$project_id-reference";
         if (!isset($GLOBALS['projects'])) {
             $GLOBALS['projects'] = [];
         }
@@ -233,11 +234,10 @@ public static function projectInformation(string $project_id)
         if (empty($auth = self::apiAuth())) {
             return false;
         }
-        $endpoint = "/api/v1/project/ec-europa/$project_id-reference/information";
-        $response = self::get(self::url() . $endpoint, $auth);
-        $data = json_decode($response, true);
-        $data = reset($data);
-        if (!empty($data['name']) && $data['name'] === "$project_id-reference") {
+        $endpoint = "api/v1/project/ec-europa/$project_id/information";
+        $data = self::getWithMockFallback(self::url() . '/' . $endpoint, $auth);
+        if (!empty($data)) {
+            $data = reset($data);
             $GLOBALS['projects'][$project_id] = $data;
             return $data;
         }
@@ -267,9 +267,9 @@ public static function projectConstraints(string $project_id)
         if (empty($auth = self::apiAuth())) {
             return false;
         }
-        $endpoint = '/api/v1/project/ec-europa/' . $project_id . '-reference/information/constraints';
-        $response = self::get(self::url() . $endpoint, $auth);
-        $data = json_decode($response, true);
+        $project_id = "$project_id-reference";
+        $endpoint = "api/v1/project/ec-europa/$project_id/information/constraints";
+        $data = self::getWithMockFallback(self::url() . '/' . $endpoint, $auth);
         if (empty($data) || !isset($data['constraints'])) {
             return false;
         }
@@ -294,13 +294,58 @@ public static function requirements()
         if (empty($auth = self::apiAuth())) {
             return false;
         }
-        $response = self::get(self::url() . '/api/v1/toolkit-requirements', $auth);
+        $data = self::getWithMockFallback(self::url() . '/api/v1/toolkit-requirements', $auth);
+        $GLOBALS['requirements'] = $data;
+        return $data;
+    }
+
+    /**
+     * Returns the packages reviews from the endpoint.
+     */
+    public static function packages()
+    {
+        if (empty($auth = self::apiAuth())) {
+            return false;
+        }
+        $data = self::getWithMockFallback(self::url() . '/api/v1/package-reviews?version=8.x', $auth);
+        return empty($data) ? false : $data;
+    }
+
+    /**
+     * Returns content from given endpoint and fallback to mock if possible.
+     *
+     * This should only be executed on CI.
+     *
+     * @param string $url
+     *    The QA endpoint url.
+     * @param AuthorizationInterface|null $auth
+     *    The authorization instance or null.
+     */
+    public static function getWithMockFallback(string $url, AuthorizationInterface $auth = null)
+    {
+        try {
+            $response = self::get($url, $auth);
+        } catch (\Exception) {
+            $response = '';
+        }
+
+        // If the request fails, try the mock if we are on CI.
+        if (empty($response) && Mock::download()) {
+            // Remove the base url from the endpoint.
+            $endpoint = str_replace(self::url() . '/', '', $url);
+            // Replace any project_id in the url.
+            $endpoint = preg_replace('#/[a-zA-Z\-]+-reference/#', '/toolkit/', $endpoint);
+            // Remove any query string from the url.
+            if (str_contains($endpoint, '?')) {
+                $endpoint = strstr($endpoint, '?', true);
+            }
+            $response = Mock::getEndpointContent($endpoint);
+        }
         if (empty($response)) {
             return false;
         }
-        $data = json_decode($response, true);
-        $GLOBALS['requirements'] = $data;
-        return $data;
+
+        return json_decode($response, true);
     }
 
 }