Bug 571722: Add ALL_FLAGS for scanner discovery

This change adds the ALL_FLAGS that does not limit tool options to
those declared as IOption::isForScannerDiscovery when launching the
compiler to discover compiler built-ins.

This is needed as many other flags, either entered manually in "Other
flags" or some of the existing flags with checkboxes such as "-ansi",
"-fPIC", and "-fstack-protector-all" which all affect scanner discovery
as they can all change what macros are built-in to the compiler.

The current solution has as a drawback that some settings, like -I and -D
then appear twice. For example in the "Includes" node in the "Project
Explorer"

My only reservation about this change is if there is an option
that can be specified successfully at build time, but when used
at scanner discovery time causes the compiler to fail, or return
incorrect results. Therefore I have added a new field,
excludeFromScannerDiscovery to tool options (buildDefinitions
extension point) that allows tool integrators to always exclude
a command line option from ALL_FLAGS. I have also added
a new "Other flags (excluded from discovery)" to the
"Miscellaneous" tab to allow compiler options to be entered
by the user.

TODO:

- [ ] N&N entry
- [ ] Provide PR to embed-cdt

build/org.eclipse.cdt.managedbuilder.core.tests/plugin.xml

@@ -9601,6 +9601,21 @@
                   resourceFilter="all"
                   valueType="string">
             </option>
+            <option
+                  command="-not-excluded-from-scanner-discovery"
+                  id="cdt.managedbuilder.lsp.tests.option.not-excluded-sd"
+                  isAbstract="false"
+                  resourceFilter="all"
+                  valueType="string">
+            </option>
+            <option
+                  command="-is-excluded-from-scanner-discovery"
+                  id="cdt.managedbuilder.lsp.tests.option.is-excluded-sd"
+                  isAbstract="false"
+                  resourceFilter="all"
+                  excludeFromScannerDiscovery="true"
+                  valueType="string">
+            </option>
             <option
                   command="-bool-option"
                   defaultValue="true"

build/org.eclipse.cdt.managedbuilder.core.tests/tests/org/eclipse/cdt/managedbuilder/core/tests/ManagedBuildCoreTests.java

@@ -34,9 +34,6 @@
 import org.eclipse.core.runtime.IConfigurationElement;
 import org.junit.Assert;
 
-import junit.framework.Test;
-import junit.framework.TestSuite;
-
 public class ManagedBuildCoreTests extends BaseTestCase {
 	private static IProjectType exeType;
 	private static IProjectType libType;
@@ -46,14 +43,6 @@ public ManagedBuildCoreTests(String name) {
 		super(name);
 	}
 
-	public static Test suite() {
-		TestSuite suite = new TestSuite(ManagedBuildCoreTests.class.getName());
-		suite.addTest(new ManagedBuildCoreTests("testLoadManifest"));
-		suite.addTest(new ManagedBuildCoreTests("testTreeOptions"));
-		suite.addTest(new ManagedBuildCoreTests("testOptionsAttributeUseByScannerDiscovery"));
-		return suite;
-	}
-
 	/**
 	 * Navigates through a CDT 2.1 manifest file and verifies that the
 	 * definitions are loaded correctly.
@@ -673,4 +662,19 @@ public void testOptionsAttributeUseByScannerDiscovery() throws Exception {
 		assertNotNull(option);
 		assertEquals(true, option.isForScannerDiscovery());
 	}
+
+	/**
+	 * Tests attribute excludedFromScannerDiscovery.
+	 * @throws Exception
+	 */
+	public void testExcludedFromScannerDiscovery() throws Exception {
+		IOption optionNotExcludedFromSD = ManagedBuildManager
+				.getExtensionOption("cdt.managedbuilder.lsp.tests.option.not-excluded-sd");
+		assertNotNull(optionNotExcludedFromSD);
+		assertEquals(false, optionNotExcludedFromSD.isExcludedFromScannerDiscovery());
+
+		IOption option = ManagedBuildManager.getExtensionOption("cdt.managedbuilder.lsp.tests.option.is-excluded-sd");
+		assertNotNull(option);
+		assertEquals(true, option.isExcludedFromScannerDiscovery());
+	}
 } // end class

build/org.eclipse.cdt.managedbuilder.core/plugin.xml

@@ -602,7 +602,7 @@
             class="org.eclipse.cdt.managedbuilder.language.settings.providers.GCCBuiltinSpecsDetector"
             id="org.eclipse.cdt.managedbuilder.core.GCCBuiltinSpecsDetector"
             name="%GCCBuiltinCompilerSettings.name"
-            parameter="${COMMAND} ${FLAGS} -E -P -v -dD "${INPUTS}""
+            parameter="${COMMAND} ${ALL_FLAGS} -E -P -v -dD "${INPUTS}""
             prefer-non-shared="true">
          <language-scope id="org.eclipse.cdt.core.gcc"/>
          <language-scope id="org.eclipse.cdt.core.g++"/>
@@ -611,7 +611,7 @@
             class="org.eclipse.cdt.managedbuilder.internal.language.settings.providers.GCCBuiltinSpecsDetectorCygwin"
             id="org.eclipse.cdt.managedbuilder.core.GCCBuiltinSpecsDetectorCygwin"
             name="%GCCBuiltinCompilerSettingsCygwin.name"
-            parameter="${COMMAND} ${FLAGS} -E -P -v -dD &quot;${INPUTS}&quot;"
+            parameter="${COMMAND} ${ALL_FLAGS} -E -P -v -dD &quot;${INPUTS}&quot;"
             prefer-non-shared="true">
          <language-scope id="org.eclipse.cdt.core.gcc"/>
          <language-scope id="org.eclipse.cdt.core.g++"/>
@@ -620,7 +620,7 @@
             class="org.eclipse.cdt.managedbuilder.internal.language.settings.providers.GCCBuiltinSpecsDetectorMinGW"
             id="org.eclipse.cdt.managedbuilder.core.GCCBuiltinSpecsDetectorMinGW"
             name="%GCCBuiltinCompilerSettingsMinGW.name"
-            parameter="${COMMAND} ${FLAGS} -E -P -v -dD &quot;${INPUTS}&quot;"
+            parameter="${COMMAND} ${ALL_FLAGS} -E -P -v -dD &quot;${INPUTS}&quot;"
             prefer-non-shared="true">
          <language-scope id="org.eclipse.cdt.core.gcc"/>
          <language-scope id="org.eclipse.cdt.core.g++"/>

build/org.eclipse.cdt.managedbuilder.core/schema/buildDefinitions.exsd

@@ -707,7 +707,7 @@ The pathConverter of a toolchain applies for all tools of the toolchain except i
          <attribute name="commandLinePattern" type="string">
             <annotation>
                <documentation>
-                  Specifies the command &quot;pattern&quot; that indicates how the parts of the command line are used to create the entire command line.  The pattern consists of the replaceable variables COMMAND, FLAGS, OUTPUT_FLAG, OUTPUT_PREFIX, OUTPUT and INPUTS.  The default command line pattern is ${COMMAND} ${FLAGS} ${OUTPUT_FLAG}${OUTPUT_PREFIX}${OUTPUT} ${INPUTS}, except when customBuildStep is true, where the default is $(COMMAND). White space and other characters are significant and are copied to the generated command.  This attribute supports MBS file context macros.
+                  Specifies the command &quot;pattern&quot; that indicates how the parts of the command line are used to create the entire command line.  The pattern consists of the replaceable variables COMMAND, FLAGS, OUTPUT_FLAG, OUTPUT_PREFIX, OUTPUT and INPUTS.  The default command line pattern is ${COMMAND} ${ALL_FLAGS} ${OUTPUT_FLAG}${OUTPUT_PREFIX}${OUTPUT} ${INPUTS}, except when customBuildStep is true, where the default is $(COMMAND). White space and other characters are significant and are copied to the generated command.  This attribute supports MBS file context macros.
                </documentation>
             </annotation>
          </attribute>
@@ -1410,7 +1410,14 @@ Additional special types exist to flag options of special relevance to the build
          <attribute name="useByScannerDiscovery" type="boolean">
             <annotation>
                <documentation>
-                  An optional field to allow the option additionally to be passed to scanner discovery of built-in compiler macros and paths. The default is false.
+                  An optional field to allow the option additionally to be passed to scanner discovery of built-in compiler macros and paths in the ${FLAGS} variable. The default is false.
+               </documentation>
+            </annotation>
+         </attribute>
+         <attribute name="excludeFromScannerDiscovery" type="boolean">
+            <annotation>
+               <documentation>
+                  An optional field to allow the option to be excluded from scanner discovery's ${ALL_FLAGS} variable. The default is false.
                </documentation>
             </annotation>
          </attribute>

build/org.eclipse.cdt.managedbuilder.core/src/org/eclipse/cdt/managedbuilder/core/IOption.java

@@ -106,6 +106,8 @@ public interface IOption extends IBuildObject {
 	public static final String COMMAND_FALSE = "commandFalse"; //$NON-NLS-1$
 	/** @since 8.3 */
 	public static final String USE_BY_SCANNER_DISCOVERY = "useByScannerDiscovery"; //$NON-NLS-1$
+	/** @since 9.5 */
+	public static final String EXCLUDE_FROM_SCANNER_DISCOVERY = "excludeFromScannerDiscovery"; //$NON-NLS-1$
 	/** @since 8.0 */
 	public static final String COMMAND_GENERATOR = "commandGenerator"; //$NON-NLS-1$
 	public static final String TOOL_TIP = "tip"; //$NON-NLS-1$
@@ -630,6 +632,15 @@ public interface IOption extends IBuildObject {
 	 */
 	public boolean isForScannerDiscovery();
 
+	/**
+	 * Flag to indicate whether the option should be excluded from scanner discovery.
+	 * @return {@code true} if the option should never be passed to scanner discovery command
+	 *    or {@code false} otherwise.
+	 *
+	 * @since 9.5
+	 */
+	public boolean isExcludedFromScannerDiscovery();
+
 	/**
 	 * Returns the tree root of this option if it is of type {@link #TREE}
 	 * @return tree root of this option or <code>null</code> if not found.

build/org.eclipse.cdt.managedbuilder.core/src/org/eclipse/cdt/managedbuilder/internal/core/Option.java

@@ -81,6 +81,7 @@ public class Option extends BuildObject implements IOption, IBuildPropertiesRest
 	private IOptionCommandGenerator commandGenerator;
 	private String commandFalse;
 	private Boolean isForScannerDiscovery;
+	private Boolean isExcludedFromScannerDiscovery;
 	private String tip;
 	private String contextId;
 	private List<String> applicableValuesList;
@@ -224,6 +225,9 @@ public Option(IHoldsOptions parent, String Id, String name, Option option) {
 		if (option.isForScannerDiscovery != null) {
 			isForScannerDiscovery = option.isForScannerDiscovery;
 		}
+		if (option.isExcludedFromScannerDiscovery != null) {
+			isExcludedFromScannerDiscovery = option.isExcludedFromScannerDiscovery;
+		}
 		if (option.tip != null) {
 			tip = option.tip;
 		}
@@ -405,6 +409,12 @@ protected void loadFromManifest(IManagedConfigElement element) {
 			isForScannerDiscovery = Boolean.parseBoolean(isForSD);
 		}
 
+		// isNotForScannerDiscovery
+		String isExcludeFromSD = element.getAttribute(EXCLUDE_FROM_SCANNER_DISCOVERY);
+		if (isExcludeFromSD != null) {
+			isExcludedFromScannerDiscovery = Boolean.parseBoolean(isExcludeFromSD);
+		}
+
 		// Get the tooltip for the option
 		tip = SafeStringInterner.safeIntern(element.getAttribute(TOOL_TIP));
 
@@ -578,6 +588,14 @@ protected void loadFromProject(ICStorageElement element) {
 			}
 		}
 
+		// isNotForScannerDiscovery
+		if (element.getAttribute(EXCLUDE_FROM_SCANNER_DISCOVERY) != null) {
+			String isExcludeFromSD = element.getAttribute(EXCLUDE_FROM_SCANNER_DISCOVERY);
+			if (isExcludeFromSD != null) {
+				isForScannerDiscovery = Boolean.parseBoolean(isExcludeFromSD);
+			}
+		}
+
 		// Get the tooltip for the option
 		if (element.getAttribute(TOOL_TIP) != null) {
 			tip = SafeStringInterner.safeIntern(element.getAttribute(TOOL_TIP));
@@ -887,6 +905,10 @@ public void serialize(ICStorageElement element) throws BuildException {
 			element.setAttribute(USE_BY_SCANNER_DISCOVERY, isForScannerDiscovery.toString());
 		}
 
+		if (isExcludedFromScannerDiscovery != null) {
+			element.setAttribute(EXCLUDE_FROM_SCANNER_DISCOVERY, isExcludedFromScannerDiscovery.toString());
+		}
+
 		if (tip != null) {
 			element.setAttribute(TOOL_TIP, tip);
 		}
@@ -1381,6 +1403,14 @@ public boolean isForScannerDiscovery() {
 		return isForScannerDiscovery;
 	}
 
+	@Override
+	public boolean isExcludedFromScannerDiscovery() {
+		if (isExcludedFromScannerDiscovery == null) {
+			isExcludedFromScannerDiscovery = superClass != null && superClass.isExcludedFromScannerDiscovery();
+		}
+		return isExcludedFromScannerDiscovery;
+	}
+
 	@Override
 	public String getToolTip() {
 		if (tip == null) {

build/org.eclipse.cdt.managedbuilder.core/src/org/eclipse/cdt/managedbuilder/internal/core/OptionReference.java

@@ -926,6 +926,11 @@ public boolean isForScannerDiscovery() {
 		return option.isForScannerDiscovery();
 	}
 
+	@Override
+	public boolean isExcludedFromScannerDiscovery() {
+		return option.isExcludedFromScannerDiscovery();
+	}
+
 	@Override
 	public ITreeRoot getTreeRoot() {
 		if (!resolved) {

build/org.eclipse.cdt.managedbuilder.core/src/org/eclipse/cdt/managedbuilder/internal/core/Tool.java

@@ -105,7 +105,7 @@
 public class Tool extends HoldsOptions
 		implements ITool, IOptionCategory, IMatchKeyProvider<Tool>, IRealBuildObjectAssociation {
 
-	public static final String DEFAULT_PATTERN = "${COMMAND} ${FLAGS} ${OUTPUT_FLAG} ${OUTPUT_PREFIX}${OUTPUT} ${INPUTS} ${EXTRA_FLAGS}"; //$NON-NLS-1$
+	public static final String DEFAULT_PATTERN = "${COMMAND} ${ALL_FLAGS} ${OUTPUT_FLAG} ${OUTPUT_PREFIX}${OUTPUT} ${INPUTS} ${EXTRA_FLAGS}"; //$NON-NLS-1$
 	public static final String DEFAULT_CBS_PATTERN = "${COMMAND}"; //$NON-NLS-1$
 
 	//property name for holding the rebuild state

build/org.eclipse.cdt.managedbuilder.core/src/org/eclipse/cdt/managedbuilder/language/settings/providers/AbstractBuiltinSpecsDetector.java

@@ -53,6 +53,7 @@
 import org.eclipse.cdt.internal.core.BuildRunnerHelper;
 import org.eclipse.cdt.internal.core.XmlUtil;
 import org.eclipse.cdt.internal.core.envvar.EnvironmentVariableManager;
+import org.eclipse.cdt.managedbuilder.core.IOption;
 import org.eclipse.cdt.managedbuilder.core.ManagedBuilderCorePlugin;
 import org.eclipse.cdt.managedbuilder.internal.core.ManagedMakeMessages;
 import org.eclipse.cdt.utils.CommandLineUtil;
@@ -101,6 +102,8 @@ public abstract class AbstractBuiltinSpecsDetector extends AbstractLanguageSetti
 	protected static final String COMPILER_MACRO = "${COMMAND}"; //$NON-NLS-1$
 	/** @since 8.3 */
 	protected static final String FLAGS_MACRO = "${FLAGS}"; //$NON-NLS-1$
+	/**	@since 9.5*/
+	protected static final String ALL_FLAGS_MACRO = "${ALL_FLAGS}"; //$NON-NLS-1$
 	protected static final String SPEC_FILE_MACRO = "${INPUTS}"; //$NON-NLS-1$
 	protected static final String SPEC_EXT_MACRO = "${EXT}"; //$NON-NLS-1$
 	protected static final String SPEC_FILE_BASE = "spec"; //$NON-NLS-1$
@@ -279,6 +282,7 @@ public void shutdown() {
 	 *    Normally would come from the tool-chain.<br>
 	 * <b>${INPUTS}</b> - path to spec file which will be placed in workspace area.<br>
 	 * <b>${EXT}</b> - file extension calculated from language ID.
+	 * <b>${ALL_FLAGS}</b> - all the command line flags.
 	 * </ul>
 	 * The parameter could be taken from the extension
 	 * in {@code plugin.xml} or from property file.
@@ -334,6 +338,11 @@ protected String resolveCommand(String languageId) throws CoreException {
 				if (flags != null)
 					cmd = cmd.replace(FLAGS_MACRO, flags);
 			}
+			if (cmd.contains(ALL_FLAGS_MACRO)) {
+				String flags = getAllToolOptions(languageId);
+				if (flags != null)
+					cmd = cmd.replace(ALL_FLAGS_MACRO, flags);
+			}
 			if (cmd.contains(SPEC_FILE_MACRO)) {
 				String specFileName = getSpecFile(languageId);
 				if (specFileName != null)
@@ -933,6 +942,9 @@ protected Optional<String> selectBestSpecFileExtension(List<String> extensions)
 	 * Determine additional options to pass to scanner discovery command.
 	 * These options are intended to come from the tool-chain.
 	 *
+	 * Unlike {@link #getAllToolOptions(String)} this method filters to only
+	 * return a subset of all options. See {@link IOption#isForScannerDiscovery()}
+	 *
 	 * @param languageId - language ID.
 	 * @return additional options to pass to scanner discovery command.
 	 *
@@ -942,6 +954,23 @@ protected String getToolOptions(String languageId) {
 		return ""; //$NON-NLS-1$
 	}
 
+	/**
+	 * Determine additional options to pass to scanner discovery command.
+	 * These options are intended to come from the tool-chain.
+	 *
+	 * Unlike {@link #getToolOptions(String)} this returns all options,
+	 * except for those specifically excluded from scanner discovery
+	 * with {@link IOption#isExcludedFromScannerDiscovery()}
+	 *
+	 * @param languageId - language ID.
+	 * @return additional options to pass to scanner discovery command.
+	 *
+	 * @since 9.5
+	 */
+	protected String getAllToolOptions(String languageId) {
+		return ""; //$NON-NLS-1$
+	}
+
 	@Override
 	public Element serializeAttributes(Element parentElement) {
 		Element elementProvider = super.serializeAttributes(parentElement);

build/org.eclipse.cdt.managedbuilder.core/src/org/eclipse/cdt/managedbuilder/language/settings/providers/ToolchainBuiltinSpecsDetector.java

@@ -21,6 +21,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.function.Predicate;
 
 import org.eclipse.cdt.core.envvar.IEnvironmentVariable;
 import org.eclipse.cdt.managedbuilder.core.BuildException;
@@ -153,16 +154,18 @@ protected String getSpecFileExtension(String languageId) {
 		return extension.get();
 	}
 
-	@Override
-	protected String getToolOptions(String languageId) {
+	/**
+	 * @since 9.5
+	 */
+	protected String getToolOptions(String languageId, Predicate<IOption> predicate) {
 		Optional<IOption[]> found = languageTool(languageId).flatMap(t -> Optional.of(t.getOptions()));
 		if (!found.isPresent()) {
 			return ""; //$NON-NLS-1$
 		}
 		StringBuilder flags = new StringBuilder();
 		IOption[] options = found.get();
 		for (IOption option : options) {
-			if (option.isForScannerDiscovery()) {
+			if (predicate.test(option)) {
 				try {
 					String optionValue = null;
 					switch (option.getBasicValueType()) {
@@ -207,6 +210,16 @@ protected String getToolOptions(String languageId) {
 		return flags.toString().trim();
 	}
 
+	@Override
+	protected String getToolOptions(String languageId) {
+		return getToolOptions(languageId, IOption::isForScannerDiscovery);
+	}
+
+	@Override
+	protected String getAllToolOptions(String languageId) {
+		return getToolOptions(languageId, Predicate.not(IOption::isExcludedFromScannerDiscovery));
+	}
+
 	@Override
 	protected List<IEnvironmentVariable> getEnvironmentVariables() {
 		if (envMngr == null && currentCfgDescription == null) {

build/org.eclipse.cdt.managedbuilder.gnu.ui/plugin.properties

@@ -188,6 +188,8 @@ Option.Posix.Warn.wwritestring=Treat strings always as const (-Wwrite-strings)
 
 Option.Posix.Verbose=Verbose (-v)
 Option.OtherFlags=Other flags
+Option.OtherFlagsExcludedFromScannerDiscovery=Other flags (excluded from discovery)
+Option.OtherFlagsExcludedFromScannerDiscoveryTip=Flags that will not be passed to the compiler when discovering compiler built-in macros and include paths. Include compiler flags here that may interfere with the correct operation of scanner discovery.
 Option.Posix.Ansi=Support ANSI programs (-ansi)
 Option.PIC=Position Independent Code (-fPIC)
 Option.codecov=Generate gcov information (-ftest-coverage -fprofile-arcs)

build/org.eclipse.cdt.managedbuilder.gnu.ui/plugin.xml

@@ -1440,6 +1440,15 @@
 			  id="gnu.c.compiler.option.misc.other"
 			  valueType="string">
 		  </option>
+		  <option
+			  defaultValue=""
+			  name="%Option.OtherFlagsExcludedFromScannerDiscovery"
+			  tip="%Option.OtherFlagsExcludedFromScannerDiscoveryTip"
+			  category="gnu.c.compiler.category.other"
+			  id="gnu.c.compiler.option.misc.otherExcludedFromScannerDiscovery"
+			  excludeFromScannerDiscovery="true"
+			  valueType="string">
+		  </option>
 		  <option
 			  defaultValue="false"
 			  name="%Option.Posix.Verbose"
@@ -2036,6 +2045,15 @@
 			  id="gnu.cpp.compiler.option.other.other"
 			  valueType="string">
 		  </option>
+		  <option
+			  defaultValue=""
+			  name="%Option.OtherFlagsExcludedFromScannerDiscovery"
+			  tip="%Option.OtherFlagsExcludedFromScannerDiscoveryTip"
+			  category="gnu.cpp.compiler.category.other"
+			  id="gnu.cpp.compiler.option.other.otherExcludedFromScannerDiscovery"
+			  excludeFromScannerDiscovery="true"
+			  valueType="string">
+		  </option>
 		  <option
 			  defaultValue="false"
 			  name="%Option.Posix.Verbose"