Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use "-i" to prevent rdmd from having to invoke compiler twice. #271

Merged
merged 1 commit into from
Jan 16, 2018
Merged

Use "-i" to prevent rdmd from having to invoke compiler twice. #271

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
287 changes: 140 additions & 147 deletions rdmd.d
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ else
private bool chatty, buildOnly, dryRun, force, preserveOutputPaths;
private string exe, userTempDir;
immutable string[] defaultExclusions = ["std", "etc", "core"];
private string[] exclusions = defaultExclusions; // packages that are to be excluded
private string[] exclusions; // packages that are to be excluded
private string[] includes;
private string[] extraFiles = [];

version (DigitalMars)
Expand Down Expand Up @@ -139,7 +140,7 @@ int main(string[] args)
"eval", &eval,
"loop", &loop,
"exclude", &exclusions,
"include", (string opt, string p) { exclusions = exclusions.filter!(ex => ex != p).array(); },
"include", &includes,
"extra-file", &extraFiles,
"force", &force,
"help", { writeln(helpString); bailout = true; },
Expand Down Expand Up @@ -250,23 +251,8 @@ int main(string[] args)
}

// Fetch dependencies
const myDeps = getDependencies(root, workDir, objDir, compilerFlags);

// --makedepend mode. Just print dependencies and exit.
if (makeDepend)
{
writeDeps(exe, root, myDeps, stdout);
return 0;
}

// --makedepfile mode. Print dependencies to a file and continue.
// This is similar to GCC's -MF option, very useful to update the
// dependencies file and compile in one go:
// -include .deps.mak
// prog:
// rdmd --makedepfile=.deps.mak --build-only prog.d
if (makeDepFile !is null)
writeDeps(exe, root, myDeps, File(makeDepFile, "w"));
const depsFilename = buildPath(workDir, "rdmd.deps");
const initialDeps = tryGetDependencies(root, objDir, compilerFlags, depsFilename);

// Compute executable name, check for freshness, rebuild
/*
Expand All @@ -287,7 +273,7 @@ int main(string[] args)
{
// user-specified exe name
buildWitness = buildPath(workDir, ".built");
if (!exe.newerThan(buildWitness))
if (initialDeps !is null && !exe.newerThan(buildWitness))
{
// Both exe and buildWitness exist, and exe is older than
// buildWitness. This is the only situation in which we
Expand All @@ -300,18 +286,22 @@ int main(string[] args)
{
exe = buildPath(workDir, exeBasename) ~ outExt;
buildWitness = exe;
yap("stat ", buildWitness);
lastBuildTime = buildWitness.timeLastModified(SysTime.min);
if(initialDeps !is null)
{
yap("stat ", buildWitness);
lastBuildTime = buildWitness.timeLastModified(SysTime.min);
}
}

// Have at it
if (chain(root.only, myDeps.byKey).anyNewerThan(lastBuildTime))
if (initialDeps is null || chain(root.only, initialDeps.byKey).anyNewerThan(lastBuildTime))
{
immutable result = rebuild(root, exe, workDir, objDir,
myDeps, compilerFlags, addStubMain);
compilerFlags, depsFilename, addStubMain);
if (result)
return result;

// TODO: only touch build witness if build actually created output
// Touch the build witness to track the build time
if (buildWitness != exe)
{
Expand All @@ -321,6 +311,28 @@ int main(string[] args)
}
}

if(makeDepend || makeDepFile !is null)
{
auto resolvedDeps = (initialDeps is null) ? readDepsFile(objDir, depsFilename) : initialDeps;
assert(resolvedDeps !is null, "code bug: dependencies were not properly generated from the build");

// --makedepend mode. Just print dependencies and exit.
if (makeDepend)
{
writeDeps(exe, root, resolvedDeps, stdout);
return true; // exit program
}

// --makedepfile mode. Print dependencies to a file and continue.
// This is similar to GCC's -MF option, very useful to update the
// dependencies file and compile in one go:
// -include .deps.mak
// prog:
// rdmd --makedepfile=.deps.mak --build-only prog.d
if (makeDepFile !is null)
writeDeps(exe, root, resolvedDeps, File(makeDepFile, "w"));
}

if (buildOnly)
{
// Pretty much done!
Expand Down Expand Up @@ -459,8 +471,8 @@ private void unlockWorkPath()
// object file.

private int rebuild(string root, string fullExe,
string workDir, string objDir, in string[string] myDeps,
string[] compilerFlags, bool addStubMain)
string workDir, string objDir,
string[] compilerFlags, string depsFilename, bool addStubMain)
{
version (Windows)
fullExe = fullExe.defaultExtension(".exe");
Expand Down Expand Up @@ -495,10 +507,20 @@ private int rebuild(string root, string fullExe,
~ [ "-of" ~ fullExeTemp ]
~ [ "-od" ~ objDir ]
~ [ "-I" ~ dirName(root) ]
~ [ "-i" ]
~ [ "-v" ]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assume this was just for development / testing?

Copy link
Contributor Author

@marler8997 marler8997 Jan 9, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually it's necessary. Before this change the compiler was called using -v for the "first call" to the compiler, now both calls are combined into a single call. rdmd still uses the output from -v to read the dependencies to know when a rebuild is necessary.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah fair enough. Thanks for the explanation!

~ [ root ];
foreach (k, objectFile; myDeps) {
if(objectFile !is null)
todo ~= [ k ];
foreach(exclusion; exclusions)
{
todo ~= ("-i=-" ~ exclusion);
}
foreach(inclusion; includes)
{
todo ~= ("-i=" ~ inclusion);
}
foreach(extraFile; extraFiles)
{
todo ~= extraFile;
}
// Need to add void main(){}?
if (addStubMain)
Expand Down Expand Up @@ -526,7 +548,7 @@ private int rebuild(string root, string fullExe,
todo = [ "@" ~ rspName ];
}

immutable result = run([ compiler ] ~ todo);
immutable result = run([ compiler ] ~ todo, depsFilename);
if (result)
{
// build failed
Expand Down Expand Up @@ -588,106 +610,104 @@ private int exec(string[] args)
return run(args, null, true);
}

// Given module rootModule, returns a mapping of all dependees .d
// source filenames to their corresponding .o files sitting in
// directory workDir. The mapping is obtained by running dmd -v against
// rootModule.

private string[string] getDependencies(string rootModule, string workDir,
string objDir, string[] compilerFlags)
const(string[string]) readDepsFile(string objDir, string depsFilename)
{
immutable depsFilename = buildPath(workDir, "rdmd.deps");

string[string] readDepsFile()
string d2obj(string dfile)
{
return buildPath(objDir, dfile.baseName.chomp(".d") ~ objExt);
}
string findLib(string libName)
{
string d2obj(string dfile)
// This can't be 100% precise without knowing exactly where the linker
// will look for libraries (which requires, but is not limited to,
// parsing the linker's command line (as specified in dmd.conf/sc.ini).
// Go for best-effort instead.
string[] dirs = ["."];
foreach (envVar; ["LIB", "LIBRARY_PATH", "LD_LIBRARY_PATH"])
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

DYLD_LIBRARY_PATH on OSX

would be nice to abstract out OS specific things eg:

foreach (envVar; ["LIB", "LIBRARY_PATH", os_LD_LIBRARY_PATH]) {...}

module rdmd.util;
string os_LD_LIBRARY_PATH(){ // find a better name
version(OSX) return "DYLD_LIBRARY_PATH";
else version(linux) return "LD_LIBRARY_PATH";
else assert(0);
}

dirs ~= environment.get(envVar, "").split(pathSeparator);
version (Windows)
string[] names = [libName ~ ".lib"];
else
{
return buildPath(objDir, dfile.baseName.chomp(".d") ~ objExt);
string[] names = ["lib" ~ libName ~ ".a", "lib" ~ libName ~ ".so"];
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

.dylib on OSX
same comment as above:

string[] names = ["lib" ~ libName ~ ".a", "lib" ~ libName ~ os_so_ext];

dirs ~= ["/lib", "/usr/lib"];
}
string findLib(string libName)
{
// This can't be 100% precise without knowing exactly where the linker
// will look for libraries (which requires, but is not limited to,
// parsing the linker's command line (as specified in dmd.conf/sc.ini).
// Go for best-effort instead.
string[] dirs = ["."];
foreach (envVar; ["LIB", "LIBRARY_PATH", "LD_LIBRARY_PATH"])
dirs ~= environment.get(envVar, "").split(pathSeparator);
version (Windows)
string[] names = [libName ~ ".lib"];
else
foreach (dir; dirs)
foreach (name; names)
{
string[] names = ["lib" ~ libName ~ ".a", "lib" ~ libName ~ ".so"];
dirs ~= ["/lib", "/usr/lib"];
auto path = buildPath(dir, name);
if (path.exists)
return absolutePath(path);
}
foreach (dir; dirs)
foreach (name; names)
{
auto path = buildPath(dir, name);
if (path.exists)
return absolutePath(path);
}
return null;
}
yap("read ", depsFilename);
auto depsReader = File(depsFilename);
scope(exit) collectException(depsReader.close()); // don't care for errors

// Fetch all dependencies and append them to myDeps
auto pattern = ctRegex!(r"^(import|file|binary|config|library)\s+([^\(]+)\(?([^\)]*)\)?\s*$");
string[string] result;
foreach (string line; lines(depsReader))
return null;
}
yap("read ", depsFilename);
auto depsReader = File(depsFilename);
scope(exit) collectException(depsReader.close()); // don't care for errors

// Fetch all dependencies and append them to myDeps
auto pattern = ctRegex!(r"^(import|file|binary|config|library)\s+([^\(]+)\(?([^\)]*)\)?\s*$");
string[string] result;
foreach (string line; lines(depsReader))
{
auto regexMatch = match(line, pattern);
if (regexMatch.empty) continue;
auto captures = regexMatch.captures;
switch(captures[1])
{
auto regexMatch = match(line, pattern);
if (regexMatch.empty) continue;
auto captures = regexMatch.captures;
switch(captures[1])
case "import":
immutable moduleName = captures[2].strip(), moduleSrc = captures[3].strip();
if (inALibrary(moduleName, moduleSrc)) continue;
immutable moduleObj = d2obj(moduleSrc);
result[moduleSrc] = moduleObj;
break;

case "file":
result[captures[3].strip()] = null;
break;

case "binary":
result[which(captures[2].strip())] = null;
break;

case "config":
auto confFile = captures[2].strip;
// The config file is special: if missing, that's fine too. So
// add it as a dependency only if it actually exists.
yap("stat ", confFile);
if (confFile.exists)
{
case "import":
immutable moduleName = captures[2].strip(), moduleSrc = captures[3].strip();
if (inALibrary(moduleName, moduleSrc)) continue;
immutable moduleObj = d2obj(moduleSrc);
result[moduleSrc] = moduleObj;
break;

case "file":
result[captures[3].strip()] = null;
break;

case "binary":
result[which(captures[2].strip())] = null;
break;

case "config":
auto confFile = captures[2].strip;
// The config file is special: if missing, that's fine too. So
// add it as a dependency only if it actually exists.
yap("stat ", confFile);
if (confFile.exists)
{
result[confFile] = null;
}
break;

case "library":
immutable libName = captures[2].strip();
immutable libPath = findLib(libName);
if (libPath.ptr)
{
yap("library ", libName, " ", libPath);
result[libPath] = null;
}
break;
result[confFile] = null;
}
break;

default: assert(0);
case "library":
immutable libName = captures[2].strip();
immutable libPath = findLib(libName);
if (libPath.ptr)
{
yap("library ", libName, " ", libPath);
result[libPath] = null;
}
break;

default: assert(0);
}
// All dependencies specified through --extra-file
foreach (immutable moduleSrc; extraFiles)
result[moduleSrc] = d2obj(moduleSrc);
return result;
}
// All dependencies specified through --extra-file
foreach (immutable moduleSrc; extraFiles)
result[moduleSrc] = d2obj(moduleSrc);
return result;
}

// Given module rootModule, returns a mapping of all dependees .d
// source filenames to their corresponding .o files sitting in
// directory workDir. The mapping is obtained by reading the 'rdmd.deps' file
// that would have been generated on a previous build.

private const(string[string]) tryGetDependencies(string rootModule,
string objDir, string[] compilerFlags, string depsFilename)
{
// Check if the old dependency file is fine
if (!force)
{
Expand All @@ -696,45 +716,18 @@ private string[string] getDependencies(string rootModule, string workDir,
if (depsT > SysTime.min)
{
// See if the deps file is still in good shape
auto deps = readDepsFile();
auto deps = readDepsFile(objDir, depsFilename);
auto allDeps = chain(rootModule.only, deps.byKey);
bool mustRebuildDeps = allDeps.anyNewerThan(depsT);
if (!mustRebuildDeps)
{
// Cool, we're in good shape
// Dependencies are up to date, return them
return deps;
}
// Fall through to rebuilding the deps file
}
}

immutable rootDir = dirName(rootModule);

// Filter out -lib. With -o-, it will create an empty library file.
compilerFlags = compilerFlags.filter!(flag => flag != "-lib").array();

// Collect dependencies
auto depsGetter =
// "cd " ~ shellQuote(rootDir) ~ " && "
[ compiler ] ~ compilerFlags ~
["-v", "-o-", rootModule, "-I" ~ rootDir];

scope(failure)
{
// Delete the deps file on failure, we don't want to be fooled
// by it next time we try
collectException(std.file.remove(depsFilename));
}

immutable depsExitCode = run(depsGetter, depsFilename);
if (depsExitCode)
{
stderr.writefln("Failed: %s", depsGetter);
collectException(std.file.remove(depsFilename));
exit(depsExitCode);
}

return dryRun ? null : readDepsFile();
return null; // dependendies are stale or have not been evaluated
}

// Is any file newer than the given file?
Expand Down