Skip to content

Commit

Permalink
Merge pull request #3177 from neobrain/feature_thunk_pointer_annotations
Browse files Browse the repository at this point in the history 
Thunks: Add new pointer annotations to assist data layout analysis
  • Loading branch information
@Sonicadvance1
Sonicadvance1 authored Oct 23, 2023
2 parents 9ba78c9 + cb215b5 commit 978f607
Show file tree
Hide file tree
Showing 20 changed files with 518 additions and 97 deletions.
87 changes: 80 additions & 7 deletions ThunkLibs/Generator/analysis.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,33 @@ FindClassTemplateDeclByName(clang::DeclContext& decl_context, std::string_view s
}
}

struct TypeAnnotations {
bool is_opaque = false;
bool assumed_compatible = false;
};

static TypeAnnotations GetTypeAnnotations(clang::ASTContext& context, clang::CXXRecordDecl* decl) {
if (!decl->hasDefinition()) {
return {};
}

ErrorReporter report_error { context };
TypeAnnotations ret;

for (const clang::CXXBaseSpecifier& base : decl->bases()) {
auto annotation = base.getType().getAsString();
if (annotation == "fexgen::opaque_type") {
ret.is_opaque = true;
} else if (annotation == "fexgen::assume_compatible_data_layout") {
ret.assumed_compatible = true;
} else {
throw report_error(base.getSourceRange().getBegin(), "Unknown type annotation");
}
}

return ret;
}

static ParameterAnnotations GetParameterAnnotations(clang::ASTContext& context, clang::CXXRecordDecl* decl) {
if (!decl->hasDefinition()) {
return {};
Expand All @@ -161,7 +188,14 @@ static ParameterAnnotations GetParameterAnnotations(clang::ASTContext& context,
ParameterAnnotations ret;

for (const clang::CXXBaseSpecifier& base : decl->bases()) {
throw report_error(base.getSourceRange().getBegin(), "Unknown parameter annotation");
auto annotation = base.getType().getAsString();
if (annotation == "fexgen::ptr_passthrough") {
ret.is_passthrough = true;
} else if (annotation == "fexgen::assume_compatible_data_layout") {
ret.assume_compatible = true;
} else {
throw report_error(base.getSourceRange().getBegin(), "Unknown parameter annotation");
}
}

return ret;
Expand All @@ -170,6 +204,8 @@ static ParameterAnnotations GetParameterAnnotations(clang::ASTContext& context,
void AnalysisAction::ParseInterface(clang::ASTContext& context) {
ErrorReporter report_error { context };

const std::unordered_map<unsigned, ParameterAnnotations> no_param_annotations {};

// TODO: Assert fex_gen_type is not declared at non-global namespaces
if (auto template_decl = FindClassTemplateDeclByName(*context.getTranslationUnitDecl(), "fex_gen_type")) {
for (auto* decl : template_decl->specializations()) {
Expand All @@ -184,9 +220,14 @@ void AnalysisAction::ParseInterface(clang::ASTContext& context) {
type = type->getLocallyUnqualifiedSingleStepDesugaredType();

if (type->isFunctionPointerType() || type->isFunctionType()) {
funcptr_types.insert(type.getTypePtr());
thunked_funcptrs[type.getAsString()] = std::pair { type.getTypePtr(), no_param_annotations };
} else {
[[maybe_unused]] auto [it, inserted] = types.emplace(context.getCanonicalType(type.getTypePtr()), RepackedType { });
const auto annotations = GetTypeAnnotations(context, decl);
RepackedType repack_info = {
.assumed_compatible = annotations.is_opaque || annotations.assumed_compatible,
.pointers_only = annotations.is_opaque && !annotations.assumed_compatible,
};
[[maybe_unused]] auto [it, inserted] = types.emplace(context.getCanonicalType(type.getTypePtr()), repack_info);
assert(inserted);
}
}
Expand Down Expand Up @@ -349,7 +390,7 @@ void AnalysisAction::ParseInterface(clang::ASTContext& context) {

data.callbacks.emplace(param_idx, callback);
if (!callback.is_stub && !callback.is_guest && !data.custom_host_impl) {
funcptr_types.insert(context.getCanonicalType(funcptr));
thunked_funcptrs[emitted_function->getNameAsString() + "_cb" + std::to_string(param_idx)] = std::pair { context.getCanonicalType(funcptr), no_param_annotations };
}

if (data.callbacks.size() != 1) {
Expand All @@ -358,20 +399,38 @@ void AnalysisAction::ParseInterface(clang::ASTContext& context) {
if (funcptr->isVariadic() && !callback.is_stub) {
throw report_error(template_arg_loc, "Variadic callbacks are not supported");
}

// Force treatment as passthrough-pointer
data.param_annotations[param_idx].is_passthrough = true;
} else if (param_type->isBuiltinType()) {
// NOTE: Intentionally not using getCanonicalType here since that would turn e.g. size_t into platform-specific types
// TODO: Still, we may want to de-duplicate some of these...
types.emplace(param_type.getTypePtr(), RepackedType { });
} else if (param_type->isEnumeralType()) {
types.emplace(context.getCanonicalType(param_type.getTypePtr()), RepackedType { });
} else if ( param_type->isStructureType()) {
} else if ( param_type->isStructureType() &&
!(types.contains(context.getCanonicalType(param_type.getTypePtr())) &&
LookupType(context, param_type.getTypePtr()).assumed_compatible)) {
check_struct_type(param_type.getTypePtr());
types.emplace(context.getCanonicalType(param_type.getTypePtr()), RepackedType { });
} else if (param_type->isPointerType()) {
auto pointee_type = param_type->getPointeeType()->getLocallyUnqualifiedSingleStepDesugaredType();
if ( pointee_type->isStructureType()) {
if ((types.contains(context.getCanonicalType(pointee_type.getTypePtr())) && LookupType(context, pointee_type.getTypePtr()).assumed_compatible)) {
// Nothing to do
data.param_annotations[param_idx].assume_compatible = true;
} else if ( pointee_type->isStructureType() &&
!(types.contains(context.getCanonicalType(pointee_type.getTypePtr())) &&
LookupType(context, pointee_type.getTypePtr()).assumed_compatible)) {
check_struct_type(pointee_type.getTypePtr());
types.emplace(context.getCanonicalType(pointee_type.getTypePtr()), RepackedType { });
} else if (data.param_annotations[param_idx].is_passthrough) {
if (!data.custom_host_impl) {
throw report_error(param_loc, "Passthrough annotation requires custom host implementation");
}

// Nothing to do
} else if (data.param_annotations[param_idx].assume_compatible) {
// Nothing to do
} else if (false /* TODO: Can't check if this is unsupported until data layout analysis is complete */) {
throw report_error(param_loc, "Unsupported parameter type")
.addNote(report_error(emitted_function->getNameInfo().getLoc(), "in function", clang::DiagnosticsEngine::Note))
Expand Down Expand Up @@ -417,7 +476,7 @@ void AnalysisAction::ParseInterface(clang::ASTContext& context) {

// For indirect calls, register the function signature as a function pointer type
if (namespace_info.indirect_guest_calls) {
funcptr_types.insert(context.getCanonicalType(emitted_function->getFunctionType()));
thunked_funcptrs[emitted_function->getNameAsString()] = std::pair { context.getCanonicalType(emitted_function->getFunctionType()), data.param_annotations };
}

thunks.push_back(std::move(data));
Expand All @@ -439,6 +498,11 @@ void AnalysisAction::CoverReferencedTypes(clang::ASTContext& context) {
continue;
}

if (type_repack_info.assumed_compatible) {
// If assumed compatible, we don't need the member definitions
continue;
}

for (auto* member : type->getAsStructureType()->getDecl()->fields()) {
auto member_type = member->getType().getTypePtr();
while (member_type->isArrayType()) {
Expand All @@ -451,6 +515,15 @@ void AnalysisAction::CoverReferencedTypes(clang::ASTContext& context) {
if (!member_type->isBuiltinType()) {
member_type = context.getCanonicalType(member_type);
}
if (types.contains(member_type) && types.at(member_type).pointers_only) {
if (member_type == context.getCanonicalType(member->getType().getTypePtr())) {
throw std::runtime_error(fmt::format("\"{}\" references opaque type \"{}\" via non-pointer member \"{}\"",
clang::QualType { type, 0 }.getAsString(),
clang::QualType { member_type, 0 }.getAsString(),
member->getNameAsString()));
}
continue;
}
if (member_type->isUnionType() && !types.contains(member_type)) {
throw std::runtime_error(fmt::format("\"{}\" has unannotated member \"{}\" of union type \"{}\"",
clang::QualType { type, 0 }.getAsString(),
Expand Down
10 changes: 9 additions & 1 deletion ThunkLibs/Generator/analysis.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@ struct ThunkedCallback : FunctionParams {
};

struct ParameterAnnotations {
bool is_passthrough = false;
bool assume_compatible = false;

bool operator==(const ParameterAnnotations&) const = default;
};

Expand Down Expand Up @@ -115,6 +118,8 @@ class AnalysisAction : public clang::ASTFrontendAction {
std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(clang::CompilerInstance&, clang::StringRef /*file*/) override;

struct RepackedType {
bool assumed_compatible = false; // opaque_type or assume_compatible_data_layout
bool pointers_only = assumed_compatible; // if true, only pointers to this type may be used
};

protected:
Expand All @@ -132,7 +137,10 @@ class AnalysisAction : public clang::ASTFrontendAction {
std::vector<ThunkedFunction> thunks;
std::vector<ThunkedAPIFunction> thunked_api;

std::unordered_set<const clang::Type*> funcptr_types;
// Set of function types for which to generate Guest->Host thunking trampolines.
// The map key is a unique identifier that must be consistent between guest/host processing passes.
// The map value is a pair of the function pointer's clang::Type and the mapping of parameter annotations
std::unordered_map<std::string, std::pair<const clang::Type*, std::unordered_map<unsigned, ParameterAnnotations>>> thunked_funcptrs;

std::unordered_map<const clang::Type*, RepackedType> types;
std::optional<unsigned> lib_version;
Expand Down
54 changes: 52 additions & 2 deletions ThunkLibs/Generator/data_layout.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,14 @@ std::unordered_map<const clang::Type*, TypeInfo> ComputeDataLayout(const clang::

// First, add all types directly used in function signatures of the library API to the meta set
for (const auto& [type, type_repack_info] : types) {
if (type_repack_info.assumed_compatible) {
auto [_, inserted] = layout.insert(std::pair { context.getCanonicalType(type), TypeInfo {} });
if (!inserted) {
throw std::runtime_error("Failed to gather type metadata: Opaque type \"" + clang::QualType { type, 0 }.getAsString() + "\" already registered");
}
continue;
}

if (type->isIncompleteType()) {
throw std::runtime_error("Cannot compute data layout of incomplete type \"" + clang::QualType { type, 0 }.getAsString() + "\". Did you forget any annotations?");
}
Expand Down Expand Up @@ -54,7 +62,7 @@ std::unordered_map<const clang::Type*, TypeInfo> ComputeDataLayout(const clang::

// Then, add information about members
for (const auto& [type, type_repack_info] : types) {
if (!type->isStructureType()) {
if (!type->isStructureType() || type_repack_info.assumed_compatible) {
continue;
}

Expand Down Expand Up @@ -151,10 +159,34 @@ ABI GetStableLayout(const clang::ASTContext& context, const std::unordered_map<c
return stable_layout;
}

static std::array<uint8_t, 32> GetSha256(const std::string& function_name) {
std::array<uint8_t, 32> sha256;
SHA256(reinterpret_cast<const unsigned char*>(function_name.data()),
function_name.size(),
sha256.data());
return sha256;
};

void AnalyzeDataLayoutAction::OnAnalysisComplete(clang::ASTContext& context) {
if (StrictModeEnabled(context)) {
type_abi = GetStableLayout(context, ComputeDataLayout(context, types));
}

// Register functions that must be guest-callable through host function pointers
for (auto funcptr_type_it = thunked_funcptrs.begin(); funcptr_type_it != thunked_funcptrs.end(); ++funcptr_type_it) {
auto& funcptr_id = funcptr_type_it->first;
auto& [type, param_annotations] = funcptr_type_it->second;
auto func_type = type->getAs<clang::FunctionProtoType>();
std::string mangled_name = clang::QualType { type, 0 }.getAsString();
auto cb_sha256 = GetSha256("fexcallback_" + mangled_name);
FuncPtrInfo info = { cb_sha256 };

info.result = func_type->getReturnType().getAsString();
for (auto arg : func_type->getParamTypes()) {
info.args.push_back(arg.getAsString());
}
type_abi.thunked_funcptrs[funcptr_id] = std::move(info);
}
}

TypeCompatibility DataLayoutCompareAction::GetTypeCompatibility(
Expand All @@ -178,6 +210,14 @@ TypeCompatibility DataLayoutCompareAction::GetTypeCompatibility(
}
}

if (types.contains(type) && types.at(type).assumed_compatible) {
if (types.at(type).pointers_only && !type->isPointerType()) {
throw std::runtime_error("Tried to dereference opaque type \"" + clang::QualType { type, 0 }.getAsString() + "\" when querying data layout compatibility");
}
type_compat.at(type) = TypeCompatibility::Full;
return TypeCompatibility::Full;
}

auto type_name = get_type_name(context, type);
auto& guest_info = guest_abi.at(type_name);
auto& host_info = host_abi.at(type->isBuiltinType() ? type : context.getCanonicalType(type));
Expand Down Expand Up @@ -233,12 +273,18 @@ TypeCompatibility DataLayoutCompareAction::GetTypeCompatibility(
// * Pointer member is annotated
// TODO: Don't restrict this to structure types. it applies to pointers to builtin types too!
auto host_member_pointee_type = context.getCanonicalType(host_member_type->getPointeeType().getTypePtr());
if (host_member_pointee_type->isPointerType()) {
if (types.contains(host_member_pointee_type) && types.at(host_member_pointee_type).assumed_compatible) {
// Pointee doesn't need repacking, but pointer needs extending on 32-bit
member_compat.push_back(is_32bit ? TypeCompatibility::Repackable : TypeCompatibility::Full);
} else if (host_member_pointee_type->isPointerType()) {
// This is a nested pointer, e.g. void**

if (is_32bit) {
// Nested pointers can't be repacked on 32-bit
member_compat.push_back(TypeCompatibility::None);
} else if (types.contains(host_member_pointee_type->getPointeeType().getTypePtr()) && types.at(host_member_pointee_type->getPointeeType().getTypePtr()).assumed_compatible) {
// Pointers to opaque types are fine
member_compat.push_back(TypeCompatibility::Full);
} else {
// Check the innermost type's compatibility on 64-bit
auto pointee_pointee_type = host_member_pointee_type->getPointeeType().getTypePtr();
Expand Down Expand Up @@ -297,6 +343,10 @@ TypeCompatibility DataLayoutCompareAction::GetTypeCompatibility(
return compat;
}

FuncPtrInfo DataLayoutCompareAction::LookupGuestFuncPtrInfo(const char* funcptr_id) {
return guest_abi.thunked_funcptrs.at(funcptr_id);
}

DataLayoutCompareActionFactory::DataLayoutCompareActionFactory(const ABI& abi) : abi(abi) {

}
Expand Down
3 changes: 3 additions & 0 deletions ThunkLibs/Generator/data_layout.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ struct FuncPtrInfo {
};

struct ABI : std::unordered_map<std::string, TypeInfo> {
std::unordered_map<std::string, FuncPtrInfo> thunked_funcptrs;
int pointer_size; // in bytes
};

Expand Down Expand Up @@ -111,6 +112,8 @@ class DataLayoutCompareAction : public AnalysisAction {
const std::unordered_map<const clang::Type*, TypeInfo> host_abi,
std::unordered_map<const clang::Type*, TypeCompatibility>& type_compat);

FuncPtrInfo LookupGuestFuncPtrInfo(const char* funcptr_id);

private:
const ABI& guest_abi;
};
Loading

0 comments on commit 978f607

Please sign in to comment.