diff --git a/src/modules/physics/box2d/Body.cpp b/src/modules/physics/box2d/Body.cpp
index 0e846cdc3..a21f0be4d 100644
--- a/src/modules/physics/box2d/Body.cpp
+++ b/src/modules/physics/box2d/Body.cpp
@@ -39,29 +39,20 @@ namespace box2d
 
 Body::Body(World *world, b2Vec2 p, Body::Type type)
 	: world(world)
-	, udata(nullptr)
 {
-	udata = new bodyudata();
-	udata->ref = nullptr;
 	b2BodyDef def;
 	def.position = Physics::scaleDown(p);
-	def.userData.pointer = (uintptr_t)udata;
+	def.userData.pointer = (uintptr_t)this;
 	body = world->world->CreateBody(&def);
 	// Box2D body holds a reference to the love Body.
 	this->retain();
 	this->setType(type);
-	world->registerObject(body, this);
 }
 
 Body::~Body()
 {
-	if (!udata)
-		return;
-
-	if (udata->ref)
-		delete udata->ref;
-
-	delete udata;
+	if (ref)
+		delete ref;
 }
 
 float Body::getX()
@@ -479,7 +470,7 @@ int Body::getFixtures(lua_State *L) const
 	{
 		if (!f)
 			break;
-		Fixture *fixture = (Fixture *)world->findObject(f);
+		Fixture *fixture = (Fixture *)(f->GetUserData().pointer);
 		if (!fixture)
 			throw love::Exception("A fixture has escaped Memoizer!");
 		luax_pushtype(L, fixture);
@@ -501,7 +492,7 @@ int Body::getJoints(lua_State *L) const
 		if (!je)
 			break;
 
-		Joint *joint = (Joint *) world->findObject(je->joint);
+		Joint *joint = (Joint *) (je->joint->GetUserData().pointer);
 		if (!joint)
 			throw love::Exception("A joint has escaped Memoizer!");
 
@@ -550,12 +541,11 @@ void Body::destroy()
 	}
 
 	world->world->DestroyBody(body);
-	world->unregisterObject(body);
-	body = NULL;
+	body = nullptr;
 
 	// Remove userdata reference to avoid it sticking around after GC
-	if (udata && udata->ref)
-		udata->ref->unref();
+	if (ref)
+		ref->unref();
 
 	// Box2D body destroyed. Release its reference to the love Body.
 	this->release();
@@ -565,24 +555,18 @@ int Body::setUserData(lua_State *L)
 {
 	love::luax_assert_argc(L, 1, 1);
 
-	if (udata == nullptr)
-	{
-		udata = new bodyudata();
-		body->GetUserData().pointer = (uintptr_t)udata;
-	}
-
-	if(!udata->ref)
-		udata->ref = new Reference();
+	if(!ref)
+		ref = new Reference();
 
-	udata->ref->ref(L);
+	ref->ref(L);
 
 	return 0;
 }
 
 int Body::getUserData(lua_State *L)
 {
-	if (udata != nullptr && udata->ref != nullptr)
-		udata->ref->push(L);
+	if (ref != nullptr)
+		ref->push(L);
 	else
 		lua_pushnil(L);
 
diff --git a/src/modules/physics/box2d/Body.h b/src/modules/physics/box2d/Body.h
index 95ae911e8..563812fe2 100644
--- a/src/modules/physics/box2d/Body.h
+++ b/src/modules/physics/box2d/Body.h
@@ -41,16 +41,6 @@ class World;
 class Shape;
 class Fixture;
 
-/**
- * This struct is stored in a void pointer in the Box2D Body class. For now, all
- * we need is a Lua reference to arbitrary data, but we might need more later.
- **/
-struct bodyudata
-{
-	// Reference to arbitrary data.
-	Reference *ref = nullptr;
-};
-
 /**
  * A Body is an entity which has position and orientation
  * in world space. A Body does have collision geometry
@@ -441,7 +431,8 @@ class Body : public love::physics::Body
 	// unowned?
 	World *world;
 
-	bodyudata *udata;
+	// Reference to arbitrary data.
+	Reference* ref = nullptr;
 
 }; // Body
 
diff --git a/src/modules/physics/box2d/Contact.cpp b/src/modules/physics/box2d/Contact.cpp
index 5b110dcc3..46e2b2a89 100644
--- a/src/modules/physics/box2d/Contact.cpp
+++ b/src/modules/physics/box2d/Contact.cpp
@@ -35,6 +35,7 @@ Contact::Contact(World *world, b2Contact *contact)
 	: contact(contact)
 	, world(world)
 {
+	//contact->user
 	world->registerObject(contact, this);
 }
 
@@ -145,8 +146,8 @@ void Contact::getChildren(int &childA, int &childB)
 
 void Contact::getFixtures(Fixture *&fixtureA, Fixture *&fixtureB)
 {
-	fixtureA = (Fixture *) world->findObject(contact->GetFixtureA());
-	fixtureB = (Fixture *) world->findObject(contact->GetFixtureB());
+	fixtureA = (Fixture *) (contact->GetFixtureA()->GetUserData().pointer);
+	fixtureB = (Fixture *) (contact->GetFixtureB()->GetUserData().pointer);
 
 	if (!fixtureA || !fixtureB)
 		throw love::Exception("A fixture has escaped Memoizer!");
diff --git a/src/modules/physics/box2d/Fixture.cpp b/src/modules/physics/box2d/Fixture.cpp
index a1d3dc994..12fc9b2d7 100644
--- a/src/modules/physics/box2d/Fixture.cpp
+++ b/src/modules/physics/box2d/Fixture.cpp
@@ -41,26 +41,18 @@ Fixture::Fixture(Body *body, Shape *shape, float density)
 	: body(body)
 	, fixture(nullptr)
 {
-	udata = new fixtureudata();
-	udata->ref = nullptr;
 	b2FixtureDef def;
 	def.shape = shape->shape;
-	def.userData.pointer = (uintptr_t)udata;
+	def.userData.pointer = (uintptr_t)this;
 	def.density = density;
 	fixture = body->body->CreateFixture(&def);
 	this->retain();
-	body->world->registerObject(fixture, this);
 }
 
 Fixture::~Fixture()
 {
-	if (!udata)
-		return;
-
-	if (udata->ref)
-		delete udata->ref;
-
-	delete udata;
+	if (ref)
+		delete ref;
 }
 
 void Fixture::checkCreateShape()
@@ -259,24 +251,18 @@ int Fixture::setUserData(lua_State *L)
 {
 	love::luax_assert_argc(L, 1, 1);
 
-	if (udata == nullptr)
-	{
-		udata = new fixtureudata();
-		fixture->GetUserData().pointer = (uintptr_t)udata;
-	}
-
-	if(!udata->ref)
-		udata->ref = new Reference();
+	if(!ref)
+		ref = new Reference();
 
-	udata->ref->ref(L);
+	ref->ref(L);
 
 	return 0;
 }
 
 int Fixture::getUserData(lua_State *L)
 {
-	if (udata->ref != nullptr)
-		udata->ref->push(L);
+	if (ref != nullptr)
+		ref->push(L);
 	else
 		lua_pushnil(L);
 
@@ -348,12 +334,11 @@ void Fixture::destroy(bool implicit)
 
 	if (!implicit && fixture != nullptr)
 		body->body->DestroyFixture(fixture);
-	body->world->unregisterObject(fixture);
 	fixture = nullptr;
 
 	// Remove userdata reference to avoid it sticking around after GC
-	if (udata && udata->ref)
-		udata->ref->unref();
+	if (ref)
+		ref->unref();
 
 	// Box2D fixture destroyed. Release its reference to the love Fixture.
 	this->release();
diff --git a/src/modules/physics/box2d/Fixture.h b/src/modules/physics/box2d/Fixture.h
index d4087c5c3..4f0d54e99 100644
--- a/src/modules/physics/box2d/Fixture.h
+++ b/src/modules/physics/box2d/Fixture.h
@@ -40,18 +40,6 @@ namespace box2d
 
 class World;
 
-/**
- * This struct is stored in a void pointer
- * in the Box2D Fixture class. For now, all we
- * need is a Lua reference to arbitrary data,
- * but we might need more later.
- **/
-struct fixtureudata
-{
-	// Reference to arbitrary data.
-	Reference *ref = nullptr;
-};
-
 /**
  * A Fixture is used to attach a shape to a body for collision detection.
  * A Fixture inherits its transform from its parent. Fixtures hold
@@ -212,9 +200,11 @@ class Fixture : public Object
 	void checkCreateShape();
 
 	Body *body;
-	fixtureudata *udata;
 	b2Fixture *fixture;
 
+	// Reference to arbitrary data.
+	Reference* ref = nullptr;
+
 	StrongRef<Shape> shape;
 
 };
diff --git a/src/modules/physics/box2d/GearJoint.cpp b/src/modules/physics/box2d/GearJoint.cpp
index 8f0968285..474e19448 100644
--- a/src/modules/physics/box2d/GearJoint.cpp
+++ b/src/modules/physics/box2d/GearJoint.cpp
@@ -69,7 +69,7 @@ Joint *GearJoint::getJointA() const
 	if (b2joint == nullptr)
 		return nullptr;
 
-	Joint *j = (Joint *) world->findObject(b2joint);
+	Joint *j = (Joint *) (b2joint->GetUserData().pointer);
 	if (j == nullptr)
 		throw love::Exception("A joint has escaped Memoizer!");
 
@@ -82,7 +82,7 @@ Joint *GearJoint::getJointB() const
 	if (b2joint == nullptr)
 		return nullptr;
 
-	Joint *j = (Joint *) world->findObject(b2joint);
+	Joint *j = (Joint *) (b2joint->GetUserData().pointer);
 	if (j == nullptr)
 		throw love::Exception("A joint has escaped Memoizer!");
 
diff --git a/src/modules/physics/box2d/Joint.cpp b/src/modules/physics/box2d/Joint.cpp
index bb5cf8aba..d387b7ae2 100644
--- a/src/modules/physics/box2d/Joint.cpp
+++ b/src/modules/physics/box2d/Joint.cpp
@@ -38,33 +38,22 @@ namespace box2d
 
 Joint::Joint(Body *body1)
 	: world(body1->world)
-	, udata(nullptr)
 	, body1(body1)
 	, body2(nullptr)
 {
-	udata = new jointudata();
-	udata->ref = nullptr;
 }
 
 Joint::Joint(Body *body1, Body *body2)
 	: world(body1->world)
-	, udata(nullptr)
 	, body1(body1)
 	, body2(body2)
 {
-	udata = new jointudata();
-	udata->ref = nullptr;
 }
 
 Joint::~Joint()
 {
-	if (!udata)
-		return;
-
-	if (udata->ref)
-		delete udata->ref;
-
-	delete udata;
+	if (ref)
+		delete ref;
 }
 
 Joint::Type Joint::getType() const
@@ -104,7 +93,7 @@ Body *Joint::getBodyA() const
 	if (b2body == nullptr)
 		return nullptr;
 
-	Body *body = (Body *) world->findObject(b2body);
+	Body *body = (Body *) (b2body->GetUserData().pointer);
 	if (body == nullptr)
 		throw love::Exception("A body has escaped Memoizer!");
 
@@ -117,7 +106,7 @@ Body *Joint::getBodyB() const
 	if (b2body == nullptr)
 		return nullptr;
 
-	Body *body = (Body *) world->findObject(b2body);
+	Body *body = (Body *) (b2body->GetUserData().pointer);
 	if (body == nullptr)
 		throw love::Exception("A body has escaped Memoizer!");
 
@@ -154,9 +143,8 @@ float Joint::getReactionTorque(float dt)
 
 b2Joint *Joint::createJoint(b2JointDef *def)
 {
-	def->userData.pointer = (uintptr_t)udata;
+	def->userData.pointer = (uintptr_t)this;
 	joint = world->world->CreateJoint(def);
-	world->registerObject(joint, this);
 	// Box2D joint has a reference to this love Joint.
 	this->retain();
 	return joint;
@@ -174,12 +162,11 @@ void Joint::destroyJoint(bool implicit)
 
 	if (!implicit && joint != nullptr)
 		world->world->DestroyJoint(joint);
-	world->unregisterObject(joint);
-	joint = NULL;
+	joint = nullptr;
 
 	// Remove userdata reference to avoid it sticking around after GC
-	if (udata && udata->ref)
-		udata->ref->unref();
+	if (ref)
+		ref->unref();
 
 	// Release the reference of the Box2D joint.
 	this->release();
@@ -199,24 +186,18 @@ int Joint::setUserData(lua_State *L)
 {
 	love::luax_assert_argc(L, 1, 1);
 
-	if (udata == nullptr)
-	{
-		udata = new jointudata();
-		joint->GetUserData().pointer = (uintptr_t)udata;
-	}
-
-	if(!udata->ref)
-		udata->ref = new Reference();
+	if(!ref)
+		ref = new Reference();
 
-	udata->ref->ref(L);
+	ref->ref(L);
 
 	return 0;
 }
 
 int Joint::getUserData(lua_State *L)
 {
-	if (udata != nullptr && udata->ref != nullptr)
-		udata->ref->push(L);
+	if (ref != nullptr)
+		ref->push(L);
 	else
 		lua_pushnil(L);
 
diff --git a/src/modules/physics/box2d/Joint.h b/src/modules/physics/box2d/Joint.h
index 0ef205588..140842df4 100644
--- a/src/modules/physics/box2d/Joint.h
+++ b/src/modules/physics/box2d/Joint.h
@@ -39,16 +39,6 @@ namespace box2d
 class Body;
 class World;
 
-/**
- * This struct is stored in a void pointer in the Box2D Joint class. For now, all
- * we need is a Lua reference to arbitrary data, but we might need more later.
- **/
-struct jointudata
-{
-    // Reference to arbitrary data.
-    Reference *ref = nullptr;
-};
-
 /**
  * A Joint acts as positioning constraints on Bodies.
  * A Joint can be used to prevent Bodies from going to
@@ -140,7 +130,8 @@ class Joint : public love::physics::Joint
 
 	World *world;
 
-    jointudata *udata;
+	// Reference to arbitrary data.
+	Reference* ref = nullptr;
 
 private:
 
diff --git a/src/modules/physics/box2d/World.cpp b/src/modules/physics/box2d/World.cpp
index 0538730a5..0b7ec8653 100644
--- a/src/modules/physics/box2d/World.cpp
+++ b/src/modules/physics/box2d/World.cpp
@@ -60,7 +60,7 @@ void World::ContactCallback::process(b2Contact *contact, const b2ContactImpulse
 
 		// Push first fixture.
 		{
-			Fixture *a = (Fixture *)world->findObject(contact->GetFixtureA());
+			Fixture *a = (Fixture *)(contact->GetFixtureA()->GetUserData().pointer);
 			if (a != nullptr)
 				luax_pushtype(L, a);
 			else
@@ -69,7 +69,7 @@ void World::ContactCallback::process(b2Contact *contact, const b2ContactImpulse
 
 		// Push second fixture.
 		{
-			Fixture *b = (Fixture *)world->findObject(contact->GetFixtureB());
+			Fixture *b = (Fixture *)(contact->GetFixtureB()->GetUserData().pointer);
 			if (b != nullptr)
 				luax_pushtype(L, b);
 			else
@@ -158,7 +158,7 @@ bool World::QueryCallback::ReportFixture(b2Fixture *fixture)
 	if (L != nullptr)
 	{
 		lua_pushvalue(L, funcidx);
-		Fixture *f = (Fixture *)world->findObject(fixture);
+		Fixture *f = (Fixture *)(fixture->GetUserData().pointer);
 		if (!f)
 			throw love::Exception("A fixture has escaped Memoizer!");
 		luax_pushtype(L, f);
@@ -184,7 +184,7 @@ World::CollectCallback::~CollectCallback()
 
 bool World::CollectCallback::ReportFixture(b2Fixture *f)
 {
-	Fixture *fixture = (Fixture *)world->findObject(f);
+	Fixture *fixture = (Fixture *)(f->GetUserData().pointer);
 	if (!fixture)
 		throw love::Exception("A fixture has escaped Memoizer!");
 	luax_pushtype(L, fixture);
@@ -210,7 +210,7 @@ float World::RayCastCallback::ReportFixture(b2Fixture *fixture, const b2Vec2 &po
 	if (L != nullptr)
 	{
 		lua_pushvalue(L, funcidx);
-		Fixture *f = (Fixture *)world->findObject(fixture);
+		Fixture *f = (Fixture *)(fixture->GetUserData().pointer);
 		if (!f)
 			throw love::Exception("A fixture has escaped Memoizer!");
 		luax_pushtype(L, f);
@@ -233,14 +233,14 @@ float World::RayCastCallback::ReportFixture(b2Fixture *fixture, const b2Vec2 &po
 
 void World::SayGoodbye(b2Fixture *fixture)
 {
-	Fixture *f = (Fixture *)findObject(fixture);
+	Fixture *f = (Fixture *)(fixture->GetUserData().pointer);
 	// Hint implicit destruction with true.
 	if (f) f->destroy(true);
 }
 
 void World::SayGoodbye(b2Joint *joint)
 {
-	Joint *j = (Joint *)findObject(joint);
+	Joint *j = (Joint *)(joint->GetUserData().pointer);
 	// Hint implicit destruction with true.
 	if (j) j->destroyJoint(true);
 }
@@ -351,8 +351,8 @@ void World::PostSolve(b2Contact *contact, const b2ContactImpulse *impulse)
 bool World::ShouldCollide(b2Fixture *fixtureA, b2Fixture *fixtureB)
 {
 	// Fixtures should be memoized, if we created them
-	Fixture *a = (Fixture *)findObject(fixtureA);
-	Fixture *b = (Fixture *)findObject(fixtureB);
+	Fixture *a = (Fixture *)(fixtureA->GetUserData().pointer);
+	Fixture *b = (Fixture *)(fixtureB->GetUserData().pointer);
 	if (!a || !b)
 		throw love::Exception("A fixture has escaped Memoizer!");
 	return filter.process(a, b);
@@ -507,7 +507,7 @@ int World::getBodies(lua_State *L) const
 			break;
 		if (b == groundBody)
 			continue;
-		Body *body = (Body *)findObject(b);
+		Body *body = (Body *)(b->GetUserData().pointer);
 		if (!body)
 			throw love::Exception("A body has escaped Memoizer!");
 		luax_pushtype(L, body);
@@ -526,7 +526,7 @@ int World::getJoints(lua_State *L) const
 	do
 	{
 		if (!j) break;
-		Joint *joint = (Joint *)findObject(j);
+		Joint *joint = (Joint *)(j->GetUserData().pointer);
 		if (!joint) throw love::Exception("A joint has escaped Memoizer!");
 		luax_pushjoint(L, joint);
 		lua_rawseti(L, -2, i);
@@ -635,7 +635,7 @@ void World::destroy()
 		b = b->GetNext();
 		if (t == groundBody)
 			continue;
-		Body *body = (Body *)findObject(t);
+		Body *body = (Body *)(t->GetUserData().pointer);
 		if (!body)
 			throw love::Exception("A body has escaped Memoizer!");
 		body->destroy();