Use large strings to possibly avoid apache/arrow#36295

B612-Asteroid-Institute · Dec 27, 2023 · ef8ee48 · ef8ee48
1 parent 16b5d1e
commit ef8ee48
Show file tree

Hide file tree

Showing 9 changed files with 20 additions and 20 deletions.
diff --git a/adam_core/coordinates/origin.py b/adam_core/coordinates/origin.py
@@ -84,7 +84,7 @@ def SOLAR_SYSTEM_BARYCENTER(cls) -> float:
 # TODO: Replace with DictionaryColumn or similar
 #       Investigate whether this class is even necessary
 class Origin(qv.Table):
-    code = qv.StringColumn()
+    code = qv.LargeStringColumn()
 
     def __eq__(self, other: object) -> np.ndarray:
         if isinstance(other, (str, np.ndarray)):

diff --git a/adam_core/observations/associations.py b/adam_core/observations/associations.py
@@ -10,12 +10,12 @@
 
 class Associations(qv.Table):
 
-    detection_id = qv.StringColumn()
-    object_id = qv.StringColumn(nullable=True)
+    detection_id = qv.LargeStringColumn()
+    object_id = qv.LargeStringColumn(nullable=True)
 
     # TODO: We may want to create a derivative class called "ProbabilisticAssociations" that
     # includes residuals with respect to an orbit
-    # orbit_id = qv.StringColumn(nullable=True)
+    # orbit_id = qv.LargeStringColumn(nullable=True)
     # residuals = Residuals.as_column(nullable=True) # from adam_core.coordinates.residuals import Residuals
 
     def group_by_object(self) -> Iterator["Associations"]:

diff --git a/adam_core/observations/detections.py b/adam_core/observations/detections.py
@@ -20,9 +20,9 @@ class PointSourceDetections(qv.Table):
 
     """
 
-    id = qv.StringColumn()
+    id = qv.LargeStringColumn()
 
-    exposure_id = qv.StringColumn(nullable=True)
+    exposure_id = qv.LargeStringColumn(nullable=True)
 
     # Some, but not all, point source data may include times for
     # individual observations within an exposure.

diff --git a/adam_core/observations/exposures.py b/adam_core/observations/exposures.py
@@ -17,12 +17,12 @@ class Exposures(qv.Table):
     Exposures is a table of data about exposures that provide point source observations.
     """
 
-    id = qv.StringColumn()
+    id = qv.LargeStringColumn()
     start_time = Timestamp.as_column()
     duration = qv.Float64Column(validator=and_(ge(0), le(3600)))
     filter = qv.DictionaryColumn(index_type=pa.int32(), value_type=pa.string())
 
-    observatory_code = qv.StringColumn()
+    observatory_code = qv.LargeStringColumn()
 
     def group_by_observatory_code(self) -> Iterator[tuple[str, Exposures]]:
         """

diff --git a/adam_core/observers/observers.py b/adam_core/observers/observers.py
@@ -12,11 +12,11 @@
 
 
 class ObservatoryGeodetics(qv.Table):
-    code = qv.StringColumn()
+    code = qv.LargeStringColumn()
     longitude = qv.Float64Column()
     cos_phi = qv.Float64Column()
     sin_phi = qv.Float64Column()
-    name = qv.StringColumn()
+    name = qv.LargeStringColumn()
 
 
 # Read MPC extended observatory codes file
@@ -49,7 +49,7 @@ class ObservatoryGeodetics(qv.Table):
 
 
 class Observers(qv.Table):
-    code = qv.StringColumn(nullable=False)
+    code = qv.LargeStringColumn(nullable=False)
     coordinates = CartesianCoordinates.as_column()
 
     @classmethod

diff --git a/adam_core/orbits/ephemeris.py b/adam_core/orbits/ephemeris.py
@@ -10,8 +10,8 @@
 
 class Ephemeris(qv.Table):
 
-    orbit_id = qv.StringColumn()
-    object_id = qv.StringColumn(nullable=True)
+    orbit_id = qv.LargeStringColumn()
+    object_id = qv.LargeStringColumn(nullable=True)
     coordinates = SphericalCoordinates.as_column()
 
     # The coordinates as observed by the observer will be the result of

diff --git a/adam_core/orbits/orbits.py b/adam_core/orbits/orbits.py
@@ -12,8 +12,8 @@
 
 class Orbits(qv.Table):
 
-    orbit_id = qv.StringColumn(default=lambda: uuid.uuid4().hex)
-    object_id = qv.StringColumn(nullable=True)
+    orbit_id = qv.LargeStringColumn(default=lambda: uuid.uuid4().hex)
+    object_id = qv.LargeStringColumn(nullable=True)
     coordinates = CartesianCoordinates.as_column()
 
     def group_by_orbit_id(self) -> Iterable[Tuple[str, "Orbits"]]:

diff --git a/adam_core/orbits/variants.py b/adam_core/orbits/variants.py
@@ -14,8 +14,8 @@
 
 class VariantOrbits(qv.Table):
 
-    orbit_id = qv.StringColumn(default=lambda: uuid.uuid4().hex)
-    object_id = qv.StringColumn(nullable=True)
+    orbit_id = qv.LargeStringColumn(default=lambda: uuid.uuid4().hex)
+    object_id = qv.LargeStringColumn(nullable=True)
     weights = qv.Float64Column(nullable=True)
     weights_cov = qv.Float64Column(nullable=True)
     coordinates = CartesianCoordinates.as_column()
@@ -164,8 +164,8 @@ def collapse(self, orbits: Orbits) -> Orbits:
 
 class VariantEphemeris(qv.Table):
 
-    orbit_id = qv.StringColumn(default=lambda: uuid.uuid4().hex)
-    object_id = qv.StringColumn(nullable=True)
+    orbit_id = qv.LargeStringColumn(default=lambda: uuid.uuid4().hex)
+    object_id = qv.LargeStringColumn(nullable=True)
     weights = qv.Float64Column(nullable=True)
     weights_cov = qv.Float64Column(nullable=True)
     coordinates = SphericalCoordinates.as_column()

diff --git a/adam_core/time/tests/test_time.py b/adam_core/time/tests/test_time.py
@@ -11,7 +11,7 @@
 
 
 class Wrapper(qv.Table):
-    id = qv.StringColumn()
+    id = qv.LargeStringColumn()
     times = Timestamp.as_column(nullable=True)