Use large strings to avoid overflow issues (#100)

* Use large strings to possibly avoid apache/arrow#36295

* Packed numpy arrays can't be seamlessly converted

* linting

adam_core/coordinates/origin.py

@@ -84,7 +84,7 @@ def SOLAR_SYSTEM_BARYCENTER(cls) -> float:
 # TODO: Replace with DictionaryColumn or similar
 #       Investigate whether this class is even necessary
 class Origin(qv.Table):
-    code = qv.StringColumn()
+    code = qv.LargeStringColumn()
 
     def __eq__(self, other: object) -> np.ndarray:
         if isinstance(other, (str, np.ndarray)):

adam_core/coordinates/tests/test_residuals.py

@@ -288,7 +288,7 @@ def test_Residuals_calculate(observed_array, predicted_array, expected_residuals
         vy=observed_array[:, 4],
         vz=observed_array[:, 5],
         covariance=CoordinateCovariances.from_sigmas(np.full((4, 6), 0.1)),
-        origin=Origin.from_kwargs(code=np.full(4, "SUN")),
+        origin=Origin.from_kwargs(code=np.full(4, "SUN", dtype="object")),
         frame="ecliptic",
     )
     predicted = CartesianCoordinates.from_kwargs(
@@ -298,7 +298,7 @@ def test_Residuals_calculate(observed_array, predicted_array, expected_residuals
         vx=predicted_array[:, 3],
         vy=predicted_array[:, 4],
         vz=predicted_array[:, 5],
-        origin=Origin.from_kwargs(code=np.full(4, "SUN")),
+        origin=Origin.from_kwargs(code=np.full(4, "SUN", dtype="object")),
         frame="ecliptic",
     )
 
@@ -370,7 +370,7 @@ def test_Residuals_calculate_missing_covariance_values(
         vy=observed_array[:, 4],
         vz=observed_array[:, 5],
         covariance=CoordinateCovariances.from_matrix(observed_covariances),
-        origin=Origin.from_kwargs(code=np.full(4, "SUN")),
+        origin=Origin.from_kwargs(code=np.full(4, "SUN", dtype="object")),
         frame="ecliptic",
     )
     predicted = CartesianCoordinates.from_kwargs(
@@ -380,7 +380,7 @@ def test_Residuals_calculate_missing_covariance_values(
         vx=predicted_array[:, 3],
         vy=predicted_array[:, 4],
         vz=predicted_array[:, 5],
-        origin=Origin.from_kwargs(code=np.full(4, "SUN")),
+        origin=Origin.from_kwargs(code=np.full(4, "SUN", dtype="object")),
         frame="ecliptic",
     )
 
@@ -452,7 +452,7 @@ def test_Residuals_calculate_missing_off_diagonal_covariance_values(
         vy=observed_array[:, 4],
         vz=observed_array[:, 5],
         covariance=CoordinateCovariances.from_matrix(observed_covariances),
-        origin=Origin.from_kwargs(code=np.full(4, "SUN")),
+        origin=Origin.from_kwargs(code=np.full(4, "SUN", dtype="object")),
         frame="ecliptic",
     )
     predicted = CartesianCoordinates.from_kwargs(
@@ -462,7 +462,7 @@ def test_Residuals_calculate_missing_off_diagonal_covariance_values(
         vx=predicted_array[:, 3],
         vy=predicted_array[:, 4],
         vz=predicted_array[:, 5],
-        origin=Origin.from_kwargs(code=np.full(4, "SUN")),
+        origin=Origin.from_kwargs(code=np.full(4, "SUN", dtype="object")),
         frame="ecliptic",
     )
 
@@ -496,7 +496,7 @@ def test_Residuals_calculate_raises_frames():
         vx=observed_array[:, 3],
         vy=observed_array[:, 4],
         vz=observed_array[:, 5],
-        origin=Origin.from_kwargs(code=np.full(10, "SUN")),
+        origin=Origin.from_kwargs(code=np.full(10, "SUN", dtype="object")),
         frame="ecliptic",
     )
 
@@ -508,7 +508,7 @@ def test_Residuals_calculate_raises_frames():
         vx=predicted_array[:, 3],
         vy=predicted_array[:, 4],
         vz=predicted_array[:, 5],
-        origin=Origin.from_kwargs(code=np.full(10, "SUN")),
+        origin=Origin.from_kwargs(code=np.full(10, "SUN", dtype="object")),
         frame="equatorial",
     )
 
@@ -526,7 +526,7 @@ def test_Residuals_calculate_raises_origins():
         vx=observed_array[:, 3],
         vy=observed_array[:, 4],
         vz=observed_array[:, 5],
-        origin=Origin.from_kwargs(code=np.full(10, "SUN")),
+        origin=Origin.from_kwargs(code=np.full(10, "SUN", dtype="object")),
         frame="equatorial",
     )
 
@@ -538,7 +538,7 @@ def test_Residuals_calculate_raises_origins():
         vx=predicted_array[:, 3],
         vy=predicted_array[:, 4],
         vz=predicted_array[:, 5],
-        origin=Origin.from_kwargs(code=np.full(10, "EARTH")),
+        origin=Origin.from_kwargs(code=np.full(10, "EARTH", dtype="object")),
         frame="equatorial",
     )
 

adam_core/dynamics/propagation.py

@@ -142,7 +142,7 @@ def propagate_2body(
     else:
         cartesian_covariances = None
 
-    origin_code = np.empty(n_orbits * n_times, dtype="U3")
+    origin_code = np.empty(n_orbits * n_times, dtype="object")
     origin_code.fill("SUN")
 
     orbits_propagated = Orbits.from_kwargs(

adam_core/dynamics/tests/test_kepler.py

@@ -234,7 +234,9 @@ def test_calc_period(orbital_elements):
     # Our test data from JPL Horizons uses a value of 1e99 for infinite
     P_desired = np.where(P_desired > 1e99, np.inf, P_desired)
     a = orbital_elements["a"].values
-    origin = Origin.from_kwargs(code=np.full(len(orbital_elements), "SUN"))
+    origin = Origin.from_kwargs(
+        code=np.full(len(orbital_elements), "SUN", dtype="object")
+    )
     mu = origin.mu()
 
     P_actual = calc_period(a, mu)
@@ -288,7 +290,9 @@ def test_calc_mean_motion(orbital_elements):
     # Test mean motion calculations
     n_desired = orbital_elements["n"]
     a = orbital_elements["a"].values
-    origin = Origin.from_kwargs(code=np.full(len(orbital_elements), "SUN"))
+    origin = Origin.from_kwargs(
+        code=np.full(len(orbital_elements), "SUN", dtype="object")
+    )
     mu = origin.mu()
 
     n_actual = np.degrees(calc_mean_motion(a, mu))

adam_core/observations/associations.py

@@ -10,12 +10,12 @@
 
 class Associations(qv.Table):
 
-    detection_id = qv.StringColumn()
-    object_id = qv.StringColumn(nullable=True)
+    detection_id = qv.LargeStringColumn()
+    object_id = qv.LargeStringColumn(nullable=True)
 
     # TODO: We may want to create a derivative class called "ProbabilisticAssociations" that
     # includes residuals with respect to an orbit
-    # orbit_id = qv.StringColumn(nullable=True)
+    # orbit_id = qv.LargeStringColumn(nullable=True)
     # residuals = Residuals.as_column(nullable=True) # from adam_core.coordinates.residuals import Residuals
 
     def group_by_object(self) -> Iterator["Associations"]:

adam_core/observations/detections.py

@@ -20,9 +20,9 @@ class PointSourceDetections(qv.Table):
 
     """
 
-    id = qv.StringColumn()
+    id = qv.LargeStringColumn()
 
-    exposure_id = qv.StringColumn(nullable=True)
+    exposure_id = qv.LargeStringColumn(nullable=True)
 
     # Some, but not all, point source data may include times for
     # individual observations within an exposure.

adam_core/observations/exposures.py

@@ -17,12 +17,12 @@ class Exposures(qv.Table):
     Exposures is a table of data about exposures that provide point source observations.
     """
 
-    id = qv.StringColumn()
+    id = qv.LargeStringColumn()
     start_time = Timestamp.as_column()
     duration = qv.Float64Column(validator=and_(ge(0), le(3600)))
     filter = qv.DictionaryColumn(index_type=pa.int32(), value_type=pa.string())
 
-    observatory_code = qv.StringColumn()
+    observatory_code = qv.LargeStringColumn()
 
     def group_by_observatory_code(self) -> Iterator[tuple[str, Exposures]]:
         """

adam_core/observers/observers.py

@@ -12,11 +12,11 @@
 
 
 class ObservatoryGeodetics(qv.Table):
-    code = qv.StringColumn()
+    code = qv.LargeStringColumn()
     longitude = qv.Float64Column()
     cos_phi = qv.Float64Column()
     sin_phi = qv.Float64Column()
-    name = qv.StringColumn()
+    name = qv.LargeStringColumn()
 
 
 # Read MPC extended observatory codes file
@@ -49,7 +49,7 @@ class ObservatoryGeodetics(qv.Table):
 
 
 class Observers(qv.Table):
-    code = qv.StringColumn(nullable=False)
+    code = qv.LargeStringColumn(nullable=False)
     coordinates = CartesianCoordinates.as_column()
 
     @classmethod

adam_core/orbits/ephemeris.py

@@ -10,8 +10,8 @@
 
 class Ephemeris(qv.Table):
 
-    orbit_id = qv.StringColumn()
-    object_id = qv.StringColumn(nullable=True)
+    orbit_id = qv.LargeStringColumn()
+    object_id = qv.LargeStringColumn(nullable=True)
     coordinates = SphericalCoordinates.as_column()
 
     # The coordinates as observed by the observer will be the result of

adam_core/orbits/orbits.py

@@ -12,8 +12,8 @@
 
 class Orbits(qv.Table):
 
-    orbit_id = qv.StringColumn(default=lambda: uuid.uuid4().hex)
-    object_id = qv.StringColumn(nullable=True)
+    orbit_id = qv.LargeStringColumn(default=lambda: uuid.uuid4().hex)
+    object_id = qv.LargeStringColumn(nullable=True)
     coordinates = CartesianCoordinates.as_column()
 
     def group_by_orbit_id(self) -> Iterable[Tuple[str, "Orbits"]]:

adam_core/orbits/query/sbdb.py

@@ -223,8 +223,8 @@ def query_sbdb(ids: npt.ArrayLike) -> Orbits:
         frame=frame,
     )
 
-    orbit_ids = np.array(orbit_ids)
-    object_ids = np.array(object_ids)
+    orbit_ids = np.array(orbit_ids, dtype="object")
+    object_ids = np.array(object_ids, dtype="object")
     classes = np.array(classes)
 
     return Orbits.from_kwargs(

adam_core/orbits/variants.py

@@ -14,8 +14,8 @@
 
 class VariantOrbits(qv.Table):
 
-    orbit_id = qv.StringColumn(default=lambda: uuid.uuid4().hex)
-    object_id = qv.StringColumn(nullable=True)
+    orbit_id = qv.LargeStringColumn(default=lambda: uuid.uuid4().hex)
+    object_id = qv.LargeStringColumn(nullable=True)
     weights = qv.Float64Column(nullable=True)
     weights_cov = qv.Float64Column(nullable=True)
     coordinates = CartesianCoordinates.as_column()
@@ -164,8 +164,8 @@ def collapse(self, orbits: Orbits) -> Orbits:
 
 class VariantEphemeris(qv.Table):
 
-    orbit_id = qv.StringColumn(default=lambda: uuid.uuid4().hex)
-    object_id = qv.StringColumn(nullable=True)
+    orbit_id = qv.LargeStringColumn(default=lambda: uuid.uuid4().hex)
+    object_id = qv.LargeStringColumn(nullable=True)
     weights = qv.Float64Column(nullable=True)
     weights_cov = qv.Float64Column(nullable=True)
     coordinates = SphericalCoordinates.as_column()

adam_core/time/tests/test_time.py

@@ -11,7 +11,7 @@
 
 
 class Wrapper(qv.Table):
-    id = qv.StringColumn()
+    id = qv.LargeStringColumn()
     times = Timestamp.as_column(nullable=True)