Initial pass, needs more work

datasette/app.py

@@ -443,6 +443,49 @@ def __init__(
         self._root_token = secrets.token_hex(32)
         self.client = DatasetteClient(self)
 
+    async def initialize_internal_database(self):
+        await self.get_internal_database().execute_write_script(
+            """
+              CREATE TABLE IF NOT EXISTS datasette_metadata_instance_entries(
+                key text,
+                value text,
+                unique(key)
+              );
+
+              CREATE TABLE IF NOT EXISTS datasette_metadata_database_entries(
+                database_name text,
+                key text,
+                value text,
+                unique(database_name, key)
+              );
+
+              CREATE TABLE IF NOT EXISTS datasette_metadata_resource_entries(
+                database_name text,
+                resource_name text,
+                key text,
+                value text,
+                unique(database_name, resource_name, key)
+              );
+
+              CREATE TABLE IF NOT EXISTS datasette_metadata_column_entries(
+                database_name text,
+                resource_name text,
+                column_name text,
+                key text,
+                value text,
+                unique(database_name, resource_name, column_name, key)
+              );
+            """
+        )
+
+        for key in self._metadata_local or {}:
+            if key == "databases":
+                continue
+            await self.set_instance_metadata(key, self._metadata_local[key])
+            # else:
+            #  self.set_database_metadata(database, key, value)
+            #  self.set_database_metadata(database, key, value)
+
     def get_jinja_environment(self, request: Request = None) -> Environment:
         environment = self._jinja_env
         if request:
@@ -646,7 +689,101 @@ def _metadata_recursive_update(self, orig, updated):
                 orig[key] = upd_value
         return orig
 
-    def metadata(self, key=None, database=None, table=None, fallback=True):
+    async def get_instance_metadata(self) -> dict[str, any]:
+        rows = await self.get_internal_database().execute(
+            """
+              SELECT
+                key,
+                value
+              FROM datasette_metadata_instance_entries
+            """
+        )
+        return dict(rows)
+
+    async def get_database_metadata(self, database_name: str) -> dict[str, any]:
+        rows = await self.get_internal_database().execute(
+            """
+              SELECT
+                key,
+                value
+              FROM datasette_metadata_database_entries
+              WHERE database_name = ?
+            """,
+            [database_name],
+        )
+        return dict(rows)
+
+    async def get_resource_metadata(
+        self, database_name: str, resource_name: str
+    ) -> dict[str, any]:
+        rows = await self.get_internal_database().execute(
+            """
+              SELECT
+                key,
+                value
+              FROM datasette_metadata_resource_entries
+              WHERE database_name = ?
+                AND resource_name = ?
+            """,
+            [database_name, resource_name],
+        )
+        return dict(rows)
+
+    async def get_column_metadata(
+        self, database_name: str, resource_name: str, column_name: str
+    ) -> dict[str, any]:
+        rows = await self.get_internal_database().execute(
+            """
+              SELECT
+                key,
+                value
+              FROM datasette_metadata_column_entries
+              WHERE database_name = ?
+                AND resource_name = ?
+                AND column_name = ?
+            """,
+            [database_name, resource_name, column_name],
+        )
+        return dict(rows)
+
+    async def set_instance_metadata(self, key: str, value: str):
+        # TODO upsert only supported on SQLite 3.24.0 (2018-06-04)
+        await self.get_internal_database().execute_write(
+            """
+              INSERT INTO datasette_metadata_instance_entries(key, value)
+                VALUES(?, ?)
+                ON CONFLICT(key) DO UPDATE SET value = excluded.value;
+            """,
+            [key, value],
+        )
+
+    async def set_database_metadata(self, database_name: str, key: str, value: str):
+        # TODO upsert only supported on SQLite 3.24.0 (2018-06-04)
+        await self.get_internal_database().execute_write(
+            """
+              INSERT INTO datasette_metadata_database_entries(database_name, key, value)
+                VALUES(?, ?, ?)
+                ON CONFLICT(database_name, key) DO UPDATE SET value = excluded.value;
+            """,
+            [database_name, key, value],
+        )
+
+    async def set_resource_metadata(
+        self, database_name: str, resource_name: str, key: str, value: str
+    ):
+        pass
+
+    async def set_column_metadata(
+        self,
+        database_name: str,
+        resource_name: str,
+        column_name: str,
+        key: str,
+        value: str,
+    ):
+        pass
+
+    def __metadata(self, key=None, database=None, table=None, fallback=True):
         """
         Looks up metadata, cascading backwards from specified level.
         Returns None if metadata value is not found.
@@ -694,10 +831,6 @@ def metadata(self, key=None, database=None, table=None, fallback=True):
                 m.update(item)
             return m
 
-    @property
-    def _metadata(self):
-        return self.metadata()
-
     def get_internal_database(self):
         return self._internal_database
 
@@ -774,20 +907,6 @@ async def get_canned_query(self, database_name, query_name, actor):
         if query:
             return query
 
-    def update_with_inherited_metadata(self, metadata):
-        # Fills in source/license with defaults, if available
-        metadata.update(
-            {
-                "source": metadata.get("source") or self.metadata("source"),
-                "source_url": metadata.get("source_url") or self.metadata("source_url"),
-                "license": metadata.get("license") or self.metadata("license"),
-                "license_url": metadata.get("license_url")
-                or self.metadata("license_url"),
-                "about": metadata.get("about") or self.metadata("about"),
-                "about_url": metadata.get("about_url") or self.metadata("about_url"),
-            }
-        )
-
     def _prepare_connection(self, conn, database):
         conn.row_factory = sqlite3.Row
         conn.text_factory = lambda x: str(x, "utf-8", "replace")
@@ -1079,11 +1198,6 @@ def absolute_url(self, request, path):
             url = "https://" + url[len("http://") :]
         return url
 
-    def _register_custom_units(self):
-        """Register any custom units defined in the metadata.json with Pint"""
-        for unit in self.metadata("custom_units") or []:
-            ureg.define(unit)
-
     def _connected_databases(self):
         return [
             {
@@ -1395,6 +1509,8 @@ def _config(self):
         return redact_keys(
             self.config, ("secret", "key", "password", "token", "hash", "dsn")
         )
+    async def _metadata(self):
+        return {"lox": "lol"}
 
     def _routes(self):
         routes = []
@@ -1436,10 +1552,6 @@ def add_route(view, regex):
             ),
             r"/:memory:(?P<rest>.*)$",
         )
-        add_route(
-            JsonDataView.as_view(self, "metadata.json", lambda: self.metadata()),
-            r"/-/metadata(\.(?P<format>json))?$",
-        )
         add_route(
             JsonDataView.as_view(self, "versions.json", self._versions),
             r"/-/versions(\.(?P<format>json))?$",
@@ -1585,9 +1697,9 @@ async def resolve_row(self, request):
     def app(self):
         """Returns an ASGI app function that serves the whole of Datasette"""
         routes = self._routes()
-        self._register_custom_units()
 
         async def setup_db():
+            await self.initialize_internal_database()
             # First time server starts up, calculate table counts for immutable databases
             for database in self.databases.values():
                 if not database.is_mutable:

datasette/facets.py

@@ -98,13 +98,14 @@ def get_querystring_pairs(self):
         # [('_foo', 'bar'), ('_foo', '2'), ('empty', '')]
         return urllib.parse.parse_qsl(self.request.query_string, keep_blank_values=True)
 
-    def get_facet_size(self):
+    async def get_facet_size(self):
         facet_size = self.ds.setting("default_facet_size")
         max_returned_rows = self.ds.setting("max_returned_rows")
         table_facet_size = None
         if self.table:
-            tables_metadata = self.ds.metadata("tables", database=self.database) or {}
-            table_metadata = tables_metadata.get(self.table) or {}
+            table_metadata = await self.ds.get_resource_metadata(
+                self.database, self.table
+            )
             if table_metadata:
                 table_facet_size = table_metadata.get("facet_size")
         custom_facet_size = self.request.args.get("_facet_size")
@@ -158,7 +159,7 @@ class ColumnFacet(Facet):
     async def suggest(self):
         row_count = await self.get_row_count()
         columns = await self.get_columns(self.sql, self.params)
-        facet_size = self.get_facet_size()
+        facet_size = await self.get_facet_size()
         suggested_facets = []
         already_enabled = [c["config"]["simple"] for c in self.get_configs()]
         for column in columns:
@@ -212,7 +213,7 @@ async def facet_results(self):
 
         qs_pairs = self.get_querystring_pairs()
 
-        facet_size = self.get_facet_size()
+        facet_size = await self.get_facet_size()
         for source_and_config in self.get_configs():
             config = source_and_config["config"]
             source = source_and_config["source"]
@@ -371,7 +372,7 @@ async def facet_results(self):
         facet_results = []
         facets_timed_out = []
 
-        facet_size = self.get_facet_size()
+        facet_size = await self.get_facet_size()
         for source_and_config in self.get_configs():
             config = source_and_config["config"]
             source = source_and_config["source"]
@@ -503,7 +504,7 @@ async def facet_results(self):
         facet_results = []
         facets_timed_out = []
         args = dict(self.get_querystring_pairs())
-        facet_size = self.get_facet_size()
+        facet_size = await self.get_facet_size()
         for source_and_config in self.get_configs():
             config = source_and_config["config"]
             source = source_and_config["source"]

datasette/views/base.py

@@ -274,10 +274,10 @@ async def get(self, request):
 
         end = time.perf_counter()
         data["query_ms"] = (end - start) * 1000
-        for key in ("source", "source_url", "license", "license_url"):
-            value = self.ds.metadata(key)
-            if value:
-                data[key] = value
+        # for key in ("source", "source_url", "license", "license_url"):
+        #    value = self.ds.metadata z(key)
+        #    if value:
+        #        data[key] = value
 
         # Special case for .jsono extension - redirect to _shape=objects
         if _format == "jsono":
@@ -385,7 +385,7 @@ async def get(self, request):
                 },
             }
             if "metadata" not in context:
-                context["metadata"] = self.ds.metadata()
+                context["metadata"] = {}
             r = await self.render(templates, request=request, context=context)
             if status_code is not None:
                 r.status = status_code

datasette/views/database.py

@@ -63,8 +63,7 @@ async def get(self, request, datasette):
         if format_ not in ("html", "json"):
             raise NotFound("Invalid format: {}".format(format_))
 
-        metadata = (datasette.metadata("databases") or {}).get(database, {})
-        datasette.update_with_inherited_metadata(metadata)
+        metadata = await datasette.get_database_metadata(database)
 
         sql_views = []
         for view_name in await db.view_names():
@@ -625,8 +624,7 @@ async def fetch_data_for_csv(request, _next=None):
                     )
                 }
             )
-            metadata = (datasette.metadata("databases") or {}).get(database, {})
-            datasette.update_with_inherited_metadata(metadata)
+            metadata = await datasette.get_database_metadata(database)
 
             renderers = {}
             for key, (_, can_render) in datasette.renderers.items():

datasette/views/index.py

@@ -151,7 +151,7 @@ async def get(self, request):
                 request=request,
                 context={
                     "databases": databases,
-                    "metadata": self.ds.metadata(),
+                    "metadata": {},
                     "datasette_version": __version__,
                     "private": not await self.ds.permission_allowed(
                         None, "view-instance"

datasette/views/row.py

@@ -85,18 +85,7 @@ async def template_data():
                     "_table.html",
                 ],
                 "row_actions": row_actions,
-                "metadata": (self.ds.metadata("databases") or {})
-                .get(database, {})
-                .get("tables", {})
-                .get(table, {}),
-                "top_row": make_slot_function(
-                    "top_row",
-                    self.ds,
-                    request,
-                    database=resolved.db.name,
-                    table=resolved.table,
-                    row=rows[0],
-                ),
+                "metadata": {},
             }
 
         data = {

datasette/views/table.py

@@ -147,7 +147,21 @@ async def display_columns_and_rows(
     """Returns columns, rows for specified table - including fancy foreign key treatment"""
     sortable_columns = sortable_columns or set()
     db = datasette.databases[database_name]
-    column_descriptions = datasette.metadata("columns", database_name, table_name) or {}
+    column_descriptions = dict(
+        await datasette.get_internal_database().execute(
+            """
+          SELECT
+            column_name,
+            value
+          FROM datasette_metadata_column_entries
+          WHERE database_name = ?
+            AND resource_name = ?
+            AND key = 'description'
+        """,
+            [database_name, table_name],
+        )
+    )
+
     column_details = {
         col.name: col for col in await db.table_column_details(table_name)
     }
@@ -1478,14 +1492,7 @@ async def extra_query():
 
     async def extra_metadata():
         "Metadata about the table and database"
-        metadata = (
-            (datasette.metadata("databases") or {})
-            .get(database_name, {})
-            .get("tables", {})
-            .get(table_name, {})
-        )
-        datasette.update_with_inherited_metadata(metadata)
-        return metadata
+        return await datasette.get_resource_metadata(database_name, table_name)
 
     async def extra_database():
         return database_name