From 7b41521b338304bd4065a9ec0db9b4c651ec6fed Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 5 Apr 2023 16:25:29 -0700 Subject: [PATCH 01/28] WIP new JSON for queries, refs #2049 --- datasette/app.py | 13 +- datasette/views/database.py | 329 ++++++++++++++++++++++++++++++++++++ 2 files changed, 340 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index d7dace67f5..966a6faf6b 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -33,7 +33,12 @@ from jinja2.exceptions import TemplateNotFound from .views.base import ureg -from .views.database import DatabaseDownload, DatabaseView, TableCreateView +from .views.database import ( + DatabaseDownload, + DatabaseView, + TableCreateView, + database_view, +) from .views.index import IndexView from .views.special import ( JsonDataView, @@ -1365,8 +1370,12 @@ def add_route(view, regex): r"/-/patterns$", ) add_route(DatabaseDownload.as_view(self), r"/(?P[^\/\.]+)\.db$") + # add_route( + # DatabaseView.as_view(self), r"/(?P[^\/\.]+)(\.(?P\w+))?$" + # ) add_route( - DatabaseView.as_view(self), r"/(?P[^\/\.]+)(\.(?P\w+))?$" + wrap_view(database_view, self), + r"/(?P[^\/\.]+)(\.(?P\w+))?$", ) add_route(TableCreateView.as_view(self), r"/(?P[^\/\.]+)/-/create$") add_route( diff --git a/datasette/views/database.py b/datasette/views/database.py index dda8251096..7b90a2dbfa 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -757,3 +757,332 @@ async def _table_columns(datasette, database_name): for view_name in await db.view_names(): table_columns[view_name] = [] return table_columns + + +async def database_view(request, datasette): + return await database_view_impl(request, datasette) + + +async def database_view_impl( + request, + datasette, + canned_query=None, + _size=None, + named_parameters=None, + write=False, +): + db = await datasette.resolve_database(request) + database = db.name + params = {key: request.args.get(key) for key in request.args} + sql = "" + if "sql" in params: + sql = params.pop("sql") + _shape = None + if "_shape" in params: + _shape = params.pop("_shape") + + private = False + if canned_query: + # Respect canned query permissions + visible, private = await datasette.check_visibility( + request.actor, + permissions=[ + ("view-query", (database, canned_query)), + ("view-database", database), + "view-instance", + ], + ) + if not visible: + raise Forbidden("You do not have permission to view this query") + + else: + await datasette.ensure_permissions(request.actor, [("execute-sql", database)]) + + # Extract any :named parameters + named_parameters = named_parameters or await derive_named_parameters(db, sql) + named_parameter_values = { + named_parameter: params.get(named_parameter) or "" + for named_parameter in named_parameters + if not named_parameter.startswith("_") + } + + # Set to blank string if missing from params + for named_parameter in named_parameters: + if named_parameter not in params and not named_parameter.startswith("_"): + params[named_parameter] = "" + + extra_args = {} + if params.get("_timelimit"): + extra_args["custom_time_limit"] = int(params["_timelimit"]) + if _size: + extra_args["page_size"] = _size + + templates = [f"query-{to_css_class(database)}.html", "query.html"] + if canned_query: + templates.insert( + 0, + f"query-{to_css_class(database)}-{to_css_class(canned_query)}.html", + ) + + query_error = None + + # Execute query - as write or as read + if write: + raise NotImplementedError("Write queries not yet implemented") + # if request.method == "POST": + # # If database is immutable, return an error + # if not db.is_mutable: + # raise Forbidden("Database is immutable") + # body = await request.post_body() + # body = body.decode("utf-8").strip() + # if body.startswith("{") and body.endswith("}"): + # params = json.loads(body) + # # But we want key=value strings + # for key, value in params.items(): + # params[key] = str(value) + # else: + # params = dict(parse_qsl(body, keep_blank_values=True)) + # # Should we return JSON? + # should_return_json = ( + # request.headers.get("accept") == "application/json" + # or request.args.get("_json") + # or params.get("_json") + # ) + # if canned_query: + # params_for_query = MagicParameters(params, request, self.ds) + # else: + # params_for_query = params + # ok = None + # try: + # cursor = await self.ds.databases[database].execute_write( + # sql, params_for_query + # ) + # message = metadata.get( + # "on_success_message" + # ) or "Query executed, {} row{} affected".format( + # cursor.rowcount, "" if cursor.rowcount == 1 else "s" + # ) + # message_type = self.ds.INFO + # redirect_url = metadata.get("on_success_redirect") + # ok = True + # except Exception as e: + # message = metadata.get("on_error_message") or str(e) + # message_type = self.ds.ERROR + # redirect_url = metadata.get("on_error_redirect") + # ok = False + # if should_return_json: + # return Response.json( + # { + # "ok": ok, + # "message": message, + # "redirect": redirect_url, + # } + # ) + # else: + # self.ds.add_message(request, message, message_type) + # return self.redirect(request, redirect_url or request.path) + # else: + + # async def extra_template(): + # return { + # "request": request, + # "db_is_immutable": not db.is_mutable, + # "path_with_added_args": path_with_added_args, + # "path_with_removed_args": path_with_removed_args, + # "named_parameter_values": named_parameter_values, + # "canned_query": canned_query, + # "success_message": request.args.get("_success") or "", + # "canned_write": True, + # } + + # return ( + # { + # "database": database, + # "rows": [], + # "truncated": False, + # "columns": [], + # "query": {"sql": sql, "params": params}, + # "private": private, + # }, + # extra_template, + # templates, + # ) + + # Not a write + if canned_query: + params_for_query = MagicParameters(params, request, datasette) + else: + params_for_query = params + try: + results = await datasette.execute( + database, sql, params_for_query, truncate=True, **extra_args + ) + columns = [r[0] for r in results.description] + except sqlite3.DatabaseError as e: + query_error = e + results = None + columns = [] + + allow_execute_sql = await datasette.permission_allowed( + request.actor, "execute-sql", database + ) + + return Response.json( + { + "ok": True, + "rows": [dict(r) for r in results], + # "columns": columns, + # "database": database, + # "params": params, + # "sql": sql, + # "_shape": _shape, + # "named_parameters": named_parameters, + # "named_parameter_values": named_parameter_values, + # "extra_args": extra_args, + # "templates": templates, + } + ) + + async def extra_template(): + display_rows = [] + truncate_cells = datasette.setting("truncate_cells_html") + for row in results.rows if results else []: + display_row = [] + for column, value in zip(results.columns, row): + display_value = value + # Let the plugins have a go + # pylint: disable=no-member + plugin_display_value = None + for candidate in pm.hook.render_cell( + row=row, + value=value, + column=column, + table=None, + database=database, + datasette=self.ds, + request=request, + ): + candidate = await await_me_maybe(candidate) + if candidate is not None: + plugin_display_value = candidate + break + if plugin_display_value is not None: + display_value = plugin_display_value + else: + if value in ("", None): + display_value = Markup(" ") + elif is_url(str(display_value).strip()): + display_value = markupsafe.Markup( + '{truncated_url}'.format( + url=markupsafe.escape(value.strip()), + truncated_url=markupsafe.escape( + truncate_url(value.strip(), truncate_cells) + ), + ) + ) + elif isinstance(display_value, bytes): + blob_url = path_with_format( + request=request, + format="blob", + extra_qs={ + "_blob_column": column, + "_blob_hash": hashlib.sha256(display_value).hexdigest(), + }, + ) + formatted = format_bytes(len(value)) + display_value = markupsafe.Markup( + '<Binary: {:,} byte{}>'.format( + blob_url, + ' title="{}"'.format(formatted) + if "bytes" not in formatted + else "", + len(value), + "" if len(value) == 1 else "s", + ) + ) + else: + display_value = str(value) + if truncate_cells and len(display_value) > truncate_cells: + display_value = display_value[:truncate_cells] + "\u2026" + display_row.append(display_value) + display_rows.append(display_row) + + # Show 'Edit SQL' button only if: + # - User is allowed to execute SQL + # - SQL is an approved SELECT statement + # - No magic parameters, so no :_ in the SQL string + edit_sql_url = None + is_validated_sql = False + try: + validate_sql_select(sql) + is_validated_sql = True + except InvalidSql: + pass + if allow_execute_sql and is_validated_sql and ":_" not in sql: + edit_sql_url = ( + self.ds.urls.database(database) + + "?" + + urlencode( + { + **{ + "sql": sql, + }, + **named_parameter_values, + } + ) + ) + + show_hide_hidden = "" + if metadata.get("hide_sql"): + if bool(params.get("_show_sql")): + show_hide_link = path_with_removed_args(request, {"_show_sql"}) + show_hide_text = "hide" + show_hide_hidden = '' + else: + show_hide_link = path_with_added_args(request, {"_show_sql": 1}) + show_hide_text = "show" + else: + if bool(params.get("_hide_sql")): + show_hide_link = path_with_removed_args(request, {"_hide_sql"}) + show_hide_text = "show" + show_hide_hidden = '' + else: + show_hide_link = path_with_added_args(request, {"_hide_sql": 1}) + show_hide_text = "hide" + hide_sql = show_hide_text == "show" + return { + "display_rows": display_rows, + "custom_sql": True, + "named_parameter_values": named_parameter_values, + "editable": editable, + "canned_query": canned_query, + "edit_sql_url": edit_sql_url, + "metadata": metadata, + "settings": self.ds.settings_dict(), + "request": request, + "show_hide_link": self.ds.urls.path(show_hide_link), + "show_hide_text": show_hide_text, + "show_hide_hidden": markupsafe.Markup(show_hide_hidden), + "hide_sql": hide_sql, + "table_columns": await _table_columns(self.ds, database) + if allow_execute_sql + else {}, + } + + return ( + { + "ok": not query_error, + "database": database, + "query_name": canned_query, + "rows": results.rows if results else [], + "truncated": results.truncated if results else False, + "columns": columns, + "query": {"sql": sql, "params": params}, + "error": str(query_error) if query_error else None, + "private": private, + "allow_execute_sql": allow_execute_sql, + }, + extra_template, + templates, + 400 if query_error else 200, + ) From 40dc5f5c501c4c32120148d94ec7c9d130141571 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Apr 2023 17:04:26 -0700 Subject: [PATCH 02/28] WIP --- datasette/views/database.py | 227 +++++++++++++++++++++++++++++++++--- datasette/views/table.py | 1 - 2 files changed, 212 insertions(+), 16 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 7b90a2dbfa..d097c93393 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -12,6 +12,7 @@ from datasette.utils import ( add_cors_headers, await_me_maybe, + call_with_supported_arguments, derive_named_parameters, format_bytes, tilde_decode, @@ -763,6 +764,119 @@ async def database_view(request, datasette): return await database_view_impl(request, datasette) +async def database_index_view(request, datasette, db): + database = db.name + visible, private = await datasette.check_visibility( + request.actor, + permissions=[ + ("view-database", database), + "view-instance", + ], + ) + if not visible: + raise Forbidden("You do not have permission to view this database") + + metadata = (datasette.metadata("databases") or {}).get(database, {}) + datasette.update_with_inherited_metadata(metadata) + + table_counts = await db.table_counts(5) + hidden_table_names = set(await db.hidden_table_names()) + all_foreign_keys = await db.get_all_foreign_keys() + + views = [] + for view_name in await db.view_names(): + view_visible, view_private = await datasette.check_visibility( + request.actor, + permissions=[ + ("view-table", (database, view_name)), + ("view-database", database), + "view-instance", + ], + ) + if view_visible: + views.append( + { + "name": view_name, + "private": view_private, + } + ) + + tables = [] + for table in table_counts: + table_visible, table_private = await datasette.check_visibility( + request.actor, + permissions=[ + ("view-table", (database, table)), + ("view-database", database), + "view-instance", + ], + ) + if not table_visible: + continue + table_columns = await db.table_columns(table) + tables.append( + { + "name": table, + "columns": table_columns, + "primary_keys": await db.primary_keys(table), + "count": table_counts[table], + "hidden": table in hidden_table_names, + "fts_table": await db.fts_table(table), + "foreign_keys": all_foreign_keys[table], + "private": table_private, + } + ) + + tables.sort(key=lambda t: (t["hidden"], t["name"])) + canned_queries = [] + for query in (await datasette.get_canned_queries(database, request.actor)).values(): + query_visible, query_private = await datasette.check_visibility( + request.actor, + permissions=[ + ("view-query", (database, query["name"])), + ("view-database", database), + "view-instance", + ], + ) + if query_visible: + canned_queries.append(dict(query, private=query_private)) + + async def database_actions(): + links = [] + for hook in pm.hook.database_actions( + datasette=datasette, + database=database, + actor=request.actor, + request=request, + ): + extra_links = await await_me_maybe(hook) + if extra_links: + links.extend(extra_links) + return links + + attached_databases = [d.name for d in await db.attached_databases()] + + allow_execute_sql = await datasette.permission_allowed( + request.actor, "execute-sql", database + ) + return Response.json( + { + "database": db.name, + "private": private, + "path": datasette.urls.database(database), + "size": db.size, + "tables": tables, + "hidden_count": len([t for t in tables if t["hidden"]]), + "views": views, + "queries": canned_queries, + "allow_execute_sql": allow_execute_sql, + "table_columns": await _table_columns(datasette, database) + if allow_execute_sql + else {}, + } + ) + + async def database_view_impl( request, datasette, @@ -798,6 +912,12 @@ async def database_view_impl( else: await datasette.ensure_permissions(request.actor, [("execute-sql", database)]) + # If there's no sql, show the database index page + if not sql: + return await database_index_view(request, datasette, db) + + validate_sql_select(sql) + # Extract any :named parameters named_parameters = named_parameters or await derive_named_parameters(db, sql) named_parameter_values = { @@ -909,6 +1029,7 @@ async def database_view_impl( # ) # Not a write + rows = [] if canned_query: params_for_query = MagicParameters(params, request, datasette) else: @@ -918,6 +1039,7 @@ async def database_view_impl( database, sql, params_for_query, truncate=True, **extra_args ) columns = [r[0] for r in results.description] + rows = list(results.rows) except sqlite3.DatabaseError as e: query_error = e results = None @@ -927,21 +1049,96 @@ async def database_view_impl( request.actor, "execute-sql", database ) - return Response.json( - { - "ok": True, - "rows": [dict(r) for r in results], - # "columns": columns, - # "database": database, - # "params": params, - # "sql": sql, - # "_shape": _shape, - # "named_parameters": named_parameters, - # "named_parameter_values": named_parameter_values, - # "extra_args": extra_args, - # "templates": templates, - } - ) + format_ = request.url_vars.get("format") or "html" + + if format_ == "csv": + raise NotImplementedError("CSV format not yet implemented") + elif format_ in datasette.renderers.keys(): + # Dispatch request to the correct output format renderer + # (CSV is not handled here due to streaming) + result = call_with_supported_arguments( + datasette.renderers[format_][0], + datasette=datasette, + columns=columns, + rows=rows, + sql=sql, + query_name=None, + database=db.name, + table=None, + request=request, + view_name="table", # TODO: should this be "query"? + # These will be deprecated in Datasette 1.0: + args=request.args, + data={ + "rows": rows, + }, # TODO what should this be? + ) + result = await await_me_maybe(result) + if result is None: + raise NotFound("No data") + if isinstance(result, dict): + r = Response( + body=result.get("body"), + status=result.get("status_code") or 200, + content_type=result.get("content_type", "text/plain"), + headers=result.get("headers"), + ) + elif isinstance(result, Response): + r = result + # if status_code is not None: + # # Over-ride the status code + # r.status = status_code + else: + assert False, f"{result} should be dict or Response" + elif format_ == "html": + headers = {} + templates = [f"query-{to_css_class(database)}.html", "query.html"] + template = datasette.jinja_env.select_template(templates) + alternate_url_json = datasette.absolute_url( + request, + datasette.urls.path(path_with_format(request=request, format="json")), + ) + headers.update( + { + "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) + } + ) + r = Response.html( + await datasette.render_template( + template, + dict( + data, + append_querystring=append_querystring, + path_with_replaced_args=path_with_replaced_args, + fix_path=datasette.urls.path, + settings=datasette.settings_dict(), + # TODO: review up all of these hacks: + alternate_url_json=alternate_url_json, + datasette_allow_facet=( + "true" if datasette.setting("allow_facet") else "false" + ), + is_sortable=any(c["sortable"] for c in data["display_columns"]), + allow_execute_sql=await datasette.permission_allowed( + request.actor, "execute-sql", resolved.db.name + ), + query_ms=1.2, + select_templates=[ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + ), + request=request, + view_name="table", + ), + headers=headers, + ) + else: + assert False, "Invalid format: {}".format(format_) + # if next_url: + # r.headers["link"] = f'<{next_url}>; rel="next"' + return r async def extra_template(): display_rows = [] diff --git a/datasette/views/table.py b/datasette/views/table.py index c102c10319..e367a075d2 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -9,7 +9,6 @@ from datasette.plugins import pm from datasette.database import QueryInterrupted from datasette import tracer -from datasette.renderer import json_renderer from datasette.utils import ( add_cors_headers, await_me_maybe, From 026429fadd7a1f4f85ebfda1bbfe882938f455f2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Apr 2023 20:47:03 -0700 Subject: [PATCH 03/28] Work in progress on query view, refs #2049 --- datasette/views/database.py | 420 ++++++++++++++++++++++++++++++++++++ setup.py | 2 +- tests/test_cli_serve_get.py | 9 +- 3 files changed, 426 insertions(+), 5 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index d097c93393..33b6702b97 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1,3 +1,4 @@ +from asyncinject import Registry import os import hashlib import itertools @@ -877,6 +878,416 @@ async def database_actions(): ) +async def query_view( + request, + datasette, + canned_query=None, + _size=None, + named_parameters=None, + write=False, +): + print("query_view") + db = await datasette.resolve_database(request) + database = db.name + # TODO: Why do I do this? Is it to eliminate multi-args? + # It's going to break ?_extra=...&_extra=... + params = {key: request.args.get(key) for key in request.args} + sql = "" + if "sql" in params: + sql = params.pop("sql") + + # TODO: Behave differently for canned query here: + await datasette.ensure_permissions(request.actor, [("execute-sql", database)]) + + _shape = None + if "_shape" in params: + _shape = params.pop("_shape") + + # ?_shape=arrays - "rows" is the default option, shown above + # ?_shape=objects - "rows" is a list of JSON key/value objects + # ?_shape=array - an JSON array of objects + # ?_shape=array&_nl=on - a newline-separated list of JSON objects + # ?_shape=arrayfirst - a flat JSON array containing just the first value from each row + # ?_shape=object - a JSON object keyed using the primary keys of the rows + async def _results(_sql, _params): + # Returns (results, error (can be None)) + try: + return await db.execute(_sql, _params, truncate=True), None + except Exception as e: + return None, e + + async def shape_arrays(_results): + results, error = _results + if error: + return {"ok": False, "error": str(error)} + return { + "ok": True, + "rows": [list(r) for r in results.rows], + "truncated": results.truncated, + } + + async def shape_objects(_results): + results, error = _results + if error: + return {"ok": False, "error": str(error)} + return { + "ok": True, + "rows": [dict(r) for r in results.rows], + "truncated": results.truncated, + } + + async def shape_array(_results): + results, error = _results + if error: + return {"ok": False, "error": str(error)} + return [dict(r) for r in results.rows] + + shape_fn = { + "arrays": shape_arrays, + "objects": shape_objects, + "array": shape_array, + # "arrayfirst": shape_arrayfirst, + # "object": shape_object, + }[_shape or "objects"] + + registry = Registry.from_dict( + { + "_results": _results, + "_shape": shape_fn, + }, + parallel=False, + ) + + results = await registry.resolve_multi( + ["_shape"], + results={ + "_sql": sql, + "_params": params, + }, + ) + + # If "shape" does not include "rows" we return that as the response + if "rows" not in results["_shape"]: + return Response.json(results["_shape"]) + + output = results["_shape"] + output.update(dict((k, v) for k, v in results.items() if not k.startswith("_"))) + + response = Response.json(output) + + assert False + + import pdb + + pdb.set_trace() + + if isinstance(output, dict) and output.get("ok") is False: + # TODO: Other error codes? + + response.status_code = 400 + + if datasette.cors: + add_cors_headers(response.headers) + + return response + + # registry = Registry( + # extra_count, + # extra_facet_results, + # extra_facets_timed_out, + # extra_suggested_facets, + # facet_instances, + # extra_human_description_en, + # extra_next_url, + # extra_columns, + # extra_primary_keys, + # run_display_columns_and_rows, + # extra_display_columns, + # extra_display_rows, + # extra_debug, + # extra_request, + # extra_query, + # extra_metadata, + # extra_extras, + # extra_database, + # extra_table, + # extra_database_color, + # extra_table_actions, + # extra_filters, + # extra_renderers, + # extra_custom_table_templates, + # extra_sorted_facet_results, + # extra_table_definition, + # extra_view_definition, + # extra_is_view, + # extra_private, + # extra_expandable_columns, + # extra_form_hidden_args, + # ) + + results = await registry.resolve_multi( + ["extra_{}".format(extra) for extra in extras] + ) + data = { + "ok": True, + "next": next_value and str(next_value) or None, + } + data.update( + { + key.replace("extra_", ""): value + for key, value in results.items() + if key.startswith("extra_") and key.replace("extra_", "") in extras + } + ) + raw_sqlite_rows = rows[:page_size] + data["rows"] = [dict(r) for r in raw_sqlite_rows] + + private = False + if canned_query: + # Respect canned query permissions + visible, private = await datasette.check_visibility( + request.actor, + permissions=[ + ("view-query", (database, canned_query)), + ("view-database", database), + "view-instance", + ], + ) + if not visible: + raise Forbidden("You do not have permission to view this query") + + else: + await datasette.ensure_permissions(request.actor, [("execute-sql", database)]) + + # If there's no sql, show the database index page + if not sql: + return await database_index_view(request, datasette, db) + + validate_sql_select(sql) + + # Extract any :named parameters + named_parameters = named_parameters or await derive_named_parameters(db, sql) + named_parameter_values = { + named_parameter: params.get(named_parameter) or "" + for named_parameter in named_parameters + if not named_parameter.startswith("_") + } + + # Set to blank string if missing from params + for named_parameter in named_parameters: + if named_parameter not in params and not named_parameter.startswith("_"): + params[named_parameter] = "" + + extra_args = {} + if params.get("_timelimit"): + extra_args["custom_time_limit"] = int(params["_timelimit"]) + if _size: + extra_args["page_size"] = _size + + templates = [f"query-{to_css_class(database)}.html", "query.html"] + if canned_query: + templates.insert( + 0, + f"query-{to_css_class(database)}-{to_css_class(canned_query)}.html", + ) + + query_error = None + + # Execute query - as write or as read + if write: + raise NotImplementedError("Write queries not yet implemented") + # if request.method == "POST": + # # If database is immutable, return an error + # if not db.is_mutable: + # raise Forbidden("Database is immutable") + # body = await request.post_body() + # body = body.decode("utf-8").strip() + # if body.startswith("{") and body.endswith("}"): + # params = json.loads(body) + # # But we want key=value strings + # for key, value in params.items(): + # params[key] = str(value) + # else: + # params = dict(parse_qsl(body, keep_blank_values=True)) + # # Should we return JSON? + # should_return_json = ( + # request.headers.get("accept") == "application/json" + # or request.args.get("_json") + # or params.get("_json") + # ) + # if canned_query: + # params_for_query = MagicParameters(params, request, self.ds) + # else: + # params_for_query = params + # ok = None + # try: + # cursor = await self.ds.databases[database].execute_write( + # sql, params_for_query + # ) + # message = metadata.get( + # "on_success_message" + # ) or "Query executed, {} row{} affected".format( + # cursor.rowcount, "" if cursor.rowcount == 1 else "s" + # ) + # message_type = self.ds.INFO + # redirect_url = metadata.get("on_success_redirect") + # ok = True + # except Exception as e: + # message = metadata.get("on_error_message") or str(e) + # message_type = self.ds.ERROR + # redirect_url = metadata.get("on_error_redirect") + # ok = False + # if should_return_json: + # return Response.json( + # { + # "ok": ok, + # "message": message, + # "redirect": redirect_url, + # } + # ) + # else: + # self.ds.add_message(request, message, message_type) + # return self.redirect(request, redirect_url or request.path) + # else: + + # async def extra_template(): + # return { + # "request": request, + # "db_is_immutable": not db.is_mutable, + # "path_with_added_args": path_with_added_args, + # "path_with_removed_args": path_with_removed_args, + # "named_parameter_values": named_parameter_values, + # "canned_query": canned_query, + # "success_message": request.args.get("_success") or "", + # "canned_write": True, + # } + + # return ( + # { + # "database": database, + # "rows": [], + # "truncated": False, + # "columns": [], + # "query": {"sql": sql, "params": params}, + # "private": private, + # }, + # extra_template, + # templates, + # ) + + # Not a write + rows = [] + if canned_query: + params_for_query = MagicParameters(params, request, datasette) + else: + params_for_query = params + try: + results = await datasette.execute( + database, sql, params_for_query, truncate=True, **extra_args + ) + columns = [r[0] for r in results.description] + rows = list(results.rows) + except sqlite3.DatabaseError as e: + query_error = e + results = None + columns = [] + + allow_execute_sql = await datasette.permission_allowed( + request.actor, "execute-sql", database + ) + + format_ = request.url_vars.get("format") or "html" + + if format_ == "csv": + raise NotImplementedError("CSV format not yet implemented") + elif format_ in datasette.renderers.keys(): + # Dispatch request to the correct output format renderer + # (CSV is not handled here due to streaming) + result = call_with_supported_arguments( + datasette.renderers[format_][0], + datasette=datasette, + columns=columns, + rows=rows, + sql=sql, + query_name=None, + database=db.name, + table=None, + request=request, + view_name="table", # TODO: should this be "query"? + # These will be deprecated in Datasette 1.0: + args=request.args, + data={ + "rows": rows, + }, # TODO what should this be? + ) + result = await await_me_maybe(result) + if result is None: + raise NotFound("No data") + if isinstance(result, dict): + r = Response( + body=result.get("body"), + status=result.get("status_code") or 200, + content_type=result.get("content_type", "text/plain"), + headers=result.get("headers"), + ) + elif isinstance(result, Response): + r = result + # if status_code is not None: + # # Over-ride the status code + # r.status = status_code + else: + assert False, f"{result} should be dict or Response" + elif format_ == "html": + headers = {} + templates = [f"query-{to_css_class(database)}.html", "query.html"] + template = datasette.jinja_env.select_template(templates) + alternate_url_json = datasette.absolute_url( + request, + datasette.urls.path(path_with_format(request=request, format="json")), + ) + headers.update( + { + "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) + } + ) + r = Response.html( + await datasette.render_template( + template, + dict( + data, + append_querystring=append_querystring, + path_with_replaced_args=path_with_replaced_args, + fix_path=datasette.urls.path, + settings=datasette.settings_dict(), + # TODO: review up all of these hacks: + alternate_url_json=alternate_url_json, + datasette_allow_facet=( + "true" if datasette.setting("allow_facet") else "false" + ), + is_sortable=any(c["sortable"] for c in data["display_columns"]), + allow_execute_sql=await datasette.permission_allowed( + request.actor, "execute-sql", resolved.db.name + ), + query_ms=1.2, + select_templates=[ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + ), + request=request, + view_name="table", + ), + headers=headers, + ) + else: + assert False, "Invalid format: {}".format(format_) + # if next_url: + # r.headers["link"] = f'<{next_url}>; rel="next"' + return r + + async def database_view_impl( request, datasette, @@ -887,10 +1298,19 @@ async def database_view_impl( ): db = await datasette.resolve_database(request) database = db.name + + if request.args.get("sql", "").strip(): + return await query_view( + request, datasette, canned_query, _size, named_parameters, write + ) + + # Index page shows the tables/views/canned queries for this database + params = {key: request.args.get(key) for key in request.args} sql = "" if "sql" in params: sql = params.pop("sql") + _shape = None if "_shape" in params: _shape = params.pop("_shape") diff --git a/setup.py b/setup.py index d41e428a22..b591869e1f 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ def get_version(): "mergedeep>=1.1.1", "itsdangerous>=1.1", "sqlite-utils>=3.30", - "asyncinject>=0.5", + "asyncinject>=0.6", ], entry_points=""" [console_scripts] diff --git a/tests/test_cli_serve_get.py b/tests/test_cli_serve_get.py index ac44e1e285..e484a6db21 100644 --- a/tests/test_cli_serve_get.py +++ b/tests/test_cli_serve_get.py @@ -1,6 +1,7 @@ from datasette.cli import cli, serve from datasette.plugins import pm from click.testing import CliRunner +from unittest.mock import ANY import textwrap import json @@ -35,11 +36,11 @@ def startup(datasette): ], ) assert 0 == result.exit_code, result.output - assert { - "database": "_memory", + assert json.loads(result.output) == { + "ok": True, + "rows": [{"sqlite_version()": ANY}], "truncated": False, - "columns": ["sqlite_version()"], - }.items() <= json.loads(result.output).items() + } # The plugin should have created hello.txt assert (plugins_dir / "hello.txt").read_text() == "hello" From a706f34b923c5c503c1fa8e6a260925c61cec3fb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Apr 2023 22:07:05 -0700 Subject: [PATCH 04/28] Remove debug lines --- datasette/views/database.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 33b6702b97..cb31a762ce 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -975,12 +975,6 @@ async def shape_array(_results): response = Response.json(output) - assert False - - import pdb - - pdb.set_trace() - if isinstance(output, dict) and output.get("ok") is False: # TODO: Other error codes? From 8b86fb7fb4840b0351c25cf6ff40ca9f17a8fee7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 May 2023 17:50:12 -0700 Subject: [PATCH 05/28] Better debugging --- datasette/utils/testing.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py index d4990784e9..7747f7ce3a 100644 --- a/datasette/utils/testing.py +++ b/datasette/utils/testing.py @@ -16,6 +16,9 @@ def __init__(self, httpx_response): def status(self): return self.httpx_response.status_code + def __repr__(self): + return "".format(self.httpx_response.url, self.status) + # Supports both for test-writing convenience @property def status_code(self): From 3304fd43a2492739e9803f513c4954a5fd1e374d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 May 2023 17:51:05 -0700 Subject: [PATCH 06/28] refresh_schemas() on database view --- datasette/views/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index cb31a762ce..3001618839 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -886,7 +886,6 @@ async def query_view( named_parameters=None, write=False, ): - print("query_view") db = await datasette.resolve_database(request) database = db.name # TODO: Why do I do this? Is it to eliminate multi-args? @@ -1290,6 +1289,7 @@ async def database_view_impl( named_parameters=None, write=False, ): + await datasette.refresh_schemas() db = await datasette.resolve_database(request) database = db.name From 6f903d5a98c59f8a602ef7de776ea631b7145ba8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 May 2023 17:51:19 -0700 Subject: [PATCH 07/28] Fixed a test --- tests/test_api.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 780e9fa534..e700ad7b8c 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -643,9 +643,6 @@ async def test_custom_sql(ds_client): "/fixtures.json?sql=select+content+from+simple_primary_key&_shape=objects" ) data = response.json() - assert {"sql": "select content from simple_primary_key", "params": {}} == data[ - "query" - ] assert [ {"content": "hello"}, {"content": "world"}, @@ -653,8 +650,6 @@ async def test_custom_sql(ds_client): {"content": "RENDER_CELL_DEMO"}, {"content": "RENDER_CELL_ASYNC"}, ] == data["rows"] - assert ["content"] == data["columns"] - assert "fixtures" == data["database"] assert not data["truncated"] From fdb141f6225ddf7209dc6abff0c1691bbb675465 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 May 2023 17:51:29 -0700 Subject: [PATCH 08/28] shape_arrayfirst for query view --- datasette/views/database.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 3001618839..eac57cd3e5 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -941,11 +941,17 @@ async def shape_array(_results): return {"ok": False, "error": str(error)} return [dict(r) for r in results.rows] + async def shape_arrayfirst(_results): + results, error = _results + if error: + return {"ok": False, "error": str(error)} + return [r[0] for r in results.rows] + shape_fn = { "arrays": shape_arrays, "objects": shape_objects, "array": shape_array, - # "arrayfirst": shape_arrayfirst, + "arrayfirst": shape_arrayfirst, # "object": shape_object, }[_shape or "objects"] From 6ae5312158540aa12c4fc714404c0f3c1a72e4d4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 May 2023 18:44:07 -0700 Subject: [PATCH 09/28] ?sql=... now displays HTML --- datasette/views/database.py | 158 +++++++++++++++++++++++++++++++++--- 1 file changed, 145 insertions(+), 13 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index eac57cd3e5..455ebd1fcc 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -12,10 +12,12 @@ from datasette.utils import ( add_cors_headers, + append_querystring, await_me_maybe, call_with_supported_arguments, derive_named_parameters, format_bytes, + path_with_replaced_args, tilde_decode, to_css_class, validate_sql_select, @@ -887,6 +889,145 @@ async def query_view( write=False, ): db = await datasette.resolve_database(request) + + format_ = request.url_vars.get("format") or "html" + force_shape = None + if format_ == "html": + force_shape = "arrays" + + data = await query_view_data( + request, + datasette, + canned_query=canned_query, + _size=_size, + named_parameters=named_parameters, + write=write, + force_shape=force_shape, + ) + if format_ == "csv": + raise NotImplementedError("CSV format not yet implemented") + elif format_ in datasette.renderers.keys(): + # Dispatch request to the correct output format renderer + # (CSV is not handled here due to streaming) + result = call_with_supported_arguments( + datasette.renderers[format_][0], + datasette=datasette, + columns=columns, + rows=rows, + sql=sql, + query_name=None, + database=db.name, + table=None, + request=request, + view_name="table", # TODO: should this be "query"? + # These will be deprecated in Datasette 1.0: + args=request.args, + data={ + "rows": rows, + }, # TODO what should this be? + ) + result = await await_me_maybe(result) + if result is None: + raise NotFound("No data") + if isinstance(result, dict): + r = Response( + body=result.get("body"), + status=result.get("status_code") or 200, + content_type=result.get("content_type", "text/plain"), + headers=result.get("headers"), + ) + elif isinstance(result, Response): + r = result + # if status_code is not None: + # # Over-ride the status code + # r.status = status_code + else: + assert False, f"{result} should be dict or Response" + elif format_ == "html": + headers = {} + templates = [f"query-{to_css_class(db.name)}.html", "query.html"] + template = datasette.jinja_env.select_template(templates) + alternate_url_json = datasette.absolute_url( + request, + datasette.urls.path(path_with_format(request=request, format="json")), + ) + headers.update( + { + "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) + } + ) + metadata = (datasette.metadata("databases") or {}).get(db.name, {}) + datasette.update_with_inherited_metadata(metadata) + + r = Response.html( + await datasette.render_template( + template, + dict( + data, + database=db.name, + database_color=lambda database: "ff0000", + metadata=metadata, + display_rows=data["rows"], + renderers={}, + query={ + "sql": request.args.get("sql"), + }, + editable=True, + append_querystring=append_querystring, + path_with_replaced_args=path_with_replaced_args, + fix_path=datasette.urls.path, + settings=datasette.settings_dict(), + # TODO: review up all of these hacks: + alternate_url_json=alternate_url_json, + datasette_allow_facet=( + "true" if datasette.setting("allow_facet") else "false" + ), + is_sortable=False, + allow_execute_sql=await datasette.permission_allowed( + request.actor, "execute-sql", db.name + ), + query_ms=1.2, + select_templates=[ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + ), + request=request, + view_name="table", + ), + headers=headers, + ) + else: + assert False, "Invalid format: {}".format(format_) + # if next_url: + # r.headers["link"] = f'<{next_url}>; rel="next"' + return r + + response = Response.json(data) + + if isinstance(data, dict) and data.get("ok") is False: + # TODO: Other error codes? + + response.status_code = 400 + + if datasette.cors: + add_cors_headers(response.headers) + + return response + + +async def query_view_data( + request, + datasette, + canned_query=None, + _size=None, + named_parameters=None, + write=False, + force_shape=None, +): + db = await datasette.resolve_database(request) database = db.name # TODO: Why do I do this? Is it to eliminate multi-args? # It's going to break ?_extra=...&_extra=... @@ -898,11 +1039,11 @@ async def query_view( # TODO: Behave differently for canned query here: await datasette.ensure_permissions(request.actor, [("execute-sql", database)]) - _shape = None + _shape = force_shape if "_shape" in params: _shape = params.pop("_shape") - # ?_shape=arrays - "rows" is the default option, shown above + # ?_shape=arrays # ?_shape=objects - "rows" is a list of JSON key/value objects # ?_shape=array - an JSON array of objects # ?_shape=array&_nl=on - a newline-separated list of JSON objects @@ -921,6 +1062,7 @@ async def shape_arrays(_results): return {"ok": False, "error": str(error)} return { "ok": True, + "columns": [r[0] for r in results.description], "rows": [list(r) for r in results.rows], "truncated": results.truncated, } @@ -978,17 +1120,7 @@ async def shape_arrayfirst(_results): output = results["_shape"] output.update(dict((k, v) for k, v in results.items() if not k.startswith("_"))) - response = Response.json(output) - - if isinstance(output, dict) and output.get("ok") is False: - # TODO: Other error codes? - - response.status_code = 400 - - if datasette.cors: - add_cors_headers(response.headers) - - return response + return output # registry = Registry( # extra_count, From bbbfdb034ca02c1aa3f02de0e8edf12d8190175c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Apr 2023 20:36:10 -0700 Subject: [PATCH 10/28] Add setuptools to dependencies Refs #2065 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index b591869e1f..1139ff31c5 100644 --- a/setup.py +++ b/setup.py @@ -59,6 +59,7 @@ def get_version(): "itsdangerous>=1.1", "sqlite-utils>=3.30", "asyncinject>=0.6", + "setuptools", ], entry_points=""" [console_scripts] From 3a7be0c5b1911554b7f7298ef459259f24a1a9d4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Apr 2023 21:20:38 -0700 Subject: [PATCH 11/28] Hopeful fix for Python 3.7 httpx failure, refs #2066 --- tests/test_csv.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/test_csv.py b/tests/test_csv.py index c43e528b83..ed83d68529 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -6,6 +6,7 @@ app_client_with_cors, app_client_with_trace, ) +import urllib.parse EXPECTED_TABLE_CSV = """id,content 1,hello @@ -154,11 +155,24 @@ async def test_csv_with_non_ascii_characters(ds_client): def test_max_csv_mb(app_client_csv_max_mb_one): + # This query deliberately generates a really long string + # should be 100*100*100*2 = roughly 2MB response = app_client_csv_max_mb_one.get( - ( - "/fixtures.csv?sql=select+'{}'+" - "from+compound_three_primary_keys&_stream=1&_size=max" - ).format("abcdefg" * 10000) + "/fixtures.csv?" + + urllib.parse.urlencode( + { + "sql": """ + select group_concat('ab', '') + from json_each(json_array({lots})), + json_each(json_array({lots})), + json_each(json_array({lots})) + """.format( + lots=", ".join(str(i) for i in range(100)) + ), + "_stream": 1, + "_size": "max", + } + ), ) # It's a 200 because we started streaming before we knew the error assert response.status == 200 From 305655c816a70fdfa729b37529d8c51490d4f4d5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Apr 2023 22:07:35 -0700 Subject: [PATCH 12/28] Add pip as a dependency too, for Rye - refs #2065 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 1139ff31c5..0eedadb2ff 100644 --- a/setup.py +++ b/setup.py @@ -60,6 +60,7 @@ def get_version(): "sqlite-utils>=3.30", "asyncinject>=0.6", "setuptools", + "pip", ], entry_points=""" [console_scripts] From 01353c7ee83ba2c58b42bdc35f3b26dfc06159ac Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 May 2023 11:44:27 -0700 Subject: [PATCH 13/28] Build docs with 3.11 on ReadTheDocs Inspired by https://github.com/simonw/sqlite-utils/issues/540 --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index e157fb9c22..5b30e75abe 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -3,7 +3,7 @@ version: 2 build: os: ubuntu-20.04 tools: - python: "3.9" + python: "3.11" sphinx: configuration: docs/conf.py From 59c52b5874045d884bac8bac35a4ecbfdde4e73c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 9 May 2023 09:24:28 -0700 Subject: [PATCH 14/28] Action: Deploy a Datasette branch preview to Vercel Closes #2070 --- .github/workflows/deploy-branch-preview.yml | 35 +++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/deploy-branch-preview.yml diff --git a/.github/workflows/deploy-branch-preview.yml b/.github/workflows/deploy-branch-preview.yml new file mode 100644 index 0000000000..872aff7196 --- /dev/null +++ b/.github/workflows/deploy-branch-preview.yml @@ -0,0 +1,35 @@ +name: Deploy a Datasette branch preview to Vercel + +on: + workflow_dispatch: + inputs: + branch: + description: "Branch to deploy" + required: true + type: string + +jobs: + deploy-branch-preview: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Install dependencies + run: | + pip install datasette-publish-vercel + - name: Deploy the preview + env: + VERCEL_TOKEN: ${{ secrets.BRANCH_PREVIEW_VERCEL_TOKEN }} + run: | + export BRANCH="${{ github.event.inputs.branch }}" + wget https://latest.datasette.io/fixtures.db + datasette publish vercel fixtures.db \ + --branch $BRANCH \ + --project "datasette-preview-$BRANCH" \ + --token $VERCEL_TOKEN \ + --scope datasette \ + --about "Preview of $BRANCH" \ + --about_url "https://github.com/simonw/datasette/tree/$BRANCH" From 3f39fba7ea74e54673cce8a71d6c1baeaa84c513 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 May 2023 11:35:34 -0700 Subject: [PATCH 15/28] datasette.utils.check_callable(obj) - refs #2078 --- datasette/utils/callable.py | 25 ++++++++++++++++++++ tests/test_utils_callable.py | 46 ++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 datasette/utils/callable.py create mode 100644 tests/test_utils_callable.py diff --git a/datasette/utils/callable.py b/datasette/utils/callable.py new file mode 100644 index 0000000000..5b8a30aca0 --- /dev/null +++ b/datasette/utils/callable.py @@ -0,0 +1,25 @@ +import asyncio +import types +from typing import NamedTuple, Any + + +class CallableStatus(NamedTuple): + is_callable: bool + is_async_callable: bool + + +def check_callable(obj: Any) -> CallableStatus: + if not callable(obj): + return CallableStatus(False, False) + + if isinstance(obj, type): + # It's a class + return CallableStatus(True, False) + + if isinstance(obj, types.FunctionType): + return CallableStatus(True, asyncio.iscoroutinefunction(obj)) + + if hasattr(obj, "__call__"): + return CallableStatus(True, asyncio.iscoroutinefunction(obj.__call__)) + + assert False, "obj {} is somehow callable with no __call__ method".format(repr(obj)) diff --git a/tests/test_utils_callable.py b/tests/test_utils_callable.py new file mode 100644 index 0000000000..d1d0aac5d0 --- /dev/null +++ b/tests/test_utils_callable.py @@ -0,0 +1,46 @@ +from datasette.utils.callable import check_callable +import pytest + + +class AsyncClass: + async def __call__(self): + pass + + +class NotAsyncClass: + def __call__(self): + pass + + +class ClassNoCall: + pass + + +async def async_func(): + pass + + +def non_async_func(): + pass + + +@pytest.mark.parametrize( + "obj,expected_is_callable,expected_is_async_callable", + ( + (async_func, True, True), + (non_async_func, True, False), + (AsyncClass(), True, True), + (NotAsyncClass(), True, False), + (ClassNoCall(), False, False), + (AsyncClass, True, False), + (NotAsyncClass, True, False), + (ClassNoCall, True, False), + ("", False, False), + (1, False, False), + (str, True, False), + ), +) +def test_check_callable(obj, expected_is_callable, expected_is_async_callable): + status = check_callable(obj) + assert status.is_callable == expected_is_callable + assert status.is_async_callable == expected_is_async_callable From 080577106167046428c946c9978631dd59e10181 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 May 2023 11:49:40 -0700 Subject: [PATCH 16/28] Rename callable.py to check_callable.py, refs #2078 --- datasette/utils/{callable.py => check_callable.py} | 0 tests/{test_utils_callable.py => test_utils_check_callable.py} | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename datasette/utils/{callable.py => check_callable.py} (100%) rename tests/{test_utils_callable.py => test_utils_check_callable.py} (94%) diff --git a/datasette/utils/callable.py b/datasette/utils/check_callable.py similarity index 100% rename from datasette/utils/callable.py rename to datasette/utils/check_callable.py diff --git a/tests/test_utils_callable.py b/tests/test_utils_check_callable.py similarity index 94% rename from tests/test_utils_callable.py rename to tests/test_utils_check_callable.py index d1d0aac5d0..4f72f9ffc1 100644 --- a/tests/test_utils_callable.py +++ b/tests/test_utils_check_callable.py @@ -1,4 +1,4 @@ -from datasette.utils.callable import check_callable +from datasette.utils.check_callable import check_callable import pytest From 94882aa72b5ef4afb267447de27f0ec2140400d8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 May 2023 15:05:58 -0700 Subject: [PATCH 17/28] --cors Access-Control-Max-Age: 3600, closes #2079 --- datasette/utils/__init__.py | 1 + docs/json_api.rst | 18 +++++++++++++++++- tests/test_api.py | 2 ++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 925c6560df..c388673d84 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1141,6 +1141,7 @@ def add_cors_headers(headers): headers["Access-Control-Allow-Headers"] = "Authorization, Content-Type" headers["Access-Control-Expose-Headers"] = "Link" headers["Access-Control-Allow-Methods"] = "GET, POST, HEAD, OPTIONS" + headers["Access-Control-Max-Age"] = "3600" _TILDE_ENCODING_SAFE = frozenset( diff --git a/docs/json_api.rst b/docs/json_api.rst index 7b130c58f9..c273c2a8d5 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -454,12 +454,28 @@ Enabling CORS ------------- If you start Datasette with the ``--cors`` option, each JSON endpoint will be -served with the following additional HTTP headers:: +served with the following additional HTTP headers: + +.. [[[cog + from datasette.utils import add_cors_headers + import textwrap + headers = {} + add_cors_headers(headers) + output = "\n".join("{}: {}".format(k, v) for k, v in headers.items()) + cog.out("\n::\n\n") + cog.out(textwrap.indent(output, ' ')) + cog.out("\n\n") +.. ]]] + +:: Access-Control-Allow-Origin: * Access-Control-Allow-Headers: Authorization, Content-Type Access-Control-Expose-Headers: Link Access-Control-Allow-Methods: GET, POST, HEAD, OPTIONS + Access-Control-Max-Age: 3600 + +.. [[[end]]] This allows JavaScript running on any domain to make cross-origin requests to interact with the Datasette API. diff --git a/tests/test_api.py b/tests/test_api.py index e700ad7b8c..e7d8d849fe 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -936,6 +936,7 @@ def test_cors( assert ( response.headers["Access-Control-Allow-Methods"] == "GET, POST, HEAD, OPTIONS" ) + assert response.headers["Access-Control-Max-Age"] == "3600" # Same request to app_client_two_attached_databases_one_immutable # should not have those headers - I'm using that fixture because # regular app_client doesn't have immutable fixtures.db which means @@ -946,6 +947,7 @@ def test_cors( assert "Access-Control-Allow-Headers" not in response.headers assert "Access-Control-Expose-Headers" not in response.headers assert "Access-Control-Allow-Methods" not in response.headers + assert "Access-Control-Max-Age" not in response.headers @pytest.mark.parametrize( From 8ea00e038d7d43d173cebdca21a31e774ff33f1c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 May 2023 17:18:43 -0700 Subject: [PATCH 18/28] New View base class (#2080) * New View base class, closes #2078 * Use new View subclass for PatternPortfolioView --- datasette/app.py | 40 +++++++++++++++++- datasette/views/base.py | 37 +++++++++++++++++ datasette/views/special.py | 19 +++++---- tests/test_base_view.py | 84 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 170 insertions(+), 10 deletions(-) create mode 100644 tests/test_base_view.py diff --git a/datasette/app.py b/datasette/app.py index 966a6faf6b..8bc6518f42 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -17,6 +17,7 @@ import sys import threading import time +import types import urllib.parse from concurrent import futures from pathlib import Path @@ -1366,7 +1367,7 @@ def add_route(view, regex): r"/-/allow-debug$", ) add_route( - PatternPortfolioView.as_view(self), + wrap_view(PatternPortfolioView, self), r"/-/patterns$", ) add_route(DatabaseDownload.as_view(self), r"/(?P[^\/\.]+)\.db$") @@ -1682,7 +1683,42 @@ def _cleaner_task_str(task): return _cleaner_task_str_re.sub("", s) -def wrap_view(view_fn, datasette): +def wrap_view(view_fn_or_class, datasette): + is_function = isinstance(view_fn_or_class, types.FunctionType) + if is_function: + return wrap_view_function(view_fn_or_class, datasette) + else: + if not isinstance(view_fn_or_class, type): + raise ValueError("view_fn_or_class must be a function or a class") + return wrap_view_class(view_fn_or_class, datasette) + + +def wrap_view_class(view_class, datasette): + async def async_view_for_class(request, send): + instance = view_class() + if inspect.iscoroutinefunction(instance.__call__): + return await async_call_with_supported_arguments( + instance.__call__, + scope=request.scope, + receive=request.receive, + send=send, + request=request, + datasette=datasette, + ) + else: + return call_with_supported_arguments( + instance.__call__, + scope=request.scope, + receive=request.receive, + send=send, + request=request, + datasette=datasette, + ) + + return async_view_for_class + + +def wrap_view_function(view_fn, datasette): @functools.wraps(view_fn) async def async_view_fn(request, send): if inspect.iscoroutinefunction(view_fn): diff --git a/datasette/views/base.py b/datasette/views/base.py index 927d1aff79..94645cd8dd 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -53,6 +53,43 @@ def __init__( self.message_is_html = message_is_html +class View: + async def head(self, request, datasette): + if not hasattr(self, "get"): + return await self.method_not_allowed(request) + response = await self.get(request, datasette) + response.body = "" + return response + + async def method_not_allowed(self, request): + if ( + request.path.endswith(".json") + or request.headers.get("content-type") == "application/json" + ): + response = Response.json( + {"ok": False, "error": "Method not allowed"}, status=405 + ) + else: + response = Response.text("Method not allowed", status=405) + return response + + async def options(self, request, datasette): + response = Response.text("ok") + response.headers["allow"] = ", ".join( + method.upper() + for method in ("head", "get", "post", "put", "patch", "delete") + if hasattr(self, method) + ) + return response + + async def __call__(self, request, datasette): + try: + handler = getattr(self, request.method.lower()) + except AttributeError: + return await self.method_not_allowed(request) + return await handler(request, datasette) + + class BaseView: ds = None has_json_alternate = True diff --git a/datasette/views/special.py b/datasette/views/special.py index 1aeb4be659..03e085d66e 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -6,7 +6,7 @@ tilde_encode, tilde_decode, ) -from .base import BaseView +from .base import BaseView, View import secrets import urllib @@ -57,13 +57,16 @@ async def get(self, request): ) -class PatternPortfolioView(BaseView): - name = "patterns" - has_json_alternate = False - - async def get(self, request): - await self.ds.ensure_permissions(request.actor, ["view-instance"]) - return await self.render(["patterns.html"], request=request) +class PatternPortfolioView(View): + async def get(self, request, datasette): + await datasette.ensure_permissions(request.actor, ["view-instance"]) + return Response.html( + await datasette.render_template( + "patterns.html", + request=request, + view_name="patterns", + ) + ) class AuthTokenView(BaseView): diff --git a/tests/test_base_view.py b/tests/test_base_view.py new file mode 100644 index 0000000000..2cd4d601ef --- /dev/null +++ b/tests/test_base_view.py @@ -0,0 +1,84 @@ +from datasette.views.base import View +from datasette import Request, Response +from datasette.app import Datasette +import json +import pytest + + +class GetView(View): + async def get(self, request, datasette): + return Response.json( + { + "absolute_url": datasette.absolute_url(request, "/"), + "request_path": request.path, + } + ) + + +class GetAndPostView(GetView): + async def post(self, request, datasette): + return Response.json( + { + "method": request.method, + "absolute_url": datasette.absolute_url(request, "/"), + "request_path": request.path, + } + ) + + +@pytest.mark.asyncio +async def test_get_view(): + v = GetView() + datasette = Datasette() + response = await v(Request.fake("/foo"), datasette) + assert json.loads(response.body) == { + "absolute_url": "http://localhost/", + "request_path": "/foo", + } + # Try a HEAD request + head_response = await v(Request.fake("/foo", method="HEAD"), datasette) + assert head_response.body == "" + assert head_response.status == 200 + # And OPTIONS + options_response = await v(Request.fake("/foo", method="OPTIONS"), datasette) + assert options_response.body == "ok" + assert options_response.status == 200 + assert options_response.headers["allow"] == "HEAD, GET" + # And POST + post_response = await v(Request.fake("/foo", method="POST"), datasette) + assert post_response.body == "Method not allowed" + assert post_response.status == 405 + # And POST with .json extension + post_json_response = await v(Request.fake("/foo.json", method="POST"), datasette) + assert json.loads(post_json_response.body) == { + "ok": False, + "error": "Method not allowed", + } + assert post_json_response.status == 405 + + +@pytest.mark.asyncio +async def test_post_view(): + v = GetAndPostView() + datasette = Datasette() + response = await v(Request.fake("/foo"), datasette) + assert json.loads(response.body) == { + "absolute_url": "http://localhost/", + "request_path": "/foo", + } + # Try a HEAD request + head_response = await v(Request.fake("/foo", method="HEAD"), datasette) + assert head_response.body == "" + assert head_response.status == 200 + # And OPTIONS + options_response = await v(Request.fake("/foo", method="OPTIONS"), datasette) + assert options_response.body == "ok" + assert options_response.status == 200 + assert options_response.headers["allow"] == "HEAD, GET, POST" + # And POST + post_response = await v(Request.fake("/foo", method="POST"), datasette) + assert json.loads(post_response.body) == { + "method": "POST", + "absolute_url": "http://localhost/", + "request_path": "/foo", + } From ba7bc2ab0fcb58a58ebd985fada1b535725046d5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 23 Jun 2023 13:06:35 -0700 Subject: [PATCH 19/28] Better docs for startup() hook --- docs/plugin_hooks.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index a4c9d98fbc..973065292b 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -869,7 +869,9 @@ Examples: `datasette-cors `__, `dat startup(datasette) ------------------ -This hook fires when the Datasette application server first starts up. You can implement a regular function, for example to validate required plugin configuration: +This hook fires when the Datasette application server first starts up. + +Here is an example that validates required plugin configuration. The server will fail to start and show an error if the validation check fails: .. code-block:: python @@ -880,7 +882,7 @@ This hook fires when the Datasette application server first starts up. You can i "required-setting" in config ), "my-plugin requires setting required-setting" -Or you can return an async function which will be awaited on startup. Use this option if you need to make any database queries: +You can also return an async function, which will be awaited on startup. Use this option if you need to execute any database queries, for example this function which creates the ``my_table`` database table if it does not yet exist: .. code-block:: python From a88cd45ae55db50d433a1123ed50681f77e06c04 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 29 Jun 2023 07:31:54 -0700 Subject: [PATCH 20/28] Bump blacken-docs from 1.13.0 to 1.14.0 (#2083) Bumps [blacken-docs](https://github.com/asottile/blacken-docs) from 1.13.0 to 1.14.0. - [Changelog](https://github.com/adamchainz/blacken-docs/blob/main/CHANGELOG.rst) - [Commits](https://github.com/asottile/blacken-docs/compare/1.13.0...1.14.0) --- updated-dependencies: - dependency-name: blacken-docs dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0eedadb2ff..678ec5be7c 100644 --- a/setup.py +++ b/setup.py @@ -82,7 +82,7 @@ def get_version(): "pytest-asyncio>=0.17", "beautifulsoup4>=4.8.1", "black==23.3.0", - "blacken-docs==1.13.0", + "blacken-docs==1.14.0", "pytest-timeout>=1.4.2", "trustme>=0.7", "cogapp>=3.3.0", From 71491551e077571065553d19bef4f7cc14a0a87c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Jun 2023 07:43:01 -0700 Subject: [PATCH 21/28] codespell>=2.5.5, also spellcheck README - refs #2089 --- .github/workflows/spellcheck.yml | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index a2621ecc26..6bf72f9d2f 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -26,5 +26,6 @@ jobs: pip install -e '.[docs]' - name: Check spelling run: | + codespell README.md --ignore-words docs/codespell-ignore-words.txt codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt diff --git a/setup.py b/setup.py index 678ec5be7c..b8da61d0fb 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,7 @@ def get_version(): "Sphinx==6.1.3", "furo==2023.3.27", "sphinx-autobuild", - "codespell", + "codespell>=2.2.5", "blacken-docs", "sphinx-copybutton", ], From c0c764727fecedf1a53b0e3389ab25fc7379a579 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Jun 2023 07:44:10 -0700 Subject: [PATCH 22/28] Justfile I use for local development Now with codespell, refs #2089 --- Justfile | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 Justfile diff --git a/Justfile b/Justfile new file mode 100644 index 0000000000..e595a2660f --- /dev/null +++ b/Justfile @@ -0,0 +1,41 @@ +export DATASETTE_SECRET := "not_a_secret" + +# Run tests and linters +@default: test lint + +# Setup project +@init: + pipenv run pip install -e '.[test,docs]' + +# Run pytest with supplied options +@test *options: + pipenv run pytest {{options}} + +@codespell: + pipenv run codespell README.md --ignore-words docs/codespell-ignore-words.txt + pipenv run codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt + pipenv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt + +# Run linters: black, flake8, mypy, cog +@lint: + pipenv run black . --check + pipenv run flake8 + pipenv run cog --check README.md docs/*.rst + +# Rebuild docs with cog +@cog: + pipenv run cog -r README.md docs/*.rst + +# Serve live docs on localhost:8000 +@docs: cog + pipenv run blacken-docs -l 60 docs/*.rst + cd docs && pipenv run make livehtml + +# Apply Black +@black: + pipenv run black . + +@serve: + pipenv run sqlite-utils create-database data.db + pipenv run sqlite-utils create-table data.db docs id integer title text --pk id --ignore + pipenv run python -m datasette data.db --root --reload From 737a1a7fd2f31a4948dd386ae603e16fcea65617 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Jun 2023 07:46:22 -0700 Subject: [PATCH 23/28] Fixed spelling error, refs #2089 Also ensure codespell runs as part of just lint --- Justfile | 2 +- docs/metadata.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Justfile b/Justfile index e595a2660f..d349ec51c7 100644 --- a/Justfile +++ b/Justfile @@ -17,7 +17,7 @@ export DATASETTE_SECRET := "not_a_secret" pipenv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt # Run linters: black, flake8, mypy, cog -@lint: +@lint: codespell pipenv run black . --check pipenv run flake8 pipenv run cog --check README.md docs/*.rst diff --git a/docs/metadata.rst b/docs/metadata.rst index 35b8aede4c..5932cc3a26 100644 --- a/docs/metadata.rst +++ b/docs/metadata.rst @@ -189,7 +189,7 @@ Or use ``"sort_desc"`` to sort in descending order: Setting a custom page size -------------------------- -Datasette defaults to displaing 100 rows per page, for both tables and views. You can change this default page size on a per-table or per-view basis using the ``"size"`` key in ``metadata.json``: +Datasette defaults to displaying 100 rows per page, for both tables and views. You can change this default page size on a per-table or per-view basis using the ``"size"`` key in ``metadata.json``: .. code-block:: json From b5647ebd53a2c51160c0a1cc440fd5365836b4e9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Jun 2023 08:05:24 -0700 Subject: [PATCH 24/28] Fix all E741 Ambiguous variable name warnings, refs #2090 --- tests/test_plugins.py | 8 +++++--- tests/test_table_html.py | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 71b710f95e..6971bbf739 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -115,7 +115,9 @@ async def test_hook_extra_css_urls(ds_client, path, expected_decoded_object): assert response.status_code == 200 links = Soup(response.text, "html.parser").findAll("link") special_href = [ - l for l in links if l.attrs["href"].endswith("/extra-css-urls-demo.css") + link + for link in links + if link.attrs["href"].endswith("/extra-css-urls-demo.css") ][0]["href"] # This link has a base64-encoded JSON blob in it encoded = special_href.split("/")[3] @@ -543,7 +545,7 @@ async def test_hook_register_output_renderer_can_render(ds_client): .find("p", {"class": "export-links"}) .findAll("a") ) - actual = [l["href"] for l in links] + actual = [link["href"] for link in links] # Should not be present because we sent ?_no_can_render=1 assert "/fixtures/facetable.testall?_labels=on" not in actual # Check that it was passed the values we expected @@ -940,7 +942,7 @@ def get_table_actions_links(html): response_2 = await ds_client.get(f"/fixtures/{table_or_view}?_bot=1&_hello=BOB") assert sorted( - get_table_actions_links(response_2.text), key=lambda l: l["label"] + get_table_actions_links(response_2.text), key=lambda link: link["label"] ) == [ {"label": "Database: fixtures", "href": "/"}, {"label": "From async BOB", "href": "/"}, diff --git a/tests/test_table_html.py b/tests/test_table_html.py index e1886dab5a..c4c7878c88 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -481,7 +481,7 @@ async def test_table_csv_json_export_interface(ds_client): .find("p", {"class": "export-links"}) .findAll("a") ) - actual = [l["href"] for l in links] + actual = [link["href"] for link in links] expected = [ "/fixtures/simple_primary_key.json?id__gt=2", "/fixtures/simple_primary_key.testall?id__gt=2", @@ -521,7 +521,7 @@ async def test_csv_json_export_links_include_labels_if_foreign_keys(ds_client): .find("p", {"class": "export-links"}) .findAll("a") ) - actual = [l["href"] for l in links] + actual = [link["href"] for link in links] expected = [ "/fixtures/facetable.json?_labels=on", "/fixtures/facetable.testall?_labels=on", From d7aa14b17ff47b136dca67cdbbea7e99ea8b6b20 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Jun 2023 08:24:09 -0700 Subject: [PATCH 25/28] Homepage test now just asserts isinstance(x, int) - closes #2092 --- tests/test_api.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index e7d8d849fe..bdee3b98db 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -32,14 +32,12 @@ async def test_homepage(ds_client): assert data.keys() == {"fixtures": 0}.keys() d = data["fixtures"] assert d["name"] == "fixtures" - assert d["tables_count"] == 24 - assert len(d["tables_and_views_truncated"]) == 5 + assert isinstance(d["tables_count"], int) + assert isinstance(len(d["tables_and_views_truncated"]), int) assert d["tables_and_views_more"] is True - # 4 hidden FTS tables + no_primary_key (hidden in metadata) - assert d["hidden_tables_count"] == 6 - # 201 in no_primary_key, plus 6 in other hidden tables: - assert d["hidden_table_rows_sum"] == 207, data - assert d["views_count"] == 4 + assert isinstance(d["hidden_tables_count"], int) + assert isinstance(d["hidden_table_rows_sum"], int) + assert isinstance(d["views_count"], int) @pytest.mark.asyncio From e8ac498e24cc046d4e13fa15f37c0d5fce2d5e9b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Apr 2023 20:47:03 -0700 Subject: [PATCH 26/28] Work in progress on query view, refs #2049 --- datasette/views/database.py | 404 ++++++++++-------------------------- setup.py | 2 +- 2 files changed, 108 insertions(+), 298 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 455ebd1fcc..a2f6c0211a 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -887,6 +887,105 @@ async def query_view( _size=None, named_parameters=None, write=False, +): + print("query_view") + db = await datasette.resolve_database(request) + database = db.name + # TODO: Why do I do this? Is it to eliminate multi-args? + # It's going to break ?_extra=...&_extra=... + params = {key: request.args.get(key) for key in request.args} + sql = "" + if "sql" in params: + sql = params.pop("sql") + + # TODO: Behave differently for canned query here: + await datasette.ensure_permissions(request.actor, [("execute-sql", database)]) + + _shape = None + if "_shape" in params: + _shape = params.pop("_shape") + + # ?_shape=arrays - "rows" is the default option, shown above + # ?_shape=objects - "rows" is a list of JSON key/value objects + # ?_shape=array - an JSON array of objects + # ?_shape=array&_nl=on - a newline-separated list of JSON objects + # ?_shape=arrayfirst - a flat JSON array containing just the first value from each row + # ?_shape=object - a JSON object keyed using the primary keys of the rows + async def _results(_sql, _params): + # Returns (results, error (can be None)) + try: + return await db.execute(_sql, _params, truncate=True), None + except Exception as e: + return None, e + + async def shape_arrays(_results): + results, error = _results + if error: + return {"ok": False, "error": str(error)} + return { + "ok": True, + "rows": [list(r) for r in results.rows], + "truncated": results.truncated, + } + + async def shape_objects(_results): + results, error = _results + if error: + return {"ok": False, "error": str(error)} + return { + "ok": True, + "rows": [dict(r) for r in results.rows], + "truncated": results.truncated, + } + + async def shape_array(_results): + results, error = _results + if error: + return {"ok": False, "error": str(error)} + return [dict(r) for r in results.rows] + + shape_fn = { + "arrays": shape_arrays, + "objects": shape_objects, + "array": shape_array, + # "arrayfirst": shape_arrayfirst, + # "object": shape_object, + }[_shape or "objects"] + + registry = Registry.from_dict( + { + "_results": _results, + "_shape": shape_fn, + }, + parallel=False, + ) + + results = await registry.resolve_multi( + ["_shape"], + results={ + "_sql": sql, + "_params": params, + }, + ) + + # If "shape" does not include "rows" we return that as the response + # because it's likely [{...}] or similar, with no room to attach extras + if "rows" not in results["_shape"]: + return Response.json(results["_shape"]) + + output = results["_shape"] + # Include the extras: + output.update(dict((k, v) for k, v in results.items() if not k.startswith("_"))) + return Response.json(output) + + +async def database_view_impl( + request, + datasette, + canned_query=None, + _size=None, + named_parameters=None, + write=False, ): db = await datasette.resolve_database(request) @@ -1031,6 +1130,14 @@ async def query_view_data( database = db.name # TODO: Why do I do this? Is it to eliminate multi-args? # It's going to break ?_extra=...&_extra=... + + if request.args.get("sql", "").strip(): + return await query_view( + request, datasette, canned_query, _size, named_parameters, write + ) + + # Index page shows the tables/views/canned queries for this database + params = {key: request.args.get(key) for key in request.args} sql = "" if "sql" in params: @@ -1119,305 +1226,8 @@ async def shape_arrayfirst(_results): output = results["_shape"] output.update(dict((k, v) for k, v in results.items() if not k.startswith("_"))) - return output - # registry = Registry( - # extra_count, - # extra_facet_results, - # extra_facets_timed_out, - # extra_suggested_facets, - # facet_instances, - # extra_human_description_en, - # extra_next_url, - # extra_columns, - # extra_primary_keys, - # run_display_columns_and_rows, - # extra_display_columns, - # extra_display_rows, - # extra_debug, - # extra_request, - # extra_query, - # extra_metadata, - # extra_extras, - # extra_database, - # extra_table, - # extra_database_color, - # extra_table_actions, - # extra_filters, - # extra_renderers, - # extra_custom_table_templates, - # extra_sorted_facet_results, - # extra_table_definition, - # extra_view_definition, - # extra_is_view, - # extra_private, - # extra_expandable_columns, - # extra_form_hidden_args, - # ) - - results = await registry.resolve_multi( - ["extra_{}".format(extra) for extra in extras] - ) - data = { - "ok": True, - "next": next_value and str(next_value) or None, - } - data.update( - { - key.replace("extra_", ""): value - for key, value in results.items() - if key.startswith("extra_") and key.replace("extra_", "") in extras - } - ) - raw_sqlite_rows = rows[:page_size] - data["rows"] = [dict(r) for r in raw_sqlite_rows] - - private = False - if canned_query: - # Respect canned query permissions - visible, private = await datasette.check_visibility( - request.actor, - permissions=[ - ("view-query", (database, canned_query)), - ("view-database", database), - "view-instance", - ], - ) - if not visible: - raise Forbidden("You do not have permission to view this query") - - else: - await datasette.ensure_permissions(request.actor, [("execute-sql", database)]) - - # If there's no sql, show the database index page - if not sql: - return await database_index_view(request, datasette, db) - - validate_sql_select(sql) - - # Extract any :named parameters - named_parameters = named_parameters or await derive_named_parameters(db, sql) - named_parameter_values = { - named_parameter: params.get(named_parameter) or "" - for named_parameter in named_parameters - if not named_parameter.startswith("_") - } - - # Set to blank string if missing from params - for named_parameter in named_parameters: - if named_parameter not in params and not named_parameter.startswith("_"): - params[named_parameter] = "" - - extra_args = {} - if params.get("_timelimit"): - extra_args["custom_time_limit"] = int(params["_timelimit"]) - if _size: - extra_args["page_size"] = _size - - templates = [f"query-{to_css_class(database)}.html", "query.html"] - if canned_query: - templates.insert( - 0, - f"query-{to_css_class(database)}-{to_css_class(canned_query)}.html", - ) - - query_error = None - - # Execute query - as write or as read - if write: - raise NotImplementedError("Write queries not yet implemented") - # if request.method == "POST": - # # If database is immutable, return an error - # if not db.is_mutable: - # raise Forbidden("Database is immutable") - # body = await request.post_body() - # body = body.decode("utf-8").strip() - # if body.startswith("{") and body.endswith("}"): - # params = json.loads(body) - # # But we want key=value strings - # for key, value in params.items(): - # params[key] = str(value) - # else: - # params = dict(parse_qsl(body, keep_blank_values=True)) - # # Should we return JSON? - # should_return_json = ( - # request.headers.get("accept") == "application/json" - # or request.args.get("_json") - # or params.get("_json") - # ) - # if canned_query: - # params_for_query = MagicParameters(params, request, self.ds) - # else: - # params_for_query = params - # ok = None - # try: - # cursor = await self.ds.databases[database].execute_write( - # sql, params_for_query - # ) - # message = metadata.get( - # "on_success_message" - # ) or "Query executed, {} row{} affected".format( - # cursor.rowcount, "" if cursor.rowcount == 1 else "s" - # ) - # message_type = self.ds.INFO - # redirect_url = metadata.get("on_success_redirect") - # ok = True - # except Exception as e: - # message = metadata.get("on_error_message") or str(e) - # message_type = self.ds.ERROR - # redirect_url = metadata.get("on_error_redirect") - # ok = False - # if should_return_json: - # return Response.json( - # { - # "ok": ok, - # "message": message, - # "redirect": redirect_url, - # } - # ) - # else: - # self.ds.add_message(request, message, message_type) - # return self.redirect(request, redirect_url or request.path) - # else: - - # async def extra_template(): - # return { - # "request": request, - # "db_is_immutable": not db.is_mutable, - # "path_with_added_args": path_with_added_args, - # "path_with_removed_args": path_with_removed_args, - # "named_parameter_values": named_parameter_values, - # "canned_query": canned_query, - # "success_message": request.args.get("_success") or "", - # "canned_write": True, - # } - - # return ( - # { - # "database": database, - # "rows": [], - # "truncated": False, - # "columns": [], - # "query": {"sql": sql, "params": params}, - # "private": private, - # }, - # extra_template, - # templates, - # ) - - # Not a write - rows = [] - if canned_query: - params_for_query = MagicParameters(params, request, datasette) - else: - params_for_query = params - try: - results = await datasette.execute( - database, sql, params_for_query, truncate=True, **extra_args - ) - columns = [r[0] for r in results.description] - rows = list(results.rows) - except sqlite3.DatabaseError as e: - query_error = e - results = None - columns = [] - - allow_execute_sql = await datasette.permission_allowed( - request.actor, "execute-sql", database - ) - - format_ = request.url_vars.get("format") or "html" - - if format_ == "csv": - raise NotImplementedError("CSV format not yet implemented") - elif format_ in datasette.renderers.keys(): - # Dispatch request to the correct output format renderer - # (CSV is not handled here due to streaming) - result = call_with_supported_arguments( - datasette.renderers[format_][0], - datasette=datasette, - columns=columns, - rows=rows, - sql=sql, - query_name=None, - database=db.name, - table=None, - request=request, - view_name="table", # TODO: should this be "query"? - # These will be deprecated in Datasette 1.0: - args=request.args, - data={ - "rows": rows, - }, # TODO what should this be? - ) - result = await await_me_maybe(result) - if result is None: - raise NotFound("No data") - if isinstance(result, dict): - r = Response( - body=result.get("body"), - status=result.get("status_code") or 200, - content_type=result.get("content_type", "text/plain"), - headers=result.get("headers"), - ) - elif isinstance(result, Response): - r = result - # if status_code is not None: - # # Over-ride the status code - # r.status = status_code - else: - assert False, f"{result} should be dict or Response" - elif format_ == "html": - headers = {} - templates = [f"query-{to_css_class(database)}.html", "query.html"] - template = datasette.jinja_env.select_template(templates) - alternate_url_json = datasette.absolute_url( - request, - datasette.urls.path(path_with_format(request=request, format="json")), - ) - headers.update( - { - "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( - alternate_url_json - ) - } - ) - r = Response.html( - await datasette.render_template( - template, - dict( - data, - append_querystring=append_querystring, - path_with_replaced_args=path_with_replaced_args, - fix_path=datasette.urls.path, - settings=datasette.settings_dict(), - # TODO: review up all of these hacks: - alternate_url_json=alternate_url_json, - datasette_allow_facet=( - "true" if datasette.setting("allow_facet") else "false" - ), - is_sortable=any(c["sortable"] for c in data["display_columns"]), - allow_execute_sql=await datasette.permission_allowed( - request.actor, "execute-sql", resolved.db.name - ), - query_ms=1.2, - select_templates=[ - f"{'*' if template_name == template.name else ''}{template_name}" - for template_name in templates - ], - ), - request=request, - view_name="table", - ), - headers=headers, - ) - else: - assert False, "Invalid format: {}".format(format_) - # if next_url: - # r.headers["link"] = f'<{next_url}>; rel="next"' - return r - async def database_view_impl( request, diff --git a/setup.py b/setup.py index b8da61d0fb..e1856f27a4 100644 --- a/setup.py +++ b/setup.py @@ -58,9 +58,9 @@ def get_version(): "mergedeep>=1.1.1", "itsdangerous>=1.1", "sqlite-utils>=3.30", - "asyncinject>=0.6", "setuptools", "pip", + "asyncinject>=0.6", ], entry_points=""" [console_scripts] From e17a1373b32f2c5c04e59a875157cc60accb02da Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Apr 2023 20:36:10 -0700 Subject: [PATCH 27/28] Add setuptools to dependencies Refs #2065 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index e1856f27a4..a185cb9499 100644 --- a/setup.py +++ b/setup.py @@ -61,6 +61,7 @@ def get_version(): "setuptools", "pip", "asyncinject>=0.6", + "setuptools", ], entry_points=""" [console_scripts] From ee24ea94525ace221f1b4d141d01cf56410c2c6d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Apr 2023 22:07:35 -0700 Subject: [PATCH 28/28] Add pip as a dependency too, for Rye - refs #2065 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index a185cb9499..f52bd44054 100644 --- a/setup.py +++ b/setup.py @@ -62,6 +62,7 @@ def get_version(): "pip", "asyncinject>=0.6", "setuptools", + "pip", ], entry_points=""" [console_scripts]