diff --git a/datasette/app.py b/datasette/app.py index 2df6e4e8e1..93792377a9 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1041,12 +1041,12 @@ def _prepare_connection(self, conn, database): for db_name, db in self.databases.items(): if count >= SQLITE_LIMIT_ATTACHED or db.is_memory: continue - sql = 'ATTACH DATABASE "file:{path}?{qs}" AS [{name}];'.format( + sql = "ATTACH DATABASE ? AS {};".format(escape_sqlite(db_name)) + location = "file:{path}?{qs}".format( path=db.path, qs="mode=ro" if db.is_mutable else "immutable=1", - name=db_name, ) - conn.execute(sql) + conn.execute(sql, [location]) count += 1 def add_message(self, request, message, type=INFO): diff --git a/datasette/database.py b/datasette/database.py index fcf69c7f72..071419835c 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -628,8 +628,8 @@ async def hidden_table_names(self): ] + [ r[0] for r in (await self.execute(""" select name from sqlite_master - where name like "idx_%" - and type = "table" + where name like 'idx_%' + and type = 'table' """)).rows ] diff --git a/datasette/facets.py b/datasette/facets.py index bc4b69049a..1f7c6c9fd8 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -479,7 +479,7 @@ async def suggest(self): suggested_facet_sql = """ select date({column}) from ( select * from ({sql}) limit 100 - ) where {column} glob "????-??-*" + ) where {column} glob '????-??-*' """.format(column=escape_sqlite(column), sql=self.sql) try: results = await self.ds.execute( diff --git a/datasette/filters.py b/datasette/filters.py index 95cc5f3748..a5fd849b1f 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -352,14 +352,14 @@ class Filters: TemplatedFilter( "isblank", "is blank", - '("{c}" is null or "{c}" = "")', + """("{c}" is null or "{c}" = '')""", "{c} is blank", no_argument=True, ), TemplatedFilter( "notblank", "is not blank", - '("{c}" is not null and "{c}" != "")', + """("{c}" is not null and "{c}" != '')""", "{c} is not blank", no_argument=True, ), @@ -408,11 +408,15 @@ def selections(self): def has_selections(self): return bool(self.pairs) - def build_where_clauses(self, table): + def build_where_clauses(self, table, table_columns=None): sql_bits = [] params = {} i = 0 for column, lookup, value in self.selections(): + if column != "rowid" and table_columns and column not in table_columns: + # Ignore invalid column names, with SQLITE_DQS=0 they don't + # degrade to harmless string literal comparisons + continue filter = self._filters_by_key.get(lookup, None) if filter: sql_bit, param = filter.where_clause(table, column, value, i) diff --git a/datasette/inspect.py b/datasette/inspect.py index 5e681e0368..1270901d04 100644 --- a/datasette/inspect.py +++ b/datasette/inspect.py @@ -29,7 +29,7 @@ def inspect_hash(path): def inspect_views(conn): """List views in a database.""" return [ - v[0] for v in conn.execute('select name from sqlite_master where type = "view"') + v[0] for v in conn.execute("select name from sqlite_master where type = 'view'") ] @@ -38,7 +38,7 @@ def inspect_tables(conn, database_metadata): tables = {} table_names = [ r["name"] - for r in conn.execute('select * from sqlite_master where type="table"') + for r in conn.execute("select * from sqlite_master where type='table'") ] for table in table_names: @@ -90,8 +90,8 @@ def inspect_tables(conn, database_metadata): ] + [ r["name"] for r in conn.execute(""" select name from sqlite_master - where name like "idx_%" - and type = "table" + where name like 'idx_%' + and type = 'table' """) ] diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index c6973d0672..b0063b9324 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -614,7 +614,7 @@ def get_all_foreign_keys(conn): tables = [ r[0] for r in conn.execute( - 'select name from sqlite_master where type="table" order by name' + "select name from sqlite_master where type='table' order by name" ) ] table_to_foreign_keys = {} @@ -651,7 +651,7 @@ def get_all_foreign_keys(conn): def detect_spatialite(conn): rows = conn.execute( - 'select 1 from sqlite_master where tbl_name = "geometry_columns"' + "select 1 from sqlite_master where tbl_name = 'geometry_columns'" ).fetchall() return len(rows) > 0 @@ -673,7 +673,7 @@ def detect_fts_sql(table): sql like '%VIRTUAL TABLE%USING FTS%content="{table}"%' or sql like '%VIRTUAL TABLE%USING FTS%content=[{table}]%' or ( - tbl_name = "{table}" + tbl_name = '{table}' and sql like '%VIRTUAL TABLE%USING FTS%' ) ) diff --git a/datasette/views/table.py b/datasette/views/table.py index 2ee8674323..3a6e67ccf4 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1034,7 +1034,7 @@ async def table_view_data( # Build where clauses from query string arguments filters = Filters(sorted(filter_args)) - where_clauses, params = filters.build_where_clauses(table_name) + where_clauses, params = filters.build_where_clauses(table_name, table_columns) # Execute filters_from_request plugin hooks - including the default # ones that live in datasette/filters.py diff --git a/docs/json_api.rst b/docs/json_api.rst index 91a2bb15f1..9d391b7419 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -235,16 +235,16 @@ You can filter the data returned by the table based on column values using a que Returns rows where the column does not match the value. ``?column__contains=value`` - Rows where the string column contains the specified value (``column like "%value%"`` in SQL). + Rows where the string column contains the specified value (``column like '%value%'`` in SQL). ``?column__notcontains=value`` - Rows where the string column does not contain the specified value (``column not like "%value%"`` in SQL). + Rows where the string column does not contain the specified value (``column not like '%value%'`` in SQL). ``?column__endswith=value`` - Rows where the string column ends with the specified value (``column like "%value"`` in SQL). + Rows where the string column ends with the specified value (``column like '%value'`` in SQL). ``?column__startswith=value`` - Rows where the string column starts with the specified value (``column like "value%"`` in SQL). + Rows where the string column starts with the specified value (``column like 'value%'`` in SQL). ``?column__gt=value`` Rows which are greater than the specified value. @@ -358,8 +358,8 @@ Special table arguments Some examples: - * `facetable?_where=_neighborhood like "%c%"&_where=_city_id=3 `__ - * `facetable?_where=_city_id in (select id from facet_cities where name != "Detroit") `__ + * `facetable?_where=_neighborhood like '%c%'&_where=_city_id=3 `__ + * `facetable?_where=_city_id in (select id from facet_cities where name != 'Detroit') `__ ``?_through={json}`` This can be used to filter rows via a join against another table. diff --git a/tests/fixtures.py b/tests/fixtures.py index 1f6c491dcd..5c0c105528 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -19,6 +19,13 @@ PLUGINS_DIR = str(pathlib.Path(__file__).parent / "plugins") EXPECTED_PLUGINS = [ + { + "name": "disable_double_quoted_strings.py", + "static": False, + "templates": False, + "version": None, + "hooks": ["prepare_connection"], + }, { "name": "messages_output_renderer.py", "static": False, @@ -526,12 +533,12 @@ def generate_sortable_rows(num): INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog', 'panther'); INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel', 'puma'); -INSERT INTO tags VALUES ("canine"); -INSERT INTO tags VALUES ("feline"); +INSERT INTO tags VALUES ('canine'); +INSERT INTO tags VALUES ('feline'); INSERT INTO searchable_tags (searchable_id, tag) VALUES - (1, "feline"), - (2, "canine") + (1, 'feline'), + (2, 'canine') ; CREATE VIRTUAL TABLE "searchable_fts" @@ -585,21 +592,21 @@ def generate_sortable_rows(num): INSERT INTO facetable (created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n) VALUES - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]', 'one', 'n1'), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]', 'two', 'n2'), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'SOMA', '[]', '[]', null, null), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Tenderloin', '[]', '[]', null, null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Bernal Heights', '[]', '[]', null, null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Hayes Valley', '[]', '[]', null, null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Hollywood', '[]', '[]', null, null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Downtown', '[]', '[]', null, null), - ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Los Feliz', '[]', '[]', null, null), - ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Koreatown', '[]', '[]', null, null), - ("2019-01-16 08:00:00", 1, 1, 'MI', 3, 'Downtown', '[]', '[]', null, null), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Greektown', '[]', '[]', null, null), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Corktown', '[]', '[]', null, null), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Mexicantown', '[]', '[]', null, null), - ("2019-01-17 08:00:00", 2, 0, 'MC', 4, 'Arcadia Planitia', '[]', '[]', null, null) + ('2019-01-14 08:00:00', 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]', 'one', 'n1'), + ('2019-01-14 08:00:00', 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]', 'two', 'n2'), + ('2019-01-14 08:00:00', 1, 1, 'CA', 1, 'SOMA', '[]', '[]', null, null), + ('2019-01-14 08:00:00', 1, 1, 'CA', 1, 'Tenderloin', '[]', '[]', null, null), + ('2019-01-15 08:00:00', 1, 1, 'CA', 1, 'Bernal Heights', '[]', '[]', null, null), + ('2019-01-15 08:00:00', 1, 1, 'CA', 1, 'Hayes Valley', '[]', '[]', null, null), + ('2019-01-15 08:00:00', 1, 1, 'CA', 2, 'Hollywood', '[]', '[]', null, null), + ('2019-01-15 08:00:00', 1, 1, 'CA', 2, 'Downtown', '[]', '[]', null, null), + ('2019-01-16 08:00:00', 1, 1, 'CA', 2, 'Los Feliz', '[]', '[]', null, null), + ('2019-01-16 08:00:00', 1, 1, 'CA', 2, 'Koreatown', '[]', '[]', null, null), + ('2019-01-16 08:00:00', 1, 1, 'MI', 3, 'Downtown', '[]', '[]', null, null), + ('2019-01-17 08:00:00', 1, 1, 'MI', 3, 'Greektown', '[]', '[]', null, null), + ('2019-01-17 08:00:00', 1, 1, 'MI', 3, 'Corktown', '[]', '[]', null, null), + ('2019-01-17 08:00:00', 1, 1, 'MI', 3, 'Mexicantown', '[]', '[]', null, null), + ('2019-01-17 08:00:00', 2, 0, 'MC', 4, 'Arcadia Planitia', '[]', '[]', null, null) ; CREATE TABLE binary_data ( @@ -617,19 +624,19 @@ def generate_sortable_rows(num): longitude real ); INSERT INTO roadside_attractions VALUES ( - 1, "The Mystery Spot", "465 Mystery Spot Road, Santa Cruz, CA 95065", "https://www.mysteryspot.com/", + 1, 'The Mystery Spot', '465 Mystery Spot Road, Santa Cruz, CA 95065', 'https://www.mysteryspot.com/', 37.0167, -122.0024 ); INSERT INTO roadside_attractions VALUES ( - 2, "Winchester Mystery House", "525 South Winchester Boulevard, San Jose, CA 95128", "https://winchestermysteryhouse.com/", + 2, 'Winchester Mystery House', '525 South Winchester Boulevard, San Jose, CA 95128', 'https://winchestermysteryhouse.com/', 37.3184, -121.9511 ); INSERT INTO roadside_attractions VALUES ( - 3, "Burlingame Museum of PEZ Memorabilia", "214 California Drive, Burlingame, CA 94010", null, + 3, 'Burlingame Museum of PEZ Memorabilia', '214 California Drive, Burlingame, CA 94010', null, 37.5793, -122.3442 ); INSERT INTO roadside_attractions VALUES ( - 4, "Bigfoot Discovery Museum", "5497 Highway 9, Felton, CA 95018", "https://www.bigfootdiscoveryproject.com/", + 4, 'Bigfoot Discovery Museum', '5497 Highway 9, Felton, CA 95018', 'https://www.bigfootdiscoveryproject.com/', 37.0414, -122.0725 ); @@ -638,10 +645,10 @@ def generate_sortable_rows(num): name text ); INSERT INTO attraction_characteristic VALUES ( - 1, "Museum" + 1, 'Museum' ); INSERT INTO attraction_characteristic VALUES ( - 2, "Paranormal" + 2, 'Paranormal' ); CREATE TABLE roadside_attraction_characteristics ( @@ -693,24 +700,24 @@ def generate_sortable_rows(num): """ + "\n".join( [ - 'INSERT INTO no_primary_key VALUES ({i}, "a{i}", "b{i}", "c{i}");'.format( + "INSERT INTO no_primary_key VALUES ({i}, 'a{i}', 'b{i}', 'c{i}');".format( i=i + 1 ) for i in range(201) ] ) - + '\nINSERT INTO no_primary_key VALUES ("RENDER_CELL_DEMO", "a202", "b202", "c202");\n' + + "\nINSERT INTO no_primary_key VALUES ('RENDER_CELL_DEMO', 'a202', 'b202', 'c202');\n" + "\n".join( [ - 'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");'.format( + "INSERT INTO compound_three_primary_keys VALUES ('{a}', '{b}', '{c}', '{content}');".format( a=a, b=b, c=c, content=content ) for a, b, c, content in generate_compound_rows(1001) ] ) + "\n".join(["""INSERT INTO sortable VALUES ( - "{pk1}", "{pk2}", "{content}", {sortable}, - {sortable_with_nulls}, {sortable_with_nulls_2}, "{text}"); + '{pk1}', '{pk2}', '{content}', {sortable}, + {sortable_with_nulls}, {sortable_with_nulls_2}, '{text}'); """.format(**row).replace("None", "null") for row in generate_sortable_rows(201)]) ) TABLE_PARAMETERIZED_SQL = [ diff --git a/tests/plugins/disable_double_quoted_strings.py b/tests/plugins/disable_double_quoted_strings.py new file mode 100644 index 0000000000..4c4675a965 --- /dev/null +++ b/tests/plugins/disable_double_quoted_strings.py @@ -0,0 +1,10 @@ +from datasette import hookimpl +from datasette.utils.sqlite import sqlite3 + + +@hookimpl +def prepare_connection(conn): + if hasattr(conn, "setconfig") and sqlite3.sqlite_version_info >= (3, 29): + # Available only since Python 3.12 and SQLite 3.29.0 + conn.setconfig(sqlite3.SQLITE_DBCONFIG_DQS_DDL, False) + conn.setconfig(sqlite3.SQLITE_DBCONFIG_DQS_DML, False) diff --git a/tests/test_api.py b/tests/test_api.py index 95958a7291..1d7a3c3c4f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -634,7 +634,7 @@ async def test_settings_json(ds_client): ) async def test_json_columns(ds_client, extra_args, expected): sql = """ - select 1 as intval, "s" as strval, 0.5 as floatval, + select 1 as intval, 's' as strval, 0.5 as floatval, '{"foo": "bar"}' as jsonval """ path = "/fixtures/-/query.json?" + urllib.parse.urlencode( diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 5e3459cdd5..3bb69d3a91 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -89,7 +89,7 @@ def run(conn): # Table should exist assert ( conn.execute( - 'select count(*) from sqlite_master where name = "foo"' + "select count(*) from sqlite_master where name = 'foo'" ).fetchone()[0] == 1 )