Skip to content

Commit 2face11

Browse files
mugikhanmugikhan
authored
Merge pull request #125 from powersync-ja/feat/full-text-search
feat: Full text search
2 parents fec4c90 + 5e129ef commit 2face11

File tree

13 files changed

+10123
-8148
lines changed

13 files changed

+10123
-8148
lines changed

‎demos/example-webpack/src/index.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,4 +58,4 @@ const openDatabase = async () => {
5858

5959
document.addEventListener('DOMContentLoaded', (event) => {
6060
openDatabase();
61-
});
61+
});

‎demos/react-supabase-todolist/package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,10 @@
1111
"dependencies": {
1212
"@journeyapps/powersync-react": "workspace:*",
1313
"@journeyapps/powersync-sdk-web": "workspace:*",
14-
"@journeyapps/wa-sqlite": "~0.1.1",
14+
"@journeyapps/wa-sqlite": "~0.2.0",
1515
"@mui/material": "^5.15.12",
1616
"@mui/x-data-grid": "^6.19.6",
17+
"@mui/icons-material": "^5.15.12",
1718
"@supabase/supabase-js": "^2.39.7",
1819
"js-logger": "^1.6.1",
1920
"lodash": "^4.17.21",

‎demos/react-supabase-todolist/src/app/utils/fts_helpers.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
import { db } from '@/components/providers/SystemProvider';
2+
3+
/**
4+
* adding * to the end of the search term will match any word that starts with the search term
5+
* e.g. searching bl will match blue, black, etc.
6+
* consult FTS5 Full-text Query Syntax documentation for more options
7+
* @param searchTerm
8+
* @returns a modified search term with options.
9+
*/
10+
function createSearchTermWithOptions(searchTerm: string): string {
11+
const searchTermWithOptions: string = `${searchTerm}*`;
12+
return searchTermWithOptions;
13+
}
14+
15+
/**
16+
* Search the FTS table for the given searchTerm
17+
* @param searchTerm
18+
* @param tableName
19+
* @returns results from the FTS table
20+
*/
21+
export async function searchTable(searchTerm: string, tableName: string): Promise<any[]> {
22+
const searchTermWithOptions = createSearchTermWithOptions(searchTerm);
23+
return await db.getAll(`SELECT * FROM fts_${tableName} WHERE fts_${tableName} MATCH ? ORDER BY rank`, [
24+
searchTermWithOptions
25+
]);
26+
}
27+
28+
//Used to display the search results in the autocomplete text field
29+
export class SearchResult {
30+
id: string;
31+
todoName: string | null;
32+
listName: string;
33+
34+
constructor(id: string, listName: string, todoName: string | null = null) {
35+
this.id = id;
36+
this.listName = listName;
37+
this.todoName = todoName;
38+
}
39+
}

‎demos/react-supabase-todolist/src/app/utils/fts_setup.ts‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
import { AppSchema } from '../../library/powersync/AppSchema';
2+
import { Table } from '@journeyapps/powersync-sdk-web';
3+
import { db } from '@/components/providers/SystemProvider';
4+
import { ExtractType, generateJsonExtracts } from './helpers';
5+
6+
/**
7+
* Create a Full Text Search table for the given table and columns
8+
* with an option to use a different tokenizer otherwise it defaults
9+
* to unicode61. It also creates the triggers that keep the FTS table
10+
* and the PowerSync table in sync.
11+
* @param tableName
12+
* @param columns
13+
* @param tokenizationMethod
14+
*/
15+
async function createFtsTable(tableName: string, columns: string[], tokenizationMethod = 'unicode61'): Promise<void> {
16+
const internalName = (AppSchema.tables as Table[]).find((table) => table.name === tableName)?.internalName;
17+
const stringColumns = columns.join(', ');
18+
19+
return await db.writeTransaction(async (tx) => {
20+
// Add FTS table
21+
await tx.execute(`
22+
CREATE VIRTUAL TABLE IF NOT EXISTS fts_${tableName}
23+
USING fts5(id UNINDEXED, ${stringColumns}, tokenize='${tokenizationMethod}');
24+
`);
25+
// Copy over records already in table
26+
await tx.execute(`
27+
INSERT OR REPLACE INTO fts_${tableName}(rowid, id, ${stringColumns})
28+
SELECT rowid, id, ${generateJsonExtracts(ExtractType.columnOnly, 'data', columns)} FROM ${internalName};
29+
`);
30+
// Add INSERT, UPDATE and DELETE and triggers to keep fts table in sync with table
31+
await tx.execute(`
32+
CREATE TRIGGER IF NOT EXISTS fts_insert_trigger_${tableName} AFTER INSERT ON ${internalName}
33+
BEGIN
34+
INSERT INTO fts_${tableName}(rowid, id, ${stringColumns})
35+
VALUES (
36+
NEW.rowid,
37+
NEW.id,
38+
${generateJsonExtracts(ExtractType.columnOnly, 'NEW.data', columns)}
39+
);
40+
END;
41+
`);
42+
await tx.execute(`
43+
CREATE TRIGGER IF NOT EXISTS fts_update_trigger_${tableName} AFTER UPDATE ON ${internalName} BEGIN
44+
UPDATE fts_${tableName}
45+
SET ${generateJsonExtracts(ExtractType.columnInOperation, 'NEW.data', columns)}
46+
WHERE rowid = NEW.rowid;
47+
END;
48+
`);
49+
await tx.execute(`
50+
CREATE TRIGGER IF NOT EXISTS fts_delete_trigger_${tableName} AFTER DELETE ON ${internalName} BEGIN
51+
DELETE FROM fts_${tableName} WHERE rowid = OLD.rowid;
52+
END;
53+
`);
54+
});
55+
}
56+
57+
/**
58+
* This is where you can add more methods to generate FTS tables in this demo
59+
* that correspond to the tables in your schema and populate them
60+
* with the data you would like to search on
61+
*/
62+
export async function configureFts(): Promise<void> {
63+
await createFtsTable('lists', ['name'], 'porter unicode61');
64+
await createFtsTable('todos', ['description', 'list_id']);
65+
}

‎demos/react-supabase-todolist/src/app/utils/helpers.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
type ExtractGenerator = (jsonColumnName: string, columnName: string) => string;
2+
3+
export enum ExtractType {
4+
columnOnly,
5+
columnInOperation
6+
}
7+
8+
type ExtractGeneratorMap = Map<ExtractType, ExtractGenerator>;
9+
10+
function _createExtract(jsonColumnName: string, columnName: string): string {
11+
return `json_extract(${jsonColumnName}, '$.${columnName}')`;
12+
}
13+
14+
const extractGeneratorsMap: ExtractGeneratorMap = new Map<ExtractType, ExtractGenerator>([
15+
[ExtractType.columnOnly, (jsonColumnName: string, columnName: string) => _createExtract(jsonColumnName, columnName)],
16+
[
17+
ExtractType.columnInOperation,
18+
(jsonColumnName: string, columnName: string) => {
19+
let extract = _createExtract(jsonColumnName, columnName);
20+
return `${columnName} = ${extract}`;
21+
}
22+
]
23+
]);
24+
25+
export const generateJsonExtracts = (type: ExtractType, jsonColumnName: string, columns: string[]): string => {
26+
const generator = extractGeneratorsMap.get(type);
27+
if (generator == null) {
28+
throw new Error('Unexpected null generator for key: $type');
29+
}
30+
31+
if (columns.length == 1) {
32+
return generator(jsonColumnName, columns[0]);
33+
}
34+
35+
return columns.map((column) => generator(jsonColumnName, column)).join(', ');
36+
};

‎demos/react-supabase-todolist/src/app/views/todo-lists/page.tsx‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import { NavigationPage } from '@/components/navigation/NavigationPage';
1717
import { useSupabase } from '@/components/providers/SystemProvider';
1818
import { TodoListsWidget } from '@/components/widgets/TodoListsWidget';
1919
import { LISTS_TABLE } from '@/library/powersync/AppSchema';
20+
import { SearchBarWidget } from '@/components/widgets/SearchBarWidget';
2021

2122
export default function TodoListsPage() {
2223
const powerSync = usePowerSync();
@@ -50,6 +51,7 @@ export default function TodoListsPage() {
5051
<AddIcon />
5152
</S.FloatingActionButton>
5253
<Box>
54+
<SearchBarWidget />
5355
<TodoListsWidget />
5456
</Box>
5557
{/* TODO use a dialog service in future, this is just a simple example app */}

‎demos/react-supabase-todolist/src/components/providers/SystemProvider.tsx‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ import { CircularProgress } from '@mui/material';
77
import Logger from 'js-logger';
88
import React, { Suspense } from 'react';
99

10+
import { configureFts } from '../../app/utils/fts_setup';
11+
1012
const SupabaseContext = React.createContext<SupabaseConnector | null>(null);
1113
export const useSupabase = () => React.useContext(SupabaseContext);
1214

@@ -23,7 +25,6 @@ export const SystemProvider = ({ children }: { children: React.ReactNode }) => {
2325
// Linting thinks this is a hook due to it's name
2426
Logger.useDefaults(); // eslint-disable-line
2527
Logger.setLevel(Logger.DEBUG);
26-
2728
// For console testing purposes
2829
(window as any)._powersync = powerSync;
2930

@@ -37,6 +38,8 @@ export const SystemProvider = ({ children }: { children: React.ReactNode }) => {
3738

3839
connector.init();
3940

41+
configureFts();
42+
4043
return () => l?.();
4144
}, [powerSync, connector]);
4245

‎demos/react-supabase-todolist/src/components/widgets/SearchBarWidget.tsx‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
import { SearchResult, searchTable } from '@/app/utils/fts_helpers';
2+
import { Autocomplete, Box, Card, CardContent, FormControl, Paper, TextField, Typography } from '@mui/material';
3+
import React from 'react';
4+
import { useNavigate } from 'react-router-dom';
5+
import { TODO_LISTS_ROUTE } from '@/app/router';
6+
import { usePowerSync, usePowerSyncQuery } from '@journeyapps/powersync-react';
7+
import { LISTS_TABLE, ListRecord } from '@/library/powersync/AppSchema';
8+
import { todo } from 'node:test';
9+
10+
// This is a simple search bar widget that allows users to search for lists and todo items
11+
export const SearchBarWidget: React.FC<any> = (props) => {
12+
const [searchResults, setSearchResults] = React.useState<SearchResult[]>([]);
13+
const [value, setValue] = React.useState<SearchResult | null>(null);
14+
15+
const navigate = useNavigate();
16+
const powersync = usePowerSync();
17+
18+
const handleInputChange = async (value: string) => {
19+
if (value.length !== 0) {
20+
let listsSearchResults: any[] = [];
21+
let todoItemsSearchResults = await searchTable(value, 'todos');
22+
for (let i = 0; i < todoItemsSearchResults.length; i++) {
23+
let res = await powersync.get<ListRecord>(`SELECT * FROM ${LISTS_TABLE} WHERE id = ?`, [
24+
todoItemsSearchResults[i]['list_id']
25+
]);
26+
todoItemsSearchResults[i]['list_name'] = res.name;
27+
}
28+
if (!todoItemsSearchResults.length) {
29+
listsSearchResults = await searchTable(value, 'lists');
30+
}
31+
let formattedListResults: SearchResult[] = listsSearchResults.map(
32+
(result) => new SearchResult(result['id'], result['name'])
33+
);
34+
let formattedTodoItemsResults: SearchResult[] = todoItemsSearchResults.map((result) => {
35+
return new SearchResult(result['list_id'], result['list_name'] ?? '', result['description']);
36+
});
37+
setSearchResults([...formattedTodoItemsResults, ...formattedListResults]);
38+
}
39+
};
40+
41+
return (
42+
<div>
43+
<FormControl sx={{ my: 1, display: 'flex' }}>
44+
<Autocomplete
45+
freeSolo
46+
id="autocomplete-search"
47+
options={searchResults}
48+
value={value?.id}
49+
getOptionLabel={(option) => {
50+
if (option instanceof SearchResult) {
51+
return option.todoName ?? option.listName;
52+
}
53+
return option;
54+
}}
55+
renderOption={(props, option) => (
56+
<Box component="li" {...props}>
57+
<Card variant="outlined" sx={{ display: 'flex', width: '100%' }}>
58+
<CardContent>
59+
{option.listName && (
60+
<Typography sx={{ fontSize: 18 }} color="text.primary" gutterBottom>
61+
{option.listName}
62+
</Typography>
63+
)}
64+
{option.todoName && (
65+
<Typography sx={{ fontSize: 14 }} color="text.secondary">
66+
{'\u2022'} {option.todoName}
67+
</Typography>
68+
)}
69+
</CardContent>
70+
</Card>
71+
</Box>
72+
)}
73+
filterOptions={(x) => x}
74+
onInputChange={(event, newInputValue, reason) => {
75+
if (reason === 'clear') {
76+
setValue(null);
77+
setSearchResults([]);
78+
return;
79+
}
80+
handleInputChange(newInputValue);
81+
}}
82+
onChange={(event, newValue, reason) => {
83+
if (reason === 'selectOption') {
84+
if (newValue instanceof SearchResult) {
85+
navigate(TODO_LISTS_ROUTE + '/' + newValue.id);
86+
}
87+
}
88+
}}
89+
selectOnFocus
90+
clearOnBlur
91+
handleHomeEndKeys
92+
renderInput={(params) => (
93+
<TextField
94+
{...params}
95+
label="Search..."
96+
InputProps={{
97+
...params.InputProps
98+
}}
99+
/>
100+
)}
101+
/>
102+
</FormControl>
103+
</div>
104+
);
105+
};

‎packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
238238
this.sdkVersion = version.rows?.item(0)['powersync_rs_version()'] ?? '';
239239
await this.updateSchema(this.options.schema);
240240
this.updateHasSynced();
241+
await this.database.execute('PRAGMA RECURSIVE_TRIGGERS=TRUE');
241242
this.ready = true;
242243
this.iterateListeners((cb) => cb.initialized?.());
243244
}

‎packages/powersync-sdk-web/package.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
"author": "JOURNEYAPPS",
3434
"license": "Apache-2.0",
3535
"devDependencies": {
36-
"@journeyapps/wa-sqlite": "~0.1.1",
36+
"@journeyapps/wa-sqlite": "~0.2.0",
3737
"@types/lodash": "^4.14.200",
3838
"@types/uuid": "^9.0.6",
3939
"@vitest/browser": "^1.3.1",
@@ -45,7 +45,7 @@
4545
"webdriverio": "^8.32.3"
4646
},
4747
"peerDependencies": {
48-
"@journeyapps/wa-sqlite": "~0.1.1"
48+
"@journeyapps/wa-sqlite": "~0.2.0"
4949
},
5050
"dependencies": {
5151
"@journeyapps/powersync-sdk-common": "workspace:*",

0 commit comments

Comments
 (0)