Refactor search, escape query strings for Tantivy

Revert "Add betafeatures"

This reverts commit 87145e9c30e63a516739c342b42111d045d516de.

Author: joepio

CHANGELOG.md

@@ -14,6 +14,7 @@ This changelog covers all three packages, as they are (for now) updated as a who
 - Add `store.preloadClassesAndProperties` and remove `urls.properties.getAll` and `urls.classes.getAll`. This enables using `atomic-data-browser` without relying on `atomicdata.dev` being available.
 - Fix Dialogue form #308
 - Fix Race condition of `store.getResourceAsync` #309
+- Refactor search, escape query strings for Tantivy
 
 ## v0.35.0
 

data-browser/src/components/SearchFilter.tsx

@@ -18,7 +18,7 @@ export function ClassFilter({ filters, setFilters }): JSX.Element {
     // Set the filters to the default values of the properties
     setFilters({
       ...filters,
-      'is-a': klass,
+      [urls.properties.isA]: klass,
     });
   }, [klass, JSON.stringify(filters)]);
 

data-browser/src/routes/SearchRoute.tsx

@@ -18,9 +18,18 @@ export function Search(): JSX.Element {
   const [query] = useSearchQuery();
   const { drive } = useSettings();
   const { scope } = useQueryScopeHandler();
-  const [filters, setFilters] = useState({
-    // 'is-a': classes.document,
-  });
+  const [filters, setFilters] = useState({});
+  const [enableFilter, setEnableFilter] = useState(false);
+
+  useHotkeys(
+    'f12',
+    e => {
+      e.preventDefault();
+      setEnableFilter(!enableFilter);
+    },
+    [enableFilter],
+  );
+
   const [showFilter, setShowFilter] = useState(false);
 
   const [selectedIndex, setSelected] = useState(0);
@@ -115,10 +124,12 @@ export function Search(): JSX.Element {
                 )}
               </span>
             </Heading>
-            <Button onClick={() => setShowFilter(!showFilter)}>
-              <FaFilter />
-              Filter
-            </Button>
+            {enableFilter && (
+              <Button onClick={() => setShowFilter(!showFilter)}>
+                <FaFilter />
+                Filter
+              </Button>
+            )}
           </div>
           {showFilter && (
             <ClassFilter setFilters={setFilters} filters={filters} />

lib/src/endpoints.ts

@@ -9,51 +9,3 @@ export function importJsonAdString(
 ) {
   return store.postToServer(importerUrl, jsonAdString);
 }
-
-export interface SearchOpts {
-  /** Fetch full resources instead of subjects */
-  include?: boolean;
-  /** Max of how many results to return */
-  limit?: number;
-  /** Subject of resource to scope the search to. This should be a parent of the resources you're looking for. */
-  scope?: string;
-  /** Property-Value pair of set filters. For now, use the `shortname` of the property as the key. */
-  filters?: {
-    [propertyShortname: string]: string;
-  };
-}
-
-/** Uses Tantivy query syntax */
-function buildFilterString(filters: { [key: string]: string }): string {
-  return Object.entries(filters)
-    .map(([key, value]) => {
-      return value && value.length > 0 && `${key}:"${value}"`;
-    })
-    .join(' AND ');
-}
-
-/** Returns the URL of the search query. Fetch that and you get your results! */
-export function buildSearchSubject(
-  store: Store,
-  query: string,
-  opts: SearchOpts = {},
-) {
-  const { include = false, limit = 30, scope, filters } = opts;
-  const url = new URL(store.getServerUrl());
-  url.pathname = 'search';
-  query && url.searchParams.set('q', query);
-  include && url.searchParams.set('include', include.toString());
-  limit && url.searchParams.set('limit', limit.toString());
-  // Only add filters if there are any keys, and if any key is defined
-  const hasFilters =
-    filters &&
-    Object.keys(filters).length > 0 &&
-    Object.values(filters).filter(v => v && v.length > 0).length > 0;
-  hasFilters && url.searchParams.set('filters', buildFilterString(filters));
-
-  if (scope) {
-    url.searchParams.set('parent', scope);
-  }
-
-  return url.toString();
-}

lib/src/index.ts

@@ -38,6 +38,7 @@ export * from './error.js';
 export * from './endpoints.js';
 export * from './datatypes.js';
 export * from './parse.js';
+export * from './search.js';
 export * from './resource.js';
 export * from './store.js';
 export * from './value.js';

lib/src/search.test.ts

@@ -0,0 +1,14 @@
+import { escapeTantivyKey } from './search.js';
+
+const testTuples = [
+  ['https://test', 'https\\://test'],
+  ['https://test.com', 'https\\://test\\.com'],
+];
+
+describe('search.ts', () => {
+  it('Handles resources without an ID', () => {
+    for (const [input, output] of testTuples) {
+      expect(escapeTantivyKey(input)).toBe(output);
+    }
+  });
+});

lib/src/search.ts

@@ -0,0 +1,78 @@
+import { Store } from './index.js';
+
+export interface SearchOpts {
+  /** Fetch full resources instead of subjects */
+  include?: boolean;
+  /** Max of how many results to return */
+  limit?: number;
+  /** Subject of resource to scope the search to. This should be a parent of the resources you're looking for. */
+  scope?: string;
+  /** Property-Value pair of set filters. For now, use the `shortname` of the property as the key. */
+  filters?: {
+    [propertyShortname: string]: string;
+  };
+}
+
+// https://github.com/quickwit-oss/tantivy/blob/064518156f570ee2aa03cf63be6d5605a96d6285/query-grammar/src/query_grammar.rs#L19
+const specialCharsTantivy = [
+  '+',
+  '^',
+  '`',
+  ':',
+  '{',
+  '}',
+  '"',
+  '[',
+  ']',
+  '(',
+  ')',
+  '!',
+  '\\',
+  '*',
+  ' ',
+  // The dot is escaped, even though it's not in Tantivy's list.
+  '.',
+];
+
+/** escape the key conform to Tantivy syntax, escaping all specialCharsTantivy */
+export function escapeTantivyKey(key: string) {
+  return key.replace(
+    new RegExp(`([${specialCharsTantivy.join('\\')}])`, 'g'),
+    '\\$1',
+  );
+}
+
+/** Uses Tantivy query syntax */
+function buildFilterString(filters: { [key: string]: string }): string {
+  return Object.entries(filters)
+    .map(([key, value]) => {
+      return value && value.length > 0 && `${escapeTantivyKey(key)}:"${value}"`;
+    })
+    .join(' AND ');
+}
+
+/** Returns the URL of the search query. Fetch that and you get your results! */
+export function buildSearchSubject(
+  store: Store,
+  query: string,
+  opts: SearchOpts = {},
+) {
+  const { include = false, limit = 30, scope, filters } = opts;
+  const url = new URL(store.getServerUrl());
+  url.pathname = 'search';
+  query && url.searchParams.set('q', query);
+  include && url.searchParams.set('include', include.toString());
+  limit && url.searchParams.set('limit', limit.toString());
+  // Only add filters if there are any keys, and if any key is defined
+  const hasFilters =
+    filters &&
+    Object.keys(filters).length > 0 &&
+    Object.values(filters).filter(v => v && v.length > 0).length > 0;
+  hasFilters && url.searchParams.set('filters', buildFilterString(filters));
+
+  if (scope) {
+    url.searchParams.set('parent', scope);
+  }
+
+  return url.toString();
+}

lib/src/store.test.ts

@@ -32,7 +32,7 @@ describe('Store', () => {
   });
 
   it('accepts a custom fetch implementation', async () => {
-    const testResourceSubject = 'https://example.com/test';
+    const testResourceSubject = 'https://atomicdata.dev';
 
     const customFetch = jest.fn(
       async (url: RequestInfo | URL, options: RequestInit | undefined) => {