From dd554fe25285b8dee680dc9bd91a35903b41fd5d Mon Sep 17 00:00:00 2001 From: amirhouieh Date: Mon, 22 Jan 2024 13:41:42 +0100 Subject: [PATCH] new search --- package.json | 4 + .../Episode/Footer/Episode.Footer.tsx | 3 +- src/containers/ArticleContainer.tsx | 6 +- .../GlobalSearchBox/GlobalSearchBox.tsx | 20 +- src/containers/HomePage/HomePage.tsx | 1 + .../PostSearchContainer.tsx | 5 +- src/containers/Search/ListView.tsx | 6 +- src/lib/strapi/strapi.generated.ts | 370 ++++++++++++++++++ src/lib/strapi/strapi.graphql | 11 +- src/pages/api/search/index.ts | 101 +---- src/pages/api/search/post/[id].ts | 45 +-- src/pages/search.tsx | 24 +- src/queries/usePostSearch.query.ts | 21 +- src/services/api.service.ts | 17 +- src/services/post-search.service.ts | 58 +++ src/services/strapi/strapi.operators.ts | 22 ++ src/services/strapi/strapi.service.ts | 74 ++++ .../strapi/transformers/Post.transformer.ts | 12 + .../transformers/SearchResult.transformer.ts | 25 ++ .../transformers/strapi.transformers.ts | 2 + .../PodcastEpisodeDocument.dataType.ts | 2 +- src/types/lpe.types.ts | 2 + src/utils/search.utils.ts | 11 +- yarn.lock | 20 + 24 files changed, 682 insertions(+), 180 deletions(-) create mode 100644 src/services/post-search.service.ts create mode 100644 src/services/strapi/transformers/SearchResult.transformer.ts diff --git a/package.json b/package.json index 760a59c..585e308 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "feed": "^4.2.2", "graphql": "^16.7.1", "graphql-request": "^6.0.0", + "lunr": "^2.3.9", "next": "13.3.0", "next-query-params": "^4.2.3", "nextjs-progressbar": "^0.0.16", @@ -63,6 +64,7 @@ "slugify": "^1.6.6", "typescript": "5.0.4", "use-query-params": "^2.2.1", + "uuid": "^9.0.1", "yup": "^1.3.2" }, "devDependencies": { @@ -72,7 +74,9 @@ "@graphql-codegen/typescript": "^4.0.1", "@graphql-codegen/typescript-resolvers": "^4.0.1", "@parcel/watcher": "^2.2.0", + "@types/lunr": "^2.3.7", "@types/react-imgix": "^9.5.0", + "@types/uuid": "^9.0.7", "dotenv-cli": "^7.2.1", "husky": "^8.0.3", "lint-staged": "^13.2.1", diff --git a/src/components/Episode/Footer/Episode.Footer.tsx b/src/components/Episode/Footer/Episode.Footer.tsx index 531fd8f..5377521 100644 --- a/src/components/Episode/Footer/Episode.Footer.tsx +++ b/src/components/Episode/Footer/Episode.Footer.tsx @@ -16,8 +16,7 @@ const EpisodeFooter = ({ episode, relatedEpisodes }: Props) => { return ( episode.content && episode.content - .filter((b) => (b as LPE.Post.TextBlock).footnotes.length) - .map((b) => (b as LPE.Post.TextBlock).footnotes) + .map((b) => (b as LPE.Post.TextBlock).footnotes || []) .flat() ) }, [episode]) diff --git a/src/containers/ArticleContainer.tsx b/src/containers/ArticleContainer.tsx index e85db18..cd4981c 100644 --- a/src/containers/ArticleContainer.tsx +++ b/src/containers/ArticleContainer.tsx @@ -19,7 +19,11 @@ const ArticleContainer = (props: Props) => { const [tocId, setTocId] = useState(null) return ( - + {(search) => { const displaySearchResults = search.active && !search.isInitialLoading diff --git a/src/containers/GlobalSearchBox/GlobalSearchBox.tsx b/src/containers/GlobalSearchBox/GlobalSearchBox.tsx index 86b9625..5e605b4 100644 --- a/src/containers/GlobalSearchBox/GlobalSearchBox.tsx +++ b/src/containers/GlobalSearchBox/GlobalSearchBox.tsx @@ -31,16 +31,16 @@ const contentTypes = [ value: PostTypes.Podcast, category: ContentTypesCategories.Post, }, - { - label: 'Paragraphs', - value: ContentBlockTypes.Text, - category: ContentTypesCategories.Block, - }, - { - label: 'Images', - value: ContentBlockTypes.Image, - category: ContentTypesCategories.Block, - }, + // { + // label: 'Paragraphs', + // value: ContentBlockTypes.Text, + // category: ContentTypesCategories.Block, + // }, + // { + // label: 'Images', + // value: ContentBlockTypes.Image, + // category: ContentTypesCategories.Block, + // }, ] const allContentTypes = contentTypes.map((c) => c.value) diff --git a/src/containers/HomePage/HomePage.tsx b/src/containers/HomePage/HomePage.tsx index deb1b4e..f29e367 100644 --- a/src/containers/HomePage/HomePage.tsx +++ b/src/containers/HomePage/HomePage.tsx @@ -116,6 +116,7 @@ export const HomePage: React.FC = ({ -> = ({ postId = '', postTitle, children, ...props }) => { +> = ({ postId = '', postTitle, blocks = [], children, ...props }) => { const navbarState = useNavbarState() const [prevQuery, setPrevQuery] = useState('') @@ -27,6 +29,7 @@ export const PostSearchContainer: React.FC< const res = usePostSearchQuery({ id: postId, query, + blocks, active: active && query.length > 0, }) diff --git a/src/containers/Search/ListView.tsx b/src/containers/Search/ListView.tsx index a7ddc23..b61e9a0 100644 --- a/src/containers/Search/ListView.tsx +++ b/src/containers/Search/ListView.tsx @@ -1,8 +1,6 @@ import { Grid, GridItem } from '@/components/Grid/Grid' -import { SearchResultListBlocks } from '@/components/Search/SearchResult.Blocks' import { SearchResultTopPost } from '@/components/Search/SearchResult.TopPost' import { SearchResultListPosts } from '@/components/Search/SearchResultList.Posts' -import { SearchResultsListHeader } from '@/components/Search/SearchResultsList.Header' import { copyConfigs } from '@/configs/copy.configs' import { uiConfigs } from '@/configs/ui.configs' import { LPE } from '@/types/lpe.types' @@ -138,7 +136,7 @@ export const SearchResultsListView = (props: Props) => { - + {/* {!isMobile && ( { )} - + */} ) } diff --git a/src/lib/strapi/strapi.generated.ts b/src/lib/strapi/strapi.generated.ts index 1b7d421..aa8db28 100644 --- a/src/lib/strapi/strapi.generated.ts +++ b/src/lib/strapi/strapi.generated.ts @@ -886,6 +886,7 @@ export type PostEntity = { __typename?: 'PostEntity' attributes: Maybe id: Maybe + score: Maybe } export type PostEntityResponse = { @@ -968,6 +969,7 @@ export type Query = { podcastShows: Maybe post: Maybe posts: Maybe + search: Maybe tag: Maybe tags: Maybe uploadFile: Maybe @@ -1053,6 +1055,10 @@ export type QueryPostsArgs = { sort?: InputMaybe>> } +export type QuerySearchArgs = { + query: Scalars['String']['input'] +} + export type QueryTagArgs = { id?: InputMaybe } @@ -1108,6 +1114,16 @@ export type ResponseCollectionMeta = { pagination: Pagination } +export type SearchResult = { + __typename?: 'SearchResult' + posts: Maybe +} + +export type SearchResultPostsArgs = { + filters?: InputMaybe + pagination?: InputMaybe +} + export type StringFilterInput = { and?: InputMaybe>> between?: InputMaybe>> @@ -1813,6 +1829,76 @@ export type GetRelatedPostsQuery = { } } +export type SearchPostsQueryVariables = Exact<{ + query: Scalars['String']['input'] + filters?: InputMaybe + pagination?: InputMaybe +}> + +export type SearchPostsQuery = { + __typename?: 'Query' + search: { + __typename?: 'SearchResult' + posts: { + __typename?: 'PostEntityResponseCollection' + data: Array<{ + __typename?: 'PostEntity' + id: string + score: number + attributes: { + __typename?: 'Post' + type: Enum_Post_Type + title: string + subtitle: string + summary: string + slug: string + featured: boolean + episode_number: number + publish_date: any + publishedAt: any + podcast_show: { + __typename?: 'PodcastShowEntityResponse' + data: { __typename?: 'PodcastShowEntity'; id: string } + } + cover_image: { + __typename?: 'UploadFileEntityResponse' + data: { + __typename?: 'UploadFileEntity' + attributes: { + __typename?: 'UploadFile' + url: string + width: number + height: number + caption: string + } + } + } + authors: { + __typename?: 'AuthorRelationResponseCollection' + data: Array<{ + __typename?: 'AuthorEntity' + id: string + attributes: { + __typename?: 'Author' + name: string + email_address: string + } + }> + } + tags: { + __typename?: 'TagRelationResponseCollection' + data: Array<{ + __typename?: 'TagEntity' + id: string + attributes: { __typename?: 'Tag'; name: string } + }> + } + } + }> + } + } +} + export type GetStaticPagesQueryVariables = Exact<{ filters?: InputMaybe pagination?: InputMaybe @@ -2952,6 +3038,290 @@ export const GetRelatedPostsDocument = { GetRelatedPostsQuery, GetRelatedPostsQueryVariables > +export const SearchPostsDocument = { + kind: 'Document', + definitions: [ + { + kind: 'OperationDefinition', + operation: 'query', + name: { kind: 'Name', value: 'SearchPosts' }, + variableDefinitions: [ + { + kind: 'VariableDefinition', + variable: { + kind: 'Variable', + name: { kind: 'Name', value: 'query' }, + }, + type: { + kind: 'NonNullType', + type: { + kind: 'NamedType', + name: { kind: 'Name', value: 'String' }, + }, + }, + }, + { + kind: 'VariableDefinition', + variable: { + kind: 'Variable', + name: { kind: 'Name', value: 'filters' }, + }, + type: { + kind: 'NamedType', + name: { kind: 'Name', value: 'PostFiltersInput' }, + }, + }, + { + kind: 'VariableDefinition', + variable: { + kind: 'Variable', + name: { kind: 'Name', value: 'pagination' }, + }, + type: { + kind: 'NamedType', + name: { kind: 'Name', value: 'PaginationArg' }, + }, + }, + ], + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'Field', + name: { kind: 'Name', value: 'search' }, + arguments: [ + { + kind: 'Argument', + name: { kind: 'Name', value: 'query' }, + value: { + kind: 'Variable', + name: { kind: 'Name', value: 'query' }, + }, + }, + ], + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'Field', + name: { kind: 'Name', value: 'posts' }, + arguments: [ + { + kind: 'Argument', + name: { kind: 'Name', value: 'filters' }, + value: { + kind: 'Variable', + name: { kind: 'Name', value: 'filters' }, + }, + }, + { + kind: 'Argument', + name: { kind: 'Name', value: 'pagination' }, + value: { + kind: 'Variable', + name: { kind: 'Name', value: 'pagination' }, + }, + }, + ], + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'Field', + name: { kind: 'Name', value: 'data' }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'Field', + name: { kind: 'Name', value: 'id' }, + }, + { + kind: 'Field', + name: { kind: 'Name', value: 'score' }, + }, + { + kind: 'Field', + name: { kind: 'Name', value: 'attributes' }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'FragmentSpread', + name: { + kind: 'Name', + value: 'PostCommonAttributes', + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + { + kind: 'FragmentDefinition', + name: { kind: 'Name', value: 'PostCommonAttributes' }, + typeCondition: { + kind: 'NamedType', + name: { kind: 'Name', value: 'Post' }, + }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + { kind: 'Field', name: { kind: 'Name', value: 'type' } }, + { kind: 'Field', name: { kind: 'Name', value: 'title' } }, + { kind: 'Field', name: { kind: 'Name', value: 'subtitle' } }, + { kind: 'Field', name: { kind: 'Name', value: 'summary' } }, + { kind: 'Field', name: { kind: 'Name', value: 'slug' } }, + { kind: 'Field', name: { kind: 'Name', value: 'featured' } }, + { kind: 'Field', name: { kind: 'Name', value: 'episode_number' } }, + { + kind: 'Field', + name: { kind: 'Name', value: 'podcast_show' }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'Field', + name: { kind: 'Name', value: 'data' }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + { kind: 'Field', name: { kind: 'Name', value: 'id' } }, + ], + }, + }, + ], + }, + }, + { + kind: 'Field', + name: { kind: 'Name', value: 'cover_image' }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'Field', + name: { kind: 'Name', value: 'data' }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'Field', + name: { kind: 'Name', value: 'attributes' }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'Field', + name: { kind: 'Name', value: 'url' }, + }, + { + kind: 'Field', + name: { kind: 'Name', value: 'width' }, + }, + { + kind: 'Field', + name: { kind: 'Name', value: 'height' }, + }, + { + kind: 'Field', + name: { kind: 'Name', value: 'caption' }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + { + kind: 'Field', + name: { kind: 'Name', value: 'authors' }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'Field', + name: { kind: 'Name', value: 'data' }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + { kind: 'Field', name: { kind: 'Name', value: 'id' } }, + { + kind: 'Field', + name: { kind: 'Name', value: 'attributes' }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'Field', + name: { kind: 'Name', value: 'name' }, + }, + { + kind: 'Field', + name: { kind: 'Name', value: 'email_address' }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + { kind: 'Field', name: { kind: 'Name', value: 'publish_date' } }, + { kind: 'Field', name: { kind: 'Name', value: 'publishedAt' } }, + { + kind: 'Field', + name: { kind: 'Name', value: 'tags' }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'Field', + name: { kind: 'Name', value: 'data' }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + { kind: 'Field', name: { kind: 'Name', value: 'id' } }, + { + kind: 'Field', + name: { kind: 'Name', value: 'attributes' }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'Field', + name: { kind: 'Name', value: 'name' }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + ], +} as unknown as DocumentNode export const GetStaticPagesDocument = { kind: 'Document', definitions: [ diff --git a/src/lib/strapi/strapi.graphql b/src/lib/strapi/strapi.graphql index 9099fd2..9140c64 100644 --- a/src/lib/strapi/strapi.graphql +++ b/src/lib/strapi/strapi.graphql @@ -602,6 +602,7 @@ type Post { type PostEntity { attributes: Post id: ID + score: Float } type PostEntityResponse { @@ -683,6 +684,10 @@ type Query { podcastShows(filters: PodcastShowFiltersInput, pagination: PaginationArg = {}, publicationState: PublicationState = LIVE, sort: [String] = []): PodcastShowEntityResponseCollection post(id: ID): PostEntityResponse posts(filters: PostFiltersInput, pagination: PaginationArg = {}, publicationState: PublicationState = LIVE, sort: [String] = []): PostEntityResponseCollection + search( + """Search query""" + query: String! + ): SearchResult tag(id: ID): TagEntityResponse tags(filters: TagFiltersInput, pagination: PaginationArg = {}, sort: [String] = []): TagEntityResponseCollection uploadFile(id: ID): UploadFileEntityResponse @@ -699,6 +704,10 @@ type ResponseCollectionMeta { pagination: Pagination! } +type SearchResult { + posts(filters: PostFiltersInput, pagination: PaginationArg): PostEntityResponseCollection +} + input StringFilterInput { and: [String] between: [String] @@ -1082,4 +1091,4 @@ input UsersPermissionsUserInput { type UsersPermissionsUserRelationResponseCollection { data: [UsersPermissionsUserEntity!]! -} \ No newline at end of file +} diff --git a/src/pages/api/search/index.ts b/src/pages/api/search/index.ts index 5c7ea28..375bb97 100644 --- a/src/pages/api/search/index.ts +++ b/src/pages/api/search/index.ts @@ -1,4 +1,5 @@ import type { NextApiRequest, NextApiResponse } from 'next' +import { strapiApi } from '../../../services/strapi' import { LPE } from '../../../types/lpe.types' export default async function handler( @@ -6,14 +7,7 @@ export default async function handler( res: NextApiResponse, ) { const { - query: { - q = '', - tags: tagsString = '', - type: typeString = '', - - skip = 0, - limit = 50, - }, + query: { q = '', tags: tagsString = '', type: typeString = '' }, } = req const tags = @@ -25,8 +19,6 @@ export default async function handler( : undefined const validPostTypes: string[] = Object.values(LPE.PostTypes) - const validBlockTypes: string[] = Object.values(LPE.Post.ContentBlockTypes) - const validTypes = [...validPostTypes, ...validBlockTypes] const type = typeof typeString === 'string' @@ -36,20 +28,10 @@ export default async function handler( .filter((t) => t.length > 0) : undefined - const queryTypes = Array.isArray(type) - ? type.filter((t) => validTypes.includes(t)) + const postTypes = Array.isArray(type) + ? type.filter((t) => validPostTypes.includes(t)) : [] - const postTypes = - queryTypes.length > 0 - ? queryTypes.filter((type) => validPostTypes.includes(type)) - : validPostTypes - - const blockTypes = - queryTypes.length > 0 - ? queryTypes.filter((type) => validBlockTypes.includes(type)) - : validBlockTypes - const result: { posts: LPE.Search.ResultItem[] blocks: LPE.Search.ResultItem[] @@ -59,74 +41,17 @@ export default async function handler( } if (postTypes.length > 0) { - // const response = await unbodyApi.searchPosts({ - // tags, - // query: Array.isArray(q) ? q.join(' ').trim() : q.trim(), - - // type: postTypes as LPE.PostType[], - - // limit: parseInt(limit, 50), - // skip: parseInt(skip, 0), - // }) - - const response = { - data: [], - } - - result.posts.push(...response.data) - } - - if (blockTypes.length > 0) { - // const response = await unbodyApi.searchPostBlocks({ - // tags, - // query: Array.isArray(q) ? q.join(' ') : q, - - // postType: postTypes as LPE.PostType[], - // type: blockTypes as LPE.Post.ContentBlockType[], - - // method: 'hybrid', - // limit: parseInt(limit, 50), - // skip: parseInt(skip, 0), - // }) - - const response = { - data: [], - } - - result.blocks.push(...response.data) - } - - const calcPostScore = (postScore: number, blockScores: number[]): number => { - const topScoreWeight = 0.5 - const postScoreWeight = 1 - const blocksCountWeight = 0.1 - - const topScore = blockScores[0] ?? 0 - - return ( - (postScore * postScoreWeight + - (blockScores.length / result.blocks.length) * blocksCountWeight + - topScore * topScoreWeight) / - (topScoreWeight + postScoreWeight + blocksCountWeight) - ) - } - - if (skip === 0) - result.posts = [...result.posts].sort((a, b) => { - const [blocks1, blocks2] = [a, b].map((p) => - result.blocks - .filter( - (block) => - 'document' in block.data && block.data.document.id === p.data.id, - ) - .map((block) => block.score), - ) - - return calcPostScore(a.score, blocks1) > calcPostScore(b.score, blocks2) - ? -1 - : 1 + const response = await strapiApi.searchPosts({ + tags, + query: Array.isArray(q) ? q.join(' ').trim() : q.trim(), + types: postTypes as LPE.PostType[], + limit: 15, + skip: 0, }) + result.posts.push(...(response.data ?? [])) + } + res.status(200).json({ data: result, }) diff --git a/src/pages/api/search/post/[id].ts b/src/pages/api/search/post/[id].ts index 6b6f4cd..ec22fde 100644 --- a/src/pages/api/search/post/[id].ts +++ b/src/pages/api/search/post/[id].ts @@ -1,52 +1,27 @@ import type { NextApiRequest, NextApiResponse } from 'next' -import { LPE } from '../../../../types/lpe.types' +import postSearchService from '../../../../services/post-search.service' export default async function handler( req: NextApiRequest, res: NextApiResponse, ) { const { - query: { id, q = '', tags: tagsString = '', limit = 50, skip = 0 }, + query: { id, q = '' }, } = req - if (!id) { + + if (!id || typeof id !== 'string') { return res.status(400).json({ error: 'Invalid request' }) } - const tags = - typeof tagsString === 'string' - ? tagsString - .split(',') - .map((tag: string) => tag.trim()) - .filter((t) => t.length > 0) - : undefined - - const result: LPE.Search.Result = { - posts: [], - blocks: [], + if (!q || typeof q !== 'string') { + return res.status(400).json({ error: 'Invalid request' }) } - // const query: Parameters<(typeof unbodyApi)['searchPostBlocks']>[0] = { - // tags, - // type: ['text', 'image'], - // postId: Array.isArray(id) ? id[0] : id, - // query: Array.isArray(q) ? q.join(' ') : q, - // method: 'nearText', - // certainty: 0.85, - - // limit: parseInt(limit, 50), - // skip: parseInt(skip, 0), - // } - - // result.blocks = await unbodyApi - // .searchPostBlocks(query) - // .then((res) => res.data) - - // if (result.blocks.length === 0) - // result.blocks = await unbodyApi - // .searchPostBlocks({ ...query, method: 'hybrid' }) - // .then((res) => res.data) + const result = postSearchService.search(id, q) res.status(200).json({ - data: result, + data: { + blocks: result, + }, }) } diff --git a/src/pages/search.tsx b/src/pages/search.tsx index 97f7741..f5854e1 100644 --- a/src/pages/search.tsx +++ b/src/pages/search.tsx @@ -8,10 +8,10 @@ import NextAdapterPages from 'next-query-params' import { ReactNode, useState } from 'react' import { QueryParamProvider } from 'use-query-params' import SEO from '../components/SEO/SEO' -import { copyConfigs } from '../configs/copy.configs' import { GlobalSearchBox } from '../containers/GlobalSearchBox/GlobalSearchBox' import { DefaultLayout } from '../layouts/DefaultLayout' import { api } from '../services/api.service' +import { strapiApi } from '../services/strapi' interface SearchPageProps { topics: string[] @@ -75,10 +75,12 @@ export default function SearchPage({ topics, shows }: SearchPageProps) { ( ) export async function getStaticProps() { - // const { data: topics, errors: topicErrors } = await unbodyApi.getTopics() - // const { data: shows = [] } = await unbodyApi.getPodcastShows({ - // populateEpisodes: true, - // episodesLimit: 10, - // }) - - const topics = [] as any - const shows = [] as any + const { data: shows } = await strapiApi.getPodcastShows({}) + const { data: topics } = await strapiApi.getTopics() return { props: { - topics: topics.map((topic) => topic.value), shows, + topics: topics.map((topic) => topic.name), }, revalidate: 10, } diff --git a/src/queries/usePostSearch.query.ts b/src/queries/usePostSearch.query.ts index 92a6608..df0bac5 100644 --- a/src/queries/usePostSearch.query.ts +++ b/src/queries/usePostSearch.query.ts @@ -8,23 +8,30 @@ export const usePostSearchQuery = ({ query, limit = 50, active = false, + blocks = [], }: { id: string query: string limit?: number active?: boolean + blocks: LPE.Post.ContentBlock[] }) => useQuery( ['post-search-query', active, id, query, limit], async () => !active ? [] - : api - .searchPostBlocks({ limit, id, query, skip: 0 }) - .then((res) => - (res.data.blocks as LPE.Search.ResultBlockItem[]).filter( - searchBlocksBasicFilter, - ), - ), + : api.searchPostBlocks({ limit, id, query, skip: 0 }).then((res) => + res.data.blocks + .map( + (r) => + ({ + data: blocks[r.index], + score: r.score, + type: blocks[r.index].type, + } as LPE.Search.ResultBlockItem), + ) + .filter(searchBlocksBasicFilter), + ), { keepPreviousData: true }, ) diff --git a/src/services/api.service.ts b/src/services/api.service.ts index 4768192..b7ea75d 100644 --- a/src/services/api.service.ts +++ b/src/services/api.service.ts @@ -68,16 +68,19 @@ export class ApiService { tags?: string[] limit?: number skip?: number - }): Promise> => - fetch( - `/api/search/post/${id}?skip=${skip}&limit=${limit}&q=${query}&tags=${tags.join( - ',', - )}`, - ) + }): Promise< + ApiResponse<{ + blocks: { + index: number + score: number + }[] + }> + > => + fetch(`/api/search/post/${id}?q=${query}`) .then((res) => res.json()) .catch((e) => { console.error(e) - return { data: { posts: [], blocks: [] }, errors: JSON.stringify(e) } + return { data: { blocks: [], posts: [] }, errors: JSON.stringify(e) } }) subscribeToMailingList = async (payload: { diff --git a/src/services/post-search.service.ts b/src/services/post-search.service.ts new file mode 100644 index 0000000..0ca71c3 --- /dev/null +++ b/src/services/post-search.service.ts @@ -0,0 +1,58 @@ +import lunr from 'lunr' +import { LPE } from '../types/lpe.types' + +type Post = { + index: lunr.Index +} + +export class PostSearchService { + posts: Record = {} + + constructor() { + this.posts = {} + } + + index = (post: Pick) => { + const id = post.id + delete this.posts[id] + + const index = lunr(function () { + this.ref('index') + this.field('text') + + post.content.forEach((block, index) => { + this.add({ + index, + text: block.type === 'text' ? block.text : block.caption, + }) + }) + }) + + this.posts[id] = { index } + + return id + } + + search = (id: string, query: string) => { + const post = this.posts[id] + if (!post) return [] + + const idx = post.index + const results = idx.search(query + '~1') + + return results.map((r) => ({ + score: r.score, + index: parseInt(r.ref, 10), + })) + } +} + +const postSearchService: PostSearchService = (() => { + const _globalThis = globalThis as any + if (!_globalThis.postSearchService) + _globalThis.postSearchService = new PostSearchService() + + return _globalThis.postSearchService +})() + +export default postSearchService as PostSearchService diff --git a/src/services/strapi/strapi.operators.ts b/src/services/strapi/strapi.operators.ts index d69fdf1..eaba128 100644 --- a/src/services/strapi/strapi.operators.ts +++ b/src/services/strapi/strapi.operators.ts @@ -152,6 +152,28 @@ export const GET_RELATED_POSTS_QUERY = gql` } ` +export const SEARCH_POSTS_QUERY = gql` + ${POST_COMMON_ATTRIBUTES} + + query SearchPosts( + $query: String! + $filters: PostFiltersInput + $pagination: PaginationArg + ) { + search(query: $query) { + posts(filters: $filters, pagination: $pagination) { + data { + id + score + attributes { + ...PostCommonAttributes + } + } + } + } + } +` + export const GET_STATIC_PAGES = gql` query GetStaticPages( $filters: PageFiltersInput diff --git a/src/services/strapi/strapi.service.ts b/src/services/strapi/strapi.service.ts index b306fff..6afa759 100644 --- a/src/services/strapi/strapi.service.ts +++ b/src/services/strapi/strapi.service.ts @@ -8,6 +8,7 @@ import { GetStaticPagesDocument, PodcastShowFiltersInput, PostFiltersInput, + SearchPostsDocument, } from '../../lib/strapi/strapi.generated' import { ApiResponse } from '../../types/data.types' import { LPE } from '../../types/lpe.types' @@ -493,6 +494,79 @@ export class StrapiService { postsCount: tag.posts?.count ?? 0, })) }) + + searchPosts = async ({ + query = '', + skip = 0, + limit = 10, + filters = {}, + tags, + types = [LPE.PostTypes.Article, LPE.PostTypes.Podcast], + }: { + query?: string + skip?: number + limit?: number + filters?: PostFiltersInput + tags?: string[] + types?: LPE.PostType[] + }) => + this.handleRequest(async () => { + const { + data: { + search: { + posts: { data }, + }, + }, + } = await this.client.query({ + query: SearchPostsDocument, + variables: { + query, + pagination: { + start: skip, + limit: limit, + }, + filters: { + and: [ + ...(filters ? [filters] : []), + { + publishedAt: { + notNull: true, + }, + }, + ...(tags && tags.length > 0 + ? [ + { + tags: { + name: { + in: tags, + }, + }, + }, + ] + : []), + ...(types && types.length > 0 + ? [ + { + type: { + in: types.map((type) => + type === 'article' ? 'Article' : 'Episode', + ), + }, + }, + ] + : []), + ], + }, + }, + }) + + return await strapiTransformers.transformMany( + strapiTransformers.get({}), + data, + undefined, + undefined, + ) + }) } export const strapiApi = new StrapiService( diff --git a/src/services/strapi/transformers/Post.transformer.ts b/src/services/strapi/transformers/Post.transformer.ts index 03226b1..0368158 100644 --- a/src/services/strapi/transformers/Post.transformer.ts +++ b/src/services/strapi/transformers/Post.transformer.ts @@ -1,6 +1,8 @@ +import * as _uuid from 'uuid' import { Transformer } from '../../../lib/TransformPipeline/types' import { LPE } from '../../../types/lpe.types' import { calcReadingTime } from '../../../utils/string.utils' +import postSearchService from '../../post-search.service' import { StrapiPostData } from '../strapi.types' import { transformStrapiHtmlContent, transformStrapiImageData } from './utils' @@ -17,6 +19,7 @@ export const postTransformer: Transformer< isMatch: (helpers, object) => object.__typename === 'PostEntity', transform: (helpers, data, original, root, ctx) => { const { id, attributes } = data + const uuid = _uuid.v5(id, _uuid.v5.URL) const type = attributes.type const title = attributes.title @@ -50,6 +53,13 @@ export const postTransformer: Transformer< html: attributes.body || '', }) + if (attributes.body && content.length > 0) { + postSearchService.index({ + id: uuid, + content, + }) + } + // add the title as the first toc item { toc.unshift({ @@ -64,6 +74,7 @@ export const postTransformer: Transformer< if (type === 'Article') { return { id, + uuid, title, subtitle, slug, @@ -83,6 +94,7 @@ export const postTransformer: Transformer< } else { return { id, + uuid, title, subtitle, slug, diff --git a/src/services/strapi/transformers/SearchResult.transformer.ts b/src/services/strapi/transformers/SearchResult.transformer.ts new file mode 100644 index 0000000..c1ee946 --- /dev/null +++ b/src/services/strapi/transformers/SearchResult.transformer.ts @@ -0,0 +1,25 @@ +import { Transformer } from '../../../lib/TransformPipeline/types' +import { LPE } from '../../../types/lpe.types' +import { StrapiPostData } from '../strapi.types' + +export const searchResultTransformer: Transformer< + LPE.Post.Document, + LPE.Search.ResultItemBase, + StrapiPostData & { score: number }, + undefined, + undefined +> = { + key: 'SearchResultTransformer', + classes: ['post', 'search'], + objectType: 'Post', + isMatch: (helpers, object, original) => + [LPE.PostTypes.Article, LPE.PostTypes.Podcast].includes(object.type) && + typeof original.score !== 'undefined', + transform: (helpers, data, original, root, ctx) => { + return { + data: data, + type: data.type, + score: original.score, + } + }, +} diff --git a/src/services/strapi/transformers/strapi.transformers.ts b/src/services/strapi/transformers/strapi.transformers.ts index ebe7e09..75b6f6e 100644 --- a/src/services/strapi/transformers/strapi.transformers.ts +++ b/src/services/strapi/transformers/strapi.transformers.ts @@ -2,6 +2,7 @@ import { TransformPipeline } from '../../../lib/TransformPipeline/TransformPipel import { episodeTransformer } from './Episode.transformer' import { podcastShowTransformer } from './PodcastShow.transformer' import { postTransformer } from './Post.transformer' +import { searchResultTransformer } from './SearchResult.transformer' import { staticPageTransformer } from './StaticPage.transformer' export const strapiTransformers = new TransformPipeline([ @@ -9,4 +10,5 @@ export const strapiTransformers = new TransformPipeline([ staticPageTransformer, postTransformer, episodeTransformer, + searchResultTransformer, ]) diff --git a/src/services/unbody/dataTypes/PodcastEpisodeDocument.dataType.ts b/src/services/unbody/dataTypes/PodcastEpisodeDocument.dataType.ts index d21bdb8..9127680 100644 --- a/src/services/unbody/dataTypes/PodcastEpisodeDocument.dataType.ts +++ b/src/services/unbody/dataTypes/PodcastEpisodeDocument.dataType.ts @@ -117,7 +117,7 @@ export const PodcastEpisodeDataType: UnbodyDataTypeConfig< highlighted: data.highlighted, isDraft: data.isDraft, type: LPE.PostTypes.Podcast, - } + } as any }, } diff --git a/src/types/lpe.types.ts b/src/types/lpe.types.ts index eb70c48..c784255 100644 --- a/src/types/lpe.types.ts +++ b/src/types/lpe.types.ts @@ -152,6 +152,7 @@ export namespace LPE { export type Metadata = { id: string + uuid: string slug: string title: string summary: string @@ -222,6 +223,7 @@ export namespace LPE { export type Metadata = { id: string + uuid: string slug: string title: string tags: Tag.Document[] diff --git a/src/utils/search.utils.ts b/src/utils/search.utils.ts index 3a8e664..36b3dd6 100644 --- a/src/utils/search.utils.ts +++ b/src/utils/search.utils.ts @@ -3,23 +3,16 @@ import { LPE } from '@/types/lpe.types' export const searchBlocksBasicFilter = ( block: LPE.Search.ResultItemBase, ) => { - const isTitle = (b: LPE.Post.TextBlock) => { - return b.classNames.includes('title') - } const isLongEnough = (b: LPE.Post.TextBlock) => { return b.text.length > 60 } if (block.type === LPE.ContentTypes.Text) { return ( - !isTitle(block.data as LPE.Post.TextBlock) && isLongEnough(block.data as LPE.Post.TextBlock) && !block.data.labels.includes('link_only') ) - } else { - const isPodcastImage = block.data.document.type === LPE.PostTypes.Podcast - if (isPodcastImage) return false - // exclude it if its cover image - return block.data.order !== 5 } + + return true } diff --git a/yarn.lock b/yarn.lock index 0022828..034559e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1599,6 +1599,11 @@ resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" integrity sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ== +"@types/lunr@^2.3.7": + version "2.3.7" + resolved "https://registry.yarnpkg.com/@types/lunr/-/lunr-2.3.7.tgz#378a98ecf7a9fafc42466f67f73173c34a6265a0" + integrity sha512-Tb/kUm38e8gmjahQzdCKhbdsvQ9/ppzHFfsJ0dMs3ckqQsRj+P5IkSAwFTBrBxdyr3E/LoMUUrZngjDYAjiE3A== + "@types/node@*": version "20.4.8" resolved "https://registry.yarnpkg.com/@types/node/-/node-20.4.8.tgz#b5dda19adaa473a9bf0ab5cbd8f30ec7d43f5c85" @@ -1652,6 +1657,11 @@ resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.3.tgz#cef09e3ec9af1d63d2a6cc5b383a737e24e6dcf5" integrity sha512-5cJ8CB4yAx7BH1oMvdU0Jh9lrEXyPkar6F9G/ERswkCuvP4KQZfZkSjcMbAICCpQTN4OuZn8tz0HiKv9TGZgrQ== +"@types/uuid@^9.0.7": + version "9.0.7" + resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-9.0.7.tgz#b14cebc75455eeeb160d5fe23c2fcc0c64f724d8" + integrity sha512-WUtIVRUZ9i5dYXefDEAI7sh9/O7jGvHg7Df/5O/gtH3Yabe5odI3UWopVR1qbPXQtvOxWu3mM4XxlYeZtMWF4g== + "@types/ws@^8.0.0", "@types/ws@^8.5.5": version "8.5.5" resolved "https://registry.yarnpkg.com/@types/ws/-/ws-8.5.5.tgz#af587964aa06682702ee6dcbc7be41a80e4b28eb" @@ -4240,6 +4250,11 @@ lru-cache@^6.0.0: dependencies: yallist "^4.0.0" +lunr@^2.3.9: + version "2.3.9" + resolved "https://registry.yarnpkg.com/lunr/-/lunr-2.3.9.tgz#18b123142832337dd6e964df1a5a7707b25d35e1" + integrity sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow== + magic-bytes.js@^1.0.15: version "1.0.15" resolved "https://registry.yarnpkg.com/magic-bytes.js/-/magic-bytes.js-1.0.15.tgz#3c9d2b7d45bb8432482646b5f74bbf6725274616" @@ -5865,6 +5880,11 @@ uuid@^8.3.1: resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== +uuid@^9.0.1: + version "9.0.1" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.1.tgz#e188d4c8853cc722220392c424cd637f32293f30" + integrity sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA== + value-or-promise@^1.0.11, value-or-promise@^1.0.12: version "1.0.12" resolved "https://registry.yarnpkg.com/value-or-promise/-/value-or-promise-1.0.12.tgz#0e5abfeec70148c78460a849f6b003ea7986f15c"