From 9b8b2bd5ac9a0803388bff7615aaf84d3c12c848 Mon Sep 17 00:00:00 2001 From: frankporras Date: Tue, 11 Jun 2024 09:33:03 +0200 Subject: [PATCH] OpenSearch: almost implemented, fixing case sensitive issues --- package-lock.json | 22 +++ package.json | 2 + src/components/vs-input/vs-input.ts | 164 +++++++++++++----- src/components/vs-input/vs-input.vue | 3 +- src/models/dataset.ts | 22 ++- src/models/headers.ts | 2 +- src/services/dataset.service.ts | 24 ++- .../search-view/search-view-component.ts | 5 +- 8 files changed, 184 insertions(+), 60 deletions(-) diff --git a/package-lock.json b/package-lock.json index ea6b588..206df8e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "axios": "^1.2.2", "class-transformer": "^0.5.1", "dayjs": "^1.10.7", + "dompurify": "^3.1.5", "leaflet": "^1.7.1", "qs": "^6.10.1", "rxjs": "^7.5.5", @@ -29,6 +30,7 @@ "@babel/plugin-proposal-decorators": "^7.22.5", "@babel/preset-env": "^7.22.5", "@tailwindcss/forms": "^0.5.7", + "@types/dompurify": "^3.0.5", "@types/leaflet": "^1.7.9", "@typescript-eslint/eslint-plugin": "^7.2.0", "@typescript-eslint/parser": "^7.2.0", @@ -2497,6 +2499,15 @@ "@types/node": "*" } }, + "node_modules/@types/dompurify": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@types/dompurify/-/dompurify-3.0.5.tgz", + "integrity": "sha512-1Wg0g3BtQF7sSb27fJQAKck1HECM6zV1EB66j8JH9i3LCjYabJa0FSdiSgsD5K/RbrsR0SiraKacLB+T8ZVYAg==", + "dev": true, + "dependencies": { + "@types/trusted-types": "*" + } + }, "node_modules/@types/eslint": { "version": "8.56.7", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.56.7.tgz", @@ -2694,6 +2705,12 @@ "@types/node": "*" } }, + "node_modules/@types/trusted-types": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", + "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", + "dev": true + }, "node_modules/@types/webpack-env": { "version": "1.18.4", "resolved": "https://registry.npmjs.org/@types/webpack-env/-/webpack-env-1.18.4.tgz", @@ -5626,6 +5643,11 @@ "url": "https://github.com/fb55/domhandler?sponsor=1" } }, + "node_modules/dompurify": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.1.5.tgz", + "integrity": "sha512-lwG+n5h8QNpxtyrJW/gJWckL+1/DQiYMX8f7t8Z2AZTPw1esVrqjI63i7Zc2Gz0aKzLVMYC1V1PL/ky+aY/NgA==" + }, "node_modules/domutils": { "version": "2.8.0", "resolved": "https://registry.npmjs.org/domutils/-/domutils-2.8.0.tgz", diff --git a/package.json b/package.json index de5f0e1..efa7d39 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "axios": "^1.2.2", "class-transformer": "^0.5.1", "dayjs": "^1.10.7", + "dompurify": "^3.1.5", "leaflet": "^1.7.1", "qs": "^6.10.1", "rxjs": "^7.5.5", @@ -32,6 +33,7 @@ "@babel/plugin-proposal-decorators": "^7.22.5", "@babel/preset-env": "^7.22.5", "@tailwindcss/forms": "^0.5.7", + "@types/dompurify": "^3.0.5", "@types/leaflet": "^1.7.9", "@typescript-eslint/eslint-plugin": "^7.2.0", "@typescript-eslint/parser": "^7.2.0", diff --git a/src/components/vs-input/vs-input.ts b/src/components/vs-input/vs-input.ts index 04183e7..7f685a0 100644 --- a/src/components/vs-input/vs-input.ts +++ b/src/components/vs-input/vs-input.ts @@ -10,6 +10,9 @@ import { Dataset, Suggestion, SearchType } from "@/models/dataset"; import { SOLR_HOST, SOLR_CORE } from "@/constants"; import { OPEN_HOST, OPEN_CORE } from "@/constants"; // PENDING USE +import { HitHighlight } from "@/models/headers"; + +import DOMPurify from 'dompurify'; // To sanitize the HTML content to prevent XSS attacks! @Component({ name: "VsInput", @@ -30,6 +33,8 @@ export default class VsInput extends Vue { private value!: Suggestion | string; private error = ""; private results: Array = []; // Array to store search results + private highlights: Array = []; + private loading = false; // Loading state indicator private selectedIndex = -1; // Index of the currently selected suggestion // private selectedDisplay = ""; @@ -82,64 +87,123 @@ export default class VsInput extends Vue { // Computed property to generate suggestions based on search results get suggestions(): Suggestion[] { - // const suggestion = { - // titles: new Array(), - // authors: new Array(), - // subjects: new Array(), - // }; + const suggestions = new Array(); - console.log("Display:", this.display); - console.log("results:", this.results ); + console.log("Suggestions > Display:", this.display); + // console.log("results:", this.results ); + // console.log("highlights:", this.highlights); + //The method checks if there are any highlighted titles in the highlight object. If found, it joins the highlighted fragments into a single string // Generate suggestions based on search results - this.results.forEach((dataset) => { + this.results.forEach((dataset, index) => { - let foundAny = false; + const highlight = this.highlights[index]; console.log("get suggestions:id", dataset.id); - console.log("get suggestions:title_output", dataset.title_output); - console.log("get suggestions:author", dataset.author); - console.log("get suggestions:subjects", dataset.subjects); + // console.log("get suggestions:title_output", dataset.title_output); + // console.log("get suggestions:author", dataset.author); + // console.log("get suggestions:subjects", dataset.subjects); - // const del = dataset.title_output?.toLowerCase(); - if (dataset.title_output.toLowerCase().includes(this.display.toLowerCase())) { - const title = dataset.title_output; - // if (!suggestion["titles"].find((value) => value === title)) { - // suggestion.titles.push(title); - // } - // Check if there is already a suggestion with this title and type - const hasTitleSuggestion = suggestions.some((suggestion) => suggestion.value === title && suggestion.type == SearchType.Title); + // Checks if a suggestion with the same title and type already exists in the suggestions array. If not, it creates a new Suggestion object and adds it to the suggestions array. + if (highlight.title && highlight.title.length > 0) { + /** This line checks if the highlight object has a title property and if that property is an array with at least one element. + * The highlight object contains highlighted fragments of the search term in various fields (e.g., title, author, subjects) as returned by the OpenSearch API. + * This check ensures that we only process results that have highlighted titles. */ + const highlightedTitle = highlight.title.join(" "); + /** + * The highlight.title property is an array of strings, where each string is a highlighted fragment of the title. join(" ") combines these fragments into a single string with spaces between them. + * This step constructs a full highlighted title from the individual fragments. + * OpenSearch can return multiple fragments of a field (like the title) in its response, especially when the field contains multiple terms that match the search query. + * This can happen because OpenSearch's highlighting feature is designed to provide context around each match within the field, which can result in multiple highlighted fragments. + */ + const hasTitleSuggestion = suggestions.some((suggestion) => suggestion.highlight.toLowerCase() === highlightedTitle.toLowerCase() && suggestion.type == SearchType.Title); if (!hasTitleSuggestion) { - // If there is no such suggestion, create a new one and add it to the suggestions array - const suggestion = new Suggestion(title, SearchType.Title); + const suggestion = new Suggestion(dataset.title_output, highlightedTitle, SearchType.Title); suggestions.push(suggestion); - foundAny = true; } } - if (this.find(dataset.author, this.display.toLowerCase()) !== "") { - const author = this.find(dataset.author, this.display.toLowerCase()); - // Check if there is already a suggestion with this author and type - const hasAuthorSuggestion = suggestions.some((suggestion) => suggestion.value === author && suggestion.type == SearchType.Author); + if (highlight.author && highlight.author.length > 0) { + const highlightedAuthor = highlight.author.join(" "); + const datasetAuthor = this.find(dataset.author, this.display.toLowerCase()); + const hasAuthorSuggestion = suggestions.some((suggestion) => suggestion.highlight === highlightedAuthor && suggestion.type == SearchType.Author); if (!hasAuthorSuggestion) { - const suggestion = new Suggestion(author, SearchType.Author); + const suggestion = new Suggestion(datasetAuthor, highlightedAuthor, SearchType.Author); suggestions.push(suggestion); - foundAny = true; } } - if (this.find(dataset.subjects, this.display.toLowerCase()) != "") { - const subject = this.find(dataset.subjects, this.display.toLowerCase()); - const hasSubjectSuggestion = suggestions.some((suggestion) => suggestion.value === subject && suggestion.type == SearchType.Subject); + if (highlight.subjects && highlight.subjects.length > 0) { + const highlightedSubject = highlight.subjects.join(" "); + const datasetSubject = this.find(dataset.subjects, this.display.toLowerCase()); + const hasSubjectSuggestion = suggestions.some((suggestion) => suggestion.highlight === highlightedSubject && suggestion.type == SearchType.Subject); if (!hasSubjectSuggestion) { - const suggestion = new Suggestion(subject, SearchType.Subject); + const suggestion = new Suggestion(datasetSubject, highlightedSubject, SearchType.Subject); suggestions.push(suggestion); - foundAny = true; } } - // if (!foundAny) { - // const suggestion = new Suggestion(dataset.title_output, SearchType.Fuzzy); - // suggestions.push(suggestion); + // // Checks if a suggestion with the same title and type already exists in the suggestions array. If not, it creates a new Suggestion object and adds it to the suggestions array. + // if (highlight.title && highlight.title.length > 0) { + // /** This line checks if the highlight object has a title property and if that property is an array with at least one element. + // * The highlight object contains highlighted fragments of the search term in various fields (e.g., title, author, subjects) as returned by the OpenSearch API. + // * This check ensures that we only process results that have highlighted titles. */ + // const title = highlight.title.join(" "); + // /** + // * The highlight.title property is an array of strings, where each string is a highlighted fragment of the title. join(" ") combines these fragments into a single string with spaces between them. + // * This step constructs a full highlighted title from the individual fragments. + // * OpenSearch can return multiple fragments of a field (like the title) in its response, especially when the field contains multiple terms that match the search query. + // * This can happen because OpenSearch's highlighting feature is designed to provide context around each match within the field, which can result in multiple highlighted fragments. + // */ + // const hasTitleSuggestion = suggestions.some((suggestion) => suggestion.value === title && suggestion.type == SearchType.Title); + // if (!hasTitleSuggestion) { + // const suggestion = new Suggestion(title, SearchType.Title); + // suggestions.push(suggestion); + // } + // } + // if (highlight.author && highlight.author.length > 0) { + // const author = highlight.author.join(" "); + // const hasAuthorSuggestion = suggestions.some((suggestion) => suggestion.value === author && suggestion.type == SearchType.Author); + // if (!hasAuthorSuggestion) { + // const suggestion = new Suggestion(author, SearchType.Author); + // suggestions.push(suggestion); + // } + // } + // if (highlight.subjects && highlight.subjects.length > 0) { + // const subject = highlight.subjects.join(" "); + // const hasSubjectSuggestion = suggestions.some((suggestion) => suggestion.value === subject && suggestion.type == SearchType.Subject); + // if (!hasSubjectSuggestion) { + // const suggestion = new Suggestion(subject, SearchType.Subject); + // suggestions.push(suggestion); + // } + // } + + // ORIGINAL SOLR =================================================================================================== + // if (dataset.title_output.toLowerCase().includes(this.display.toLowerCase())) { + // const title = dataset.title_output; + // // Check if there is already a suggestion with this title and type + // const hasTitleSuggestion = suggestions.some((suggestion) => suggestion.value === title && suggestion.type == SearchType.Title); + // if (!hasTitleSuggestion) { + // // If there is no such suggestion, create a new one and add it to the suggestions array + // const suggestion = new Suggestion(title, SearchType.Title); + // suggestions.push(suggestion); + // } + // } + // if (this.find(dataset.author, this.display.toLowerCase()) !== "") { + // const author = this.find(dataset.author, this.display.toLowerCase()); + // // Check if there is already a suggestion with this author and type + // const hasAuthorSuggestion = suggestions.some((suggestion) => suggestion.value === author && suggestion.type == SearchType.Author); + // if (!hasAuthorSuggestion) { + // const suggestion = new Suggestion(author, SearchType.Author); + // suggestions.push(suggestion); + // } + // } + // if (this.find(dataset.subjects, this.display.toLowerCase()) != "") { + // const subject = this.find(dataset.subjects, this.display.toLowerCase()); + // const hasSubjectSuggestion = suggestions.some((suggestion) => suggestion.value === subject && suggestion.type == SearchType.Subject); + // if (!hasSubjectSuggestion) { + // const suggestion = new Suggestion(subject, SearchType.Subject); + // suggestions.push(suggestion); + // } // } }); @@ -147,10 +211,22 @@ export default class VsInput extends Vue { return suggestions; } + /** + * This method combines the suggestion value and type into a single HTML string. It also sanitizes the HTML content using DOMPurify to prevent XSS attacks. + * The vue file uses the v-html directive to bind the combined HTML string to the label element. This ensures that the HTML content (e.g., Wien) is rendered correctly in the browser. + */ + formatSuggestion(result: Suggestion): string { + const sanitizedValue = DOMPurify.sanitize(result.highlight); + // Replacing the predefined format for highlights given by OpenSearch from emphasys to bold + const replacedValue = sanitizedValue.replace(//g, '').replace(/<\/em>/g, ''); + return `${replacedValue} | ${result.type}`; + } + /** * Clear all values, results and errors **/ clear(): void { + console.log("clear"); this.display = ""; // this.value = null; this.results = []; @@ -198,15 +274,17 @@ export default class VsInput extends Vue { console.log("request()"); // DatasetService.searchTerm(this.display, this.solr.core, this.solr.host).subscribe({ DatasetService.searchTerm(this.display).subscribe({ - next: (res: Dataset[]) => this.dataHandler(res), + // next: (res: Dataset[]) => this.dataHandler(res), + next: (res: { datasets: Dataset[], highlights: HitHighlight[] }) => this.dataHandler(res.datasets, res.highlights), error: (error: string) => this.errorHandler(error), complete: () => (this.loading = false), }); } // Handle the search results - private dataHandler(datasets: Dataset[]): void { + private dataHandler(datasets: Dataset[], highlights: HitHighlight[]): void { this.results = datasets; + this.highlights = highlights; // Store highlights // console.log(datasets); // this.$emit("search", this.display); @@ -230,6 +308,7 @@ export default class VsInput extends Vue { // Handle arrow down key press to navigate suggestions onArrowDown(ev: Event): void { + console.log("onArrowDown"); ev.preventDefault(); if (this.selectedIndex === -1) { this.selectedIndex = 0; @@ -251,6 +330,7 @@ export default class VsInput extends Vue { // Handle arrow up key press to navigate suggestions onArrowUp(ev: Event): void { + console.log("onArrowUp"); ev.preventDefault(); if (this.selectedIndex === -1) { this.selectedIndex = this.suggestions.length - 1; @@ -262,6 +342,8 @@ export default class VsInput extends Vue { // Handle enter key press to select a suggestion onEnter(): void { + console.log("onEnter"); + if (this.selectedIndex === -1) { // this.$emit("nothingSelected", this.display); this.display && this.search(); @@ -277,7 +359,10 @@ export default class VsInput extends Vue { // if (!obj) { // return; // } + console.log("select"); this.value = obj; //(obj["title_output"]) ? obj["title_output"] : obj.id + console.log(obj); + this.display = obj.value; // this.formatDisplay(obj) // this.selectedDisplay = this.display; @@ -301,6 +386,7 @@ export default class VsInput extends Vue { * Close the results list. If nothing was selected clear the search */ close(): void { + console.log("close"); if (!this.value) { this.clear(); } diff --git a/src/components/vs-input/vs-input.vue b/src/components/vs-input/vs-input.vue index 85ceb3c..be7e855 100644 --- a/src/components/vs-input/vs-input.vue +++ b/src/components/vs-input/vs-input.vue @@ -56,7 +56,8 @@ >
- + +
diff --git a/src/models/dataset.ts b/src/models/dataset.ts index a9e7c3e..1f8252f 100644 --- a/src/models/dataset.ts +++ b/src/models/dataset.ts @@ -60,17 +60,29 @@ export interface Dataset { export class Suggestion { constructor( - public value: string, - public type: SearchType, + public value: string, // Store the text value returned by OpenSearch + // Store the highlight: i.e. the text value with the emphasised term that generated that results by OpenSearch. + // In this way we can highlight the real term existing in the publication independently of how different was the inserted term used for the FUZZY search. e.g. "Vien" fuzzy matched with "Wien" + public highlight: string, + public type: SearchType, // Type of search element ) {} // value!: string; // type!: SearchType; } +// export class Suggestion { +// constructor( +// public value: string, +// public type: SearchType, +// ) {} +// // value!: string; +// // type!: SearchType; +// } + export enum SearchType { - Title = "title", - Author = "author", - Subject = "subject" + Title = "Title", + Author = "Author", + Subject = "Subject" } export class DbDataset { diff --git a/src/models/headers.ts b/src/models/headers.ts index 69d1c18..bdcf21c 100644 --- a/src/models/headers.ts +++ b/src/models/headers.ts @@ -105,7 +105,7 @@ export interface Hit { _id: string; _score: number; _source: Dataset; - _highlight: HitHighlight; // !! This name is to avoid collision with Typescript "Highlight" class + highlight: HitHighlight; // !! This name is to avoid collision with Typescript "Highlight" class } export interface HitHighlight { diff --git a/src/services/dataset.service.ts b/src/services/dataset.service.ts index f49419c..3d7e687 100644 --- a/src/services/dataset.service.ts +++ b/src/services/dataset.service.ts @@ -3,7 +3,7 @@ import api from "../api/api"; import { Observable } from "rxjs"; import { tap, map } from "rxjs/operators"; import { Dataset, DbDataset, Suggestion } from "@/models/dataset"; -import { OpenSearchResponse, SolrResponse } from "@/models/headers"; +import { HitHighlight, OpenSearchResponse, SolrResponse } from "@/models/headers"; import { ActiveFilterCategories } from "@/models/solr"; import { VUE_API } from "@/constants"; import { deserialize } from "class-transformer"; @@ -147,9 +147,11 @@ class DatasetService { terms%22%2C%20field%3A%20%22year%22%20%7D&json.facet.author=%7B%20type%3A%20%22terms%22%2C%20field%3A%20%22author_facet%22%2C%20limit%3A%20-1%20%7D */ - private openSearchUrl = "http://opensearch.geoinformation.dev/tethys-records/_search"; + // private openSearchUrl = "http://opensearch.geoinformation.dev/tethys-records/_search"; + private openSearchUrl = "http://192.168.21.18/tethys-records/_search"; - public searchTerm(term: string): Observable { + // public searchTerm(term: string): Observable { + public searchTerm(term: string): Observable<{ datasets: Dataset[], highlights: HitHighlight[] }> { const body = { query: { bool: { @@ -189,20 +191,16 @@ class DatasetService { * It is used the pipe method to chain RxJS operators to the Observable returned by api.get. The map operator is used to transform the emitted items of the Observable. */ return api.post(this.openSearchUrl, body).pipe( - // tap(response => console.log("OpenSearchResponse:", response)), // Log the complete response + tap(response => console.log("OpenSearchResponse:", response)), // Log the complete response // tap(response => console.log("Aggre:", response.aggregations?.subjects.buckets[0])), // log the first subject of the array of subjects returned // tap(response => console.log("Hits:", response.hits)), // log the first subject of the array of subjects returned - map(response => response.hits.hits.map(hit => hit._source)) + // map(response => response.hits.hits.map(hit => hit._source)) - // map(response => response.hits.hits.map(hit => { - // const source = hit._source; - // const highlights = hit._highlight || {}; - // return { - // ...source, - // highlights - // }; - // })) + map(response => ({ + datasets: response.hits.hits.map(hit => hit._source), + highlights: response.hits.hits.map(hit => hit.highlight) + })) ); } diff --git a/src/views/search-view/search-view-component.ts b/src/views/search-view/search-view-component.ts index 2bd2bb4..68742da 100644 --- a/src/views/search-view/search-view-component.ts +++ b/src/views/search-view/search-view-component.ts @@ -100,12 +100,15 @@ export default class SearchViewComponent extends Vue { // Lifecycle hook: executed before the component is mounted beforeMount(): void { + console.log("beforeMount!"); + // this.rdrAPI = new DatasetService(); // Trigger search based on provided display and type props if (this.display != "" && this.type != undefined) { const enumKey: "Title" | "Author" | "Subject" | null = this.getEnumKeyByEnumValue(SearchType, this.type); if (enumKey) { - const suggestion = new Suggestion(this.display, SearchType[enumKey]); + const suggestion = new Suggestion(this.display, "NO-IDEA", SearchType[enumKey]); + // const suggestion = new Suggestion(this.display, "" , SearchType[enumKey]); this.onSearch(suggestion); } else { this.onSearch(this.display);