geolba/tethys.frontend
6
0
Fork 1
Code Issues 7 Pull Requests Packages Projects 3 Releases 1 Wiki Activity

OpenSearch: almost implemented, fixing case sensitive issues

Browse Source
This commit is contained in:
Porras-Bernardez 2024-06-11 09:33:03 +02:00
parent 4f53411d07
commit 9b8b2bd5ac
8 changed files with 184 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
package-lock.json generated
View File

@ -14,6 +14,7 @@
"axios": "^1.2.2",
"class-transformer": "^0.5.1",
"dayjs": "^1.10.7",
"dompurify": "^3.1.5",
"leaflet": "^1.7.1",
"qs": "^6.10.1",
"rxjs": "^7.5.5",
@ -29,6 +30,7 @@
"@babel/plugin-proposal-decorators": "^7.22.5",
"@babel/preset-env": "^7.22.5",
"@tailwindcss/forms": "^0.5.7",
"@types/dompurify": "^3.0.5",
"@types/leaflet": "^1.7.9",
"@typescript-eslint/eslint-plugin": "^7.2.0",
"@typescript-eslint/parser": "^7.2.0",
@ -2497,6 +2499,15 @@
"@types/node": "*"
}
},
"node_modules/@types/dompurify": {
"version": "3.0.5",
"resolved": "https://registry.npmjs.org/@types/dompurify/-/dompurify-3.0.5.tgz",
"integrity": "sha512-1Wg0g3BtQF7sSb27fJQAKck1HECM6zV1EB66j8JH9i3LCjYabJa0FSdiSgsD5K/RbrsR0SiraKacLB+T8ZVYAg==",
"dev": true,
"dependencies": {
"@types/trusted-types": "*"
}
},
"node_modules/@types/eslint": {
"version": "8.56.7",
"resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.56.7.tgz",
@ -2694,6 +2705,12 @@
"@types/node": "*"
}
},
"node_modules/@types/trusted-types": {
"version": "2.0.7",
"resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz",
"integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==",
"dev": true
},
"node_modules/@types/webpack-env": {
"version": "1.18.4",
"resolved": "https://registry.npmjs.org/@types/webpack-env/-/webpack-env-1.18.4.tgz",
@ -5626,6 +5643,11 @@
"url": "https://github.com/fb55/domhandler?sponsor=1"
}
},
"node_modules/dompurify": {
"version": "3.1.5",
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.1.5.tgz",
"integrity": "sha512-lwG+n5h8QNpxtyrJW/gJWckL+1/DQiYMX8f7t8Z2AZTPw1esVrqjI63i7Zc2Gz0aKzLVMYC1V1PL/ky+aY/NgA=="
},
"node_modules/domutils": {
"version": "2.8.0",
"resolved": "https://registry.npmjs.org/domutils/-/domutils-2.8.0.tgz",

2
package.json
View File

@ -17,6 +17,7 @@
"axios": "^1.2.2",
"class-transformer": "^0.5.1",
"dayjs": "^1.10.7",
"dompurify": "^3.1.5",
"leaflet": "^1.7.1",
"qs": "^6.10.1",
"rxjs": "^7.5.5",
@ -32,6 +33,7 @@
"@babel/plugin-proposal-decorators": "^7.22.5",
"@babel/preset-env": "^7.22.5",
"@tailwindcss/forms": "^0.5.7",
"@types/dompurify": "^3.0.5",
"@types/leaflet": "^1.7.9",
"@typescript-eslint/eslint-plugin": "^7.2.0",
"@typescript-eslint/parser": "^7.2.0",

162
src/components/vs-input/vs-input.ts
View File

@ -10,6 +10,9 @@ import { Dataset, Suggestion, SearchType } from "@/models/dataset";
import { SOLR_HOST, SOLR_CORE } from "@/constants";
import { OPEN_HOST, OPEN_CORE } from "@/constants"; // PENDING USE
import { HitHighlight } from "@/models/headers";
import DOMPurify from 'dompurify'; // To sanitize the HTML content to prevent XSS attacks!
@Component({
name: "VsInput",
@ -30,6 +33,8 @@ export default class VsInput extends Vue {
private value!: Suggestion | string;
private error = "";
private results: Array<Dataset> = []; // Array to store search results
private highlights: Array<HitHighlight> = [];
private loading = false; // Loading state indicator
private selectedIndex = -1; // Index of the currently selected suggestion
// private selectedDisplay = "";
@ -82,75 +87,146 @@ export default class VsInput extends Vue {
// Computed property to generate suggestions based on search results
get suggestions(): Suggestion[] {
// const suggestion = {
// titles: new Array<string>(),
// authors: new Array<string>(),
// subjects: new Array<string>(),
// };
const suggestions = new Array<Suggestion>();
console.log("Display:", this.display);
console.log("results:", this.results );
console.log("Suggestions > Display:", this.display);
// console.log("results:", this.results );
// console.log("highlights:", this.highlights);
//The method checks if there are any highlighted titles in the highlight object. If found, it joins the highlighted fragments into a single string
// Generate suggestions based on search results
this.results.forEach((dataset) => {
this.results.forEach((dataset, index) => {
let foundAny = false;
const highlight = this.highlights[index];
console.log("get suggestions:id", dataset.id);
console.log("get suggestions:title_output", dataset.title_output);
console.log("get suggestions:author", dataset.author);
console.log("get suggestions:subjects", dataset.subjects);
// console.log("get suggestions:title_output", dataset.title_output);
// console.log("get suggestions:author", dataset.author);
// console.log("get suggestions:subjects", dataset.subjects);
// const del = dataset.title_output?.toLowerCase();
if (dataset.title_output.toLowerCase().includes(this.display.toLowerCase())) {
const title = dataset.title_output;
// if (!suggestion["titles"].find((value) => value === title)) {
// suggestion.titles.push(title);
// }
// Check if there is already a suggestion with this title and type
const hasTitleSuggestion = suggestions.some((suggestion) => suggestion.value === title && suggestion.type == SearchType.Title);
// Checks if a suggestion with the same title and type already exists in the suggestions array. If not, it creates a new Suggestion object and adds it to the suggestions array.
if (highlight.title && highlight.title.length > 0) {
/** This line checks if the highlight object has a title property and if that property is an array with at least one element.
* The highlight object contains highlighted fragments of the search term in various fields (e.g., title, author, subjects) as returned by the OpenSearch API.
* This check ensures that we only process results that have highlighted titles. */
const highlightedTitle = highlight.title.join(" ");
/**
* The highlight.title property is an array of strings, where each string is a highlighted fragment of the title. join(" ") combines these fragments into a single string with spaces between them.
* This step constructs a full highlighted title from the individual fragments.
* OpenSearch can return multiple fragments of a field (like the title) in its response, especially when the field contains multiple terms that match the search query.
* This can happen because OpenSearch's highlighting feature is designed to provide context around each match within the field, which can result in multiple highlighted fragments.
*/
const hasTitleSuggestion = suggestions.some((suggestion) => suggestion.highlight.toLowerCase() === highlightedTitle.toLowerCase() && suggestion.type == SearchType.Title);
if (!hasTitleSuggestion) {
// If there is no such suggestion, create a new one and add it to the suggestions array
const suggestion = new Suggestion(title, SearchType.Title);
const suggestion = new Suggestion(dataset.title_output, highlightedTitle, SearchType.Title);
suggestions.push(suggestion);
foundAny = true;
}
}
if (this.find(dataset.author, this.display.toLowerCase()) !== "") {
const author = this.find(dataset.author, this.display.toLowerCase());
// Check if there is already a suggestion with this author and type
const hasAuthorSuggestion = suggestions.some((suggestion) => suggestion.value === author && suggestion.type == SearchType.Author);
if (highlight.author && highlight.author.length > 0) {
const highlightedAuthor = highlight.author.join(" ");
const datasetAuthor = this.find(dataset.author, this.display.toLowerCase());
const hasAuthorSuggestion = suggestions.some((suggestion) => suggestion.highlight === highlightedAuthor && suggestion.type == SearchType.Author);
if (!hasAuthorSuggestion) {
const suggestion = new Suggestion(author, SearchType.Author);
const suggestion = new Suggestion(datasetAuthor, highlightedAuthor, SearchType.Author);
suggestions.push(suggestion);
foundAny = true;
}
}
if (this.find(dataset.subjects, this.display.toLowerCase()) != "") {
const subject = this.find(dataset.subjects, this.display.toLowerCase());
const hasSubjectSuggestion = suggestions.some((suggestion) => suggestion.value === subject && suggestion.type == SearchType.Subject);
if (highlight.subjects && highlight.subjects.length > 0) {
const highlightedSubject = highlight.subjects.join(" ");
const datasetSubject = this.find(dataset.subjects, this.display.toLowerCase());
const hasSubjectSuggestion = suggestions.some((suggestion) => suggestion.highlight === highlightedSubject && suggestion.type == SearchType.Subject);
if (!hasSubjectSuggestion) {
const suggestion = new Suggestion(subject, SearchType.Subject);
const suggestion = new Suggestion(datasetSubject, highlightedSubject, SearchType.Subject);
suggestions.push(suggestion);
foundAny = true;
}
}
// if (!foundAny) {
// const suggestion = new Suggestion(dataset.title_output, SearchType.Fuzzy);
// // Checks if a suggestion with the same title and type already exists in the suggestions array. If not, it creates a new Suggestion object and adds it to the suggestions array.
// if (highlight.title && highlight.title.length > 0) {
// /** This line checks if the highlight object has a title property and if that property is an array with at least one element.
// * The highlight object contains highlighted fragments of the search term in various fields (e.g., title, author, subjects) as returned by the OpenSearch API.
// * This check ensures that we only process results that have highlighted titles. */
// const title = highlight.title.join(" ");
// /**
// * The highlight.title property is an array of strings, where each string is a highlighted fragment of the title. join(" ") combines these fragments into a single string with spaces between them.
// * This step constructs a full highlighted title from the individual fragments.
// * OpenSearch can return multiple fragments of a field (like the title) in its response, especially when the field contains multiple terms that match the search query.
// * This can happen because OpenSearch's highlighting feature is designed to provide context around each match within the field, which can result in multiple highlighted fragments.
// */
// const hasTitleSuggestion = suggestions.some((suggestion) => suggestion.value === title && suggestion.type == SearchType.Title);
// if (!hasTitleSuggestion) {
// const suggestion = new Suggestion(title, SearchType.Title);
// suggestions.push(suggestion);
// }
// }
// if (highlight.author && highlight.author.length > 0) {
// const author = highlight.author.join(" ");
// const hasAuthorSuggestion = suggestions.some((suggestion) => suggestion.value === author && suggestion.type == SearchType.Author);
// if (!hasAuthorSuggestion) {
// const suggestion = new Suggestion(author, SearchType.Author);
// suggestions.push(suggestion);
// }
// }
// if (highlight.subjects && highlight.subjects.length > 0) {
// const subject = highlight.subjects.join(" ");
// const hasSubjectSuggestion = suggestions.some((suggestion) => suggestion.value === subject && suggestion.type == SearchType.Subject);
// if (!hasSubjectSuggestion) {
// const suggestion = new Suggestion(subject, SearchType.Subject);
// suggestions.push(suggestion);
// }
// }
// ORIGINAL SOLR ===================================================================================================
// if (dataset.title_output.toLowerCase().includes(this.display.toLowerCase())) {
// const title = dataset.title_output;
// // Check if there is already a suggestion with this title and type
// const hasTitleSuggestion = suggestions.some((suggestion) => suggestion.value === title && suggestion.type == SearchType.Title);
// if (!hasTitleSuggestion) {
// // If there is no such suggestion, create a new one and add it to the suggestions array
// const suggestion = new Suggestion(title, SearchType.Title);
// suggestions.push(suggestion);
// }
// }
// if (this.find(dataset.author, this.display.toLowerCase()) !== "") {
// const author = this.find(dataset.author, this.display.toLowerCase());
// // Check if there is already a suggestion with this author and type
// const hasAuthorSuggestion = suggestions.some((suggestion) => suggestion.value === author && suggestion.type == SearchType.Author);
// if (!hasAuthorSuggestion) {
// const suggestion = new Suggestion(author, SearchType.Author);
// suggestions.push(suggestion);
// }
// }
// if (this.find(dataset.subjects, this.display.toLowerCase()) != "") {
// const subject = this.find(dataset.subjects, this.display.toLowerCase());
// const hasSubjectSuggestion = suggestions.some((suggestion) => suggestion.value === subject && suggestion.type == SearchType.Subject);
// if (!hasSubjectSuggestion) {
// const suggestion = new Suggestion(subject, SearchType.Subject);
// suggestions.push(suggestion);
// }
// }
});
return suggestions;
}
/**
* This method combines the suggestion value and type into a single HTML string. It also sanitizes the HTML content using DOMPurify to prevent XSS attacks.
* The vue file uses the v-html directive to bind the combined HTML string to the label element. This ensures that the HTML content (e.g., <em>Wien</em>) is rendered correctly in the browser.
*/
formatSuggestion(result: Suggestion): string {
const sanitizedValue = DOMPurify.sanitize(result.highlight);
// Replacing the predefined format for highlights given by OpenSearch from <em> emphasys to <b> bold
const replacedValue = sanitizedValue.replace(/<em>/g, '<b>').replace(/<\/em>/g, '</b>');
return `${replacedValue} <em>| ${result.type}</em>`;
}
/**
* Clear all values, results and errors
**/
clear(): void {
console.log("clear");
this.display = "";
// this.value = null;
this.results = [];
@ -198,15 +274,17 @@ export default class VsInput extends Vue {
console.log("request()");
// DatasetService.searchTerm(this.display, this.solr.core, this.solr.host).subscribe({
DatasetService.searchTerm(this.display).subscribe({
next: (res: Dataset[]) => this.dataHandler(res),
// next: (res: Dataset[]) => this.dataHandler(res),
next: (res: { datasets: Dataset[], highlights: HitHighlight[] }) => this.dataHandler(res.datasets, res.highlights),
error: (error: string) => this.errorHandler(error),
complete: () => (this.loading = false),
});
}
// Handle the search results
private dataHandler(datasets: Dataset[]): void {
private dataHandler(datasets: Dataset[], highlights: HitHighlight[]): void {
this.results = datasets;
this.highlights = highlights; // Store highlights
// console.log(datasets);
// this.$emit("search", this.display);
@ -230,6 +308,7 @@ export default class VsInput extends Vue {
// Handle arrow down key press to navigate suggestions
onArrowDown(ev: Event): void {
console.log("onArrowDown");
ev.preventDefault();
if (this.selectedIndex === -1) {
this.selectedIndex = 0;
@ -251,6 +330,7 @@ export default class VsInput extends Vue {
// Handle arrow up key press to navigate suggestions
onArrowUp(ev: Event): void {
console.log("onArrowUp");
ev.preventDefault();
if (this.selectedIndex === -1) {
this.selectedIndex = this.suggestions.length - 1;
@ -262,6 +342,8 @@ export default class VsInput extends Vue {
// Handle enter key press to select a suggestion
onEnter(): void {
console.log("onEnter");
if (this.selectedIndex === -1) {
// this.$emit("nothingSelected", this.display);
this.display && this.search();
@ -277,7 +359,10 @@ export default class VsInput extends Vue {
// if (!obj) {
// return;
// }
console.log("select");
this.value = obj; //(obj["title_output"]) ? obj["title_output"] : obj.id
console.log(obj);
this.display = obj.value; // this.formatDisplay(obj)
// this.selectedDisplay = this.display;
@ -301,6 +386,7 @@ export default class VsInput extends Vue {
* Close the results list. If nothing was selected clear the search
*/
close(): void {
console.log("close");
if (!this.value) {
this.clear();
}

3
src/components/vs-input/vs-input.vue
View File

@ -56,7 +56,8 @@
>
<!-- Displaying suggestion result -->
<div class="small-label">
<label>{{ result.value }} ({{ result.type }})</label>
<!-- <label>{{ result.value }} ({{ result.type }})</label> -->
<label v-html="formatSuggestion(result)"></label>
</div>
</li>
</ul>

22
src/models/dataset.ts
View File

@ -60,17 +60,29 @@ export interface Dataset {
export class Suggestion {
constructor(
public value: string,
public type: SearchType,
public value: string, // Store the text value returned by OpenSearch
// Store the highlight: i.e. the text value with the emphasised term that generated that results by OpenSearch.
// In this way we can highlight the real term existing in the publication independently of how different was the inserted term used for the FUZZY search. e.g. "Vien" fuzzy matched with "Wien"
public highlight: string,
public type: SearchType, // Type of search element
) {}
// value!: string;
// type!: SearchType;
}
// export class Suggestion {
// constructor(
// public value: string,
// public type: SearchType,
// ) {}
// // value!: string;
// // type!: SearchType;
// }
export enum SearchType {
Title = "title",
Author = "author",
Subject = "subject"
Title = "Title",
Author = "Author",
Subject = "Subject"
}
export class DbDataset {

2
src/models/headers.ts
View File

@ -105,7 +105,7 @@ export interface Hit {
_id: string;
_score: number;
_source: Dataset;
_highlight: HitHighlight; // !! This name is to avoid collision with Typescript "Highlight" class
highlight: HitHighlight; // !! This name is to avoid collision with Typescript "Highlight" class
}
export interface HitHighlight {

24
src/services/dataset.service.ts
View File

@ -3,7 +3,7 @@ import api from "../api/api";
import { Observable } from "rxjs";
import { tap, map } from "rxjs/operators";
import { Dataset, DbDataset, Suggestion } from "@/models/dataset";
import { OpenSearchResponse, SolrResponse } from "@/models/headers";
import { HitHighlight, OpenSearchResponse, SolrResponse } from "@/models/headers";
import { ActiveFilterCategories } from "@/models/solr";
import { VUE_API } from "@/constants";
import { deserialize } from "class-transformer";
@ -147,9 +147,11 @@ class DatasetService {
terms%22%2C%20field%3A%20%22year%22%20%7D&json.facet.author=%7B%20type%3A%20%22terms%22%2C%20field%3A%20%22author_facet%22%2C%20limit%3A%20-1%20%7D
*/
private openSearchUrl = "http://opensearch.geoinformation.dev/tethys-records/_search";
// private openSearchUrl = "http://opensearch.geoinformation.dev/tethys-records/_search";
private openSearchUrl = "http://192.168.21.18/tethys-records/_search";
public searchTerm(term: string): Observable<Dataset[]> {
// public searchTerm(term: string): Observable<Dataset[]> {
public searchTerm(term: string): Observable<{ datasets: Dataset[], highlights: HitHighlight[] }> {
const body = {
query: {
bool: {
@ -189,20 +191,16 @@ class DatasetService {
* It is used the pipe method to chain RxJS operators to the Observable returned by api.get. The map operator is used to transform the emitted items of the Observable.
*/
return api.post<OpenSearchResponse>(this.openSearchUrl, body).pipe(
// tap(response => console.log("OpenSearchResponse:", response)), // Log the complete response
tap(response => console.log("OpenSearchResponse:", response)), // Log the complete response
// tap(response => console.log("Aggre:", response.aggregations?.subjects.buckets[0])), // log the first subject of the array of subjects returned
// tap(response => console.log("Hits:", response.hits)), // log the first subject of the array of subjects returned
map(response => response.hits.hits.map(hit => hit._source))
// map(response => response.hits.hits.map(hit => hit._source))
// map(response => response.hits.hits.map(hit => {
// const source = hit._source;
// const highlights = hit._highlight || {};
// return {
// ...source,
// highlights
// };
// }))
map(response => ({
datasets: response.hits.hits.map(hit => hit._source),
highlights: response.hits.hits.map(hit => hit.highlight)
}))
);
}

5
src/views/search-view/search-view-component.ts
View File

@ -100,12 +100,15 @@ export default class SearchViewComponent extends Vue {
// Lifecycle hook: executed before the component is mounted
beforeMount(): void {
console.log("beforeMount!");
// this.rdrAPI = new DatasetService();
// Trigger search based on provided display and type props
if (this.display != "" && this.type != undefined) {
const enumKey: "Title" | "Author" | "Subject" | null = this.getEnumKeyByEnumValue(SearchType, this.type);
if (enumKey) {
const suggestion = new Suggestion(this.display, SearchType[enumKey]);
const suggestion = new Suggestion(this.display, "NO-IDEA", SearchType[enumKey]);
// const suggestion = new Suggestion(this.display, "" , SearchType[enumKey]);
this.onSearch(suggestion);
} else {
this.onSearch(this.display);