From 4f53411d0774798ba3b2f0e95dc9dc98382c96f7 Mon Sep 17 00:00:00 2001 From: frankporras Date: Fri, 7 Jun 2024 17:44:13 +0200 Subject: [PATCH] Opensearch progress. - Term search works - Pending faceted search - Ongoing: highlight of fuzzy results --- src/api/api.ts | 2 +- src/components/vs-input/vs-input.ts | 83 ++++++--- src/models/dataset.ts | 103 ++++++----- src/models/headers.ts | 30 ++- src/services/dataset.service.ts | 278 +++++++++++++++------------- 5 files changed, 292 insertions(+), 204 deletions(-) diff --git a/src/api/api.ts b/src/api/api.ts index 1b0c5f6..8a7db0a 100644 --- a/src/api/api.ts +++ b/src/api/api.ts @@ -30,5 +30,5 @@ const post = (url: string, body: any, queryParams?: any): Observable => { .pipe(map((result: AxiosResponse) => result.data)); // Use map to transform the Axios response to extract the data property }; -// Export the get function as part of the default export +// Export the get and post functions as part of the default export export default { get, post }; \ No newline at end of file diff --git a/src/components/vs-input/vs-input.ts b/src/components/vs-input/vs-input.ts index f4dba9e..04183e7 100644 --- a/src/components/vs-input/vs-input.ts +++ b/src/components/vs-input/vs-input.ts @@ -2,16 +2,14 @@ // import debounce from 'lodash/debounce'; // import { DatasetService } from "../../services/dataset.service"; import DatasetService from "../../services/dataset.service"; -import { SolrSettings } from "@/models/solr"; +import { SolrSettings } from "@/models/solr"; // PENDING USE import { OpenSettings } from "@/models/solr"; -// import { ref } from "vue"; import { Component, Vue, Prop, Emit } from "vue-facing-decorator"; -// import { Prop, Emit } from "vue-property-decorator"; import { Dataset, Suggestion, SearchType } from "@/models/dataset"; import { SOLR_HOST, SOLR_CORE } from "@/constants"; -import { OPEN_HOST, OPEN_CORE } from "@/constants"; +import { OPEN_HOST, OPEN_CORE } from "@/constants"; // PENDING USE @Component({ name: "VsInput", @@ -20,19 +18,20 @@ export default class VsInput extends Vue { // @Prop() // private title!: string; + // Define the placeholder text for the input field @Prop({ default: "Search" }) readonly placeholder!: string; - private display = ""; + private display = ""; // Input display value @Prop() private propDisplay = ""; private value!: Suggestion | string; private error = ""; - private results: Array = []; - private loading = false; - private selectedIndex = -1; + private results: Array = []; // Array to store search results + private loading = false; // Loading state indicator + private selectedIndex = -1; // Index of the currently selected suggestion // private selectedDisplay = ""; private solr: SolrSettings = { core: SOLR_CORE, //"rdr_data", // SOLR.core; @@ -49,9 +48,10 @@ export default class VsInput extends Vue { }; // private rdrAPI!: DatasetService; - itemRefs!: Array; - emits = ["filter"]; + itemRefs!: Array; // Array to store references to suggestion items + emits = ["filter"]; // Emits filter event + // Set reference for each item setItemRef(el: Element): void { this.itemRefs.push(el); } @@ -80,6 +80,7 @@ export default class VsInput extends Vue { return this.error !== null; } + // Computed property to generate suggestions based on search results get suggestions(): Suggestion[] { // const suggestion = { // titles: new Array(), @@ -89,12 +90,17 @@ export default class VsInput extends Vue { const suggestions = new Array(); console.log("Display:", this.display); - // console.log("results:", this.results ); + console.log("results:", this.results ); - + // Generate suggestions based on search results this.results.forEach((dataset) => { - console.log("suggestions:foreach:", dataset.id); + let foundAny = false; + + console.log("get suggestions:id", dataset.id); + console.log("get suggestions:title_output", dataset.title_output); + console.log("get suggestions:author", dataset.author); + console.log("get suggestions:subjects", dataset.subjects); // const del = dataset.title_output?.toLowerCase(); if (dataset.title_output.toLowerCase().includes(this.display.toLowerCase())) { @@ -102,29 +108,40 @@ export default class VsInput extends Vue { // if (!suggestion["titles"].find((value) => value === title)) { // suggestion.titles.push(title); // } + // Check if there is already a suggestion with this title and type const hasTitleSuggestion = suggestions.some((suggestion) => suggestion.value === title && suggestion.type == SearchType.Title); if (!hasTitleSuggestion) { + // If there is no such suggestion, create a new one and add it to the suggestions array const suggestion = new Suggestion(title, SearchType.Title); suggestions.push(suggestion); + foundAny = true; + } + } + if (this.find(dataset.author, this.display.toLowerCase()) !== "") { + const author = this.find(dataset.author, this.display.toLowerCase()); + // Check if there is already a suggestion with this author and type + const hasAuthorSuggestion = suggestions.some((suggestion) => suggestion.value === author && suggestion.type == SearchType.Author); + if (!hasAuthorSuggestion) { + const suggestion = new Suggestion(author, SearchType.Author); + suggestions.push(suggestion); + foundAny = true; + } + } + if (this.find(dataset.subjects, this.display.toLowerCase()) != "") { + const subject = this.find(dataset.subjects, this.display.toLowerCase()); + const hasSubjectSuggestion = suggestions.some((suggestion) => suggestion.value === subject && suggestion.type == SearchType.Subject); + if (!hasSubjectSuggestion) { + const suggestion = new Suggestion(subject, SearchType.Subject); + suggestions.push(suggestion); + foundAny = true; } } - // if (this.find(dataset.author, this.display.toLowerCase()) !== "") { - // const author = this.find(dataset.author, this.display.toLowerCase()); - // const hasAuthorSuggestion = suggestions.some((suggestion) => suggestion.value === author && suggestion.type == SearchType.Author); - // if (!hasAuthorSuggestion) { - // const suggestion = new Suggestion(author, SearchType.Author); - // suggestions.push(suggestion); - // } - // } - // if (this.find(dataset.subject, this.display.toLowerCase()) != "") { - // const subject = this.find(dataset.subject, this.display.toLowerCase()); - // const hasSubjectSuggestion = suggestions.some((suggestion) => suggestion.value === subject && suggestion.type == SearchType.Subject); - // if (!hasSubjectSuggestion) { - // const suggestion = new Suggestion(subject, SearchType.Subject); - // suggestions.push(suggestion); - // } + // if (!foundAny) { + // const suggestion = new Suggestion(dataset.title_output, SearchType.Fuzzy); + // suggestions.push(suggestion); // } + }); return suggestions; @@ -151,6 +168,7 @@ export default class VsInput extends Vue { return this.display; } + // Handler for search input change searchChanged(): void { console.log("Search changed!"); this.selectedIndex = -1; @@ -164,6 +182,7 @@ export default class VsInput extends Vue { } } + // Perform the search request private resourceSearch() { if (!this.display) { this.results = []; @@ -174,6 +193,7 @@ export default class VsInput extends Vue { this.request(); } + // Make the API request to search for datasets private request(): void { console.log("request()"); // DatasetService.searchTerm(this.display, this.solr.core, this.solr.host).subscribe({ @@ -184,6 +204,7 @@ export default class VsInput extends Vue { }); } + // Handle the search results private dataHandler(datasets: Dataset[]): void { this.results = datasets; // console.log(datasets); @@ -192,6 +213,7 @@ export default class VsInput extends Vue { // this.loading = false; } + // Handle errors from the search request private errorHandler(err: string): void { this.error = err; // this.loading = false; @@ -206,6 +228,7 @@ export default class VsInput extends Vue { return key === this.selectedIndex; } + // Handle arrow down key press to navigate suggestions onArrowDown(ev: Event): void { ev.preventDefault(); if (this.selectedIndex === -1) { @@ -216,6 +239,7 @@ export default class VsInput extends Vue { this.fixScrolling(); } + // Scroll the selected suggestion into view private fixScrolling() { const currentElement = this.itemRefs[this.selectedIndex]; currentElement.scrollIntoView({ @@ -225,6 +249,7 @@ export default class VsInput extends Vue { }); } + // Handle arrow up key press to navigate suggestions onArrowUp(ev: Event): void { ev.preventDefault(); if (this.selectedIndex === -1) { @@ -235,6 +260,7 @@ export default class VsInput extends Vue { this.fixScrolling(); } + // Handle enter key press to select a suggestion onEnter(): void { if (this.selectedIndex === -1) { // this.$emit("nothingSelected", this.display); @@ -260,6 +286,7 @@ export default class VsInput extends Vue { return this.value; } + // Find a search term in an array private find(myarray: Array, searchterm: string): string { for (let i = 0, len = myarray.length; i < len; i += 1) { if (typeof myarray[i] === "string" && myarray[i].toLowerCase().indexOf(searchterm) !== -1) { diff --git a/src/models/dataset.ts b/src/models/dataset.ts index c3de319..a9e7c3e 100644 --- a/src/models/dataset.ts +++ b/src/models/dataset.ts @@ -1,27 +1,63 @@ // import moment from "moment"; import dayjs from "dayjs"; +// // SOLR Dataset original +// export interface Dataset { +// abstract_additional: Array;// OpenSearch: abstract: Array +// abstract_output: string;// ----- +// author: Array;// EQUAL +// author_sort: Array;// ----- +// belongs_to_bibliography: boolean;// EQUAL +// creating_corporation: string;// EQUAL +// doctype: string;// EQUAL +// geo_location: string;// EQUAL +// id: number;// EQUAL +// identifier: Identifier;// OpenSearch: identifier: Array +// language: string;// EQUAL +// licence: string;// EQUAL +// publisher_name: string;// EQUAL +// server_date_published: Array;// OpenSearch not array! +// subject: Array;// OpenSearch: subjectS +// title_output: string;// EQUAL +// year: number;// EQUAL +// year_inverted: number;// EQUAL +// } + +// OpenSearch Dataset export interface Dataset { - abstract_additional: Array; - abstract_output: string; - author: Array; - author_sort: Array; - belongs_to_bibliography: boolean; - creating_corporation: string; - doctype: string; - geo_location: string; - id: number; - identifier: Identifier; - language: string; - licence: string; - publisher_name: string; - server_date_published: Array; - subject: Array; - title_output: string; - year: number; - year_inverted: number; + abstract: Array;// OpenSearch: abstract: Array + // abstract_output: string;// ----- NOT in OpenSearch + author: Array;// EQUAL + // author_sort: Array;// ----- NOT in OpenSearch + belongs_to_bibliography: boolean;// EQUAL + creating_corporation: string;// EQUAL + doctype: string;// EQUAL + geo_location: string;// EQUAL + id: number;// EQUAL + // identifier: Identifier;// OpenSearch: identifier: Array + identifier: Array// DIFF DATATYPE + language: string;// EQUAL + licence: string;// EQUAL + publisher_name: string;// EQUAL + // server_date_published: Array;// OpenSearch string! + server_date_published: string;// DIFF DATATYPE + // subject: Array;// OpenSearch: subjectS + subjects: Array;// DIFF DATATYPE + title_output: string;// EQUAL + year: number;// EQUAL + year_inverted: number;// EQUAL + + title: string // Unique in OpenSearch + title_additional: Array // Unique in OpenSearch + bbox_xmin: string // Unique in OpenSearch + bbox_xmax: string // Unique in OpenSearch + bbox_ymin: string // Unique in OpenSearch + bbox_ymax: string // Unique in OpenSearch + reference: Array // Unique in OpenSearch + abstract_additional: Array;// Unique in OpenSearch } + export class Suggestion { constructor( public value: string, @@ -34,39 +70,10 @@ export class Suggestion { export enum SearchType { Title = "title", Author = "author", - Subject = "subject", + Subject = "subject" } export class DbDataset { - // public id!: number; - // public url!: string; - // public contributing_corporation!: string; - // public creating_corporation!: string; - // public publisher_name!: string; - // public embargo_date!: string; - // public publish_id!: number; - // public project_id!: number; - // public type!: string; - // public language!: string; - // public server_state!: string; - // public belongs_to_bibliography!: boolean; - // public created_at!: string; - // public server_date_modified!: string; - // public server_date_published!: string; - // public account_id!: number; - // public editor_id!: number; - // public reviewer_id!: number; - // public preferred_reviewer!: number; - // public preferred_reviewer_email!: string; - // public reject_editor_note!: string; - // public reject_reviewer_note!: string; - // public reviewer_note_visible!: string; - // public titles!: Array; - // public abstracts!: Array<Abstract>; - // public authors!: Array<Author>; - // public contributors!: Array<Author>; - // public user!: Person; - // public subjects!: Array<Subject>; constructor( public id: string, diff --git a/src/models/headers.ts b/src/models/headers.ts index 0de9230..69d1c18 100644 --- a/src/models/headers.ts +++ b/src/models/headers.ts @@ -105,13 +105,29 @@ export interface Hit { _id: string; _score: number; _source: Dataset; + _highlight: HitHighlight; // !! This name is to avoid collision with Typescript "Highlight" class +} + +export interface HitHighlight { + subjects?: Array<string>; + title?: Array<string>; + author?: Array<string>; } export interface Aggregations { - [key: string]: Aggregation; + subjects: Subjects; + language: Language; } -export interface Aggregation { +export interface Subjects { + doc_count_error_upper_bound: number; + sum_other_doc_count: number; + buckets: Array<Bucket>; +} + +export interface Language { + doc_count_error_upper_bound: number; + sum_other_doc_count: number; buckets: Array<Bucket>; } @@ -119,3 +135,13 @@ export interface Bucket { key: string; doc_count: number; } +// // Needed? +// export interface Aggregations { +// [key: string]: Aggregation; +// } + +// export interface Aggregation { +// buckets: Array<Bucket>; +// } + + diff --git a/src/services/dataset.service.ts b/src/services/dataset.service.ts index 9aec58e..f49419c 100644 --- a/src/services/dataset.service.ts +++ b/src/services/dataset.service.ts @@ -1,7 +1,7 @@ import api from "../api/api"; // import { Observable, of } from "rxjs"; import { Observable } from "rxjs"; -import { map } from "rxjs/operators"; +import { tap, map } from "rxjs/operators"; import { Dataset, DbDataset, Suggestion } from "@/models/dataset"; import { OpenSearchResponse, SolrResponse } from "@/models/headers"; import { ActiveFilterCategories } from "@/models/solr"; @@ -16,130 +16,130 @@ class DatasetService { * * @param {string} searchTerm - The search term to query. */ - async fetchDataFromOpenSearch(searchTerm: string): Promise<void> { - // Define the OpenSearch endpoint URL - const url = "http://opensearch.geoinformation.dev/tethys-records/_search"; + // async fetchDataFromOpenSearch(searchTerm: string): Promise<void> { + // // Define the OpenSearch endpoint URL + // const url = "http://opensearch.geoinformation.dev/tethys-records/_search"; - // Set the headers for the POST request - const headers = { - "Content-Type": "application/json", - }; + // // Set the headers for the POST request + // const headers = { + // "Content-Type": "application/json", + // }; - // Construct the body of the POST request - const body = { - query: { - bool: { - // The `should` clause specifies that at least one of these conditions must match - should: [ - { - // Match the search term in the title field with fuzziness enabled and a boost of 3 - match: { - title: { - query: searchTerm, - fuzziness: "AUTO", // Enable fuzzy search - boost: 3 // Boosting the relevance of title matches - } - } - }, - { - // Match the search term in the author field with fuzziness enabled and a boost of 2 - match: { - author: { - query: searchTerm, - fuzziness: "AUTO", // Enable fuzzy search - boost: 2 // Boosting the relevance of author matches - } - } - }, - { - // Match the search term in the subject field with fuzziness enabled and a boost of 1 - match: { - subject: { - query: searchTerm, - fuzziness: "AUTO", // Enable fuzzy search - boost: 1 // Boosting the relevance of subject matches - } - } - }, - { - // Match the search term in the title field with a wildcard - wildcard: { - title: { - value: `${searchTerm}*`, // Wildcard search for terms starting with searchTerm - boost: 3 // Boosting the relevance of title matches - } - } - }, - { - // Match the search term in the author field with a wildcard - wildcard: { - author: { - value: `${searchTerm}*`, // Wildcard search for terms starting with searchTerm - boost: 2 // Boosting the relevance of author matches - } - } - }, - { - // Match the search term in the subject field with a wildcard - wildcard: { - subject: { - value: `${searchTerm}*`, // Wildcard search for terms starting with searchTerm - boost: 1 // Boosting the relevance of subject matches - } - } - } - ], - // Ensure that at least one of the `should` clauses must match - minimum_should_match: 1 - } - }, - // Limit the number of search results to 10 - size: 10, - // Start from the first result (pagination) - from: 0, - // Sort the results by the `server_date_published` field in descending order - sort: [ - { server_date_published: { order: "desc" } } - ], - // Aggregations to provide facets for the `language` and `subject` fields - aggs: { - language: { - terms: { - field: "language.keyword" // Aggregate by the exact values of the `language` field - } - }, - subject: { - terms: { - field: "subjects.keyword", // Aggregate by the exact values of the `subjects` field - size: 10 // Limit the number of aggregation buckets to 10 - } - } - } - }; + // // Construct the body of the POST request + // const body = { + // query: { + // bool: { + // // The `should` clause specifies that at least one of these conditions must match + // should: [ + // { + // // Match the search term in the title field with fuzziness enabled and a boost of 3 + // match: { + // title: { + // query: searchTerm, + // fuzziness: "AUTO", // Enable fuzzy search + // boost: 3 // Boosting the relevance of title matches + // } + // } + // }, + // { + // // Match the search term in the author field with fuzziness enabled and a boost of 2 + // match: { + // author: { + // query: searchTerm, + // fuzziness: "AUTO", // Enable fuzzy search + // boost: 2 // Boosting the relevance of author matches + // } + // } + // }, + // { + // // Match the search term in the subject field with fuzziness enabled and a boost of 1 + // match: { + // subject: { + // query: searchTerm, + // fuzziness: "AUTO", // Enable fuzzy search + // boost: 1 // Boosting the relevance of subject matches + // } + // } + // }, + // { + // // Match the search term in the title field with a wildcard + // wildcard: { + // title: { + // value: `${searchTerm}*`, // Wildcard search for terms starting with searchTerm + // boost: 3 // Boosting the relevance of title matches + // } + // } + // }, + // { + // // Match the search term in the author field with a wildcard + // wildcard: { + // author: { + // value: `${searchTerm}*`, // Wildcard search for terms starting with searchTerm + // boost: 2 // Boosting the relevance of author matches + // } + // } + // }, + // { + // // Match the search term in the subject field with a wildcard + // wildcard: { + // subject: { + // value: `${searchTerm}*`, // Wildcard search for terms starting with searchTerm + // boost: 1 // Boosting the relevance of subject matches + // } + // } + // } + // ], + // // Ensure that at least one of the `should` clauses must match + // minimum_should_match: 1 + // } + // }, + // // Limit the number of search results to 10 + // size: 10, + // // Start from the first result (pagination) + // from: 0, + // // Sort the results by the `server_date_published` field in descending order + // sort: [ + // { server_date_published: { order: "desc" } } + // ], + // // Aggregations to provide facets for the `language` and `subject` fields + // aggs: { + // language: { + // terms: { + // field: "language.keyword" // Aggregate by the exact values of the `language` field + // } + // }, + // subject: { + // terms: { + // field: "subjects.keyword", // Aggregate by the exact values of the `subjects` field + // size: 10 // Limit the number of aggregation buckets to 10 + // } + // } + // } + // }; - try { - // Send the POST request to the OpenSearch endpoint - const response = await fetch(url, { - method: "POST", - headers: headers, - body: JSON.stringify(body), - }); + // try { + // // Send the POST request to the OpenSearch endpoint + // const response = await fetch(url, { + // method: "POST", + // headers: headers, + // body: JSON.stringify(body), + // }); - // Check if the response is not successful - if (!response.ok) { - throw new Error(`Failed to fetch data from ${url}, status: ${response.status}`); - } + // // Check if the response is not successful + // if (!response.ok) { + // throw new Error(`Failed to fetch data from ${url}, status: ${response.status}`); + // } - // Parse the response JSON - const data = await response.json(); - // Log the data from OpenSearch - console.log("Data from OpenSearch:", data); - console.log("Hits:", data.hits.total.value); - } catch (error) { - // Log any errors that occur during the fetch process - console.error("Error fetching data:", error); - } - } + // // Parse the response JSON + // const data = await response.json(); + // // Log the data from OpenSearch + // console.log("Data from OpenSearch:", data); + // console.log("Hits:", data.hits.total.value); + // } catch (error) { + // // Log any errors that occur during the fetch process + // console.error("Error fetching data:", error); + // } + // } /* https://tethys.at/solr/rdr_data/select?&0=fl%3Did%2Clicence%2Cserver_date_published%2Cabstract_output%2Cidentifier%2Ctitle_output%2Ctitle_additional%2Cauthor%2Csubject%2Cdoctype&q=%2A &q.op=or&defType=edismax&qf=title%5E3%20author%5E2%20subject%5E1&indent=on&wt=json&rows=10&start=0&sort=server_date_published%20desc&facet=on&json.facet.language=%7B%20type%3A%20%22 @@ -156,25 +156,53 @@ class DatasetService { should: [ { match: { title: { query: term, fuzziness: "AUTO", boost: 3 } } }, { match: { author: { query: term, fuzziness: "AUTO", boost: 2 } } }, - { match: { subject: { query: term, fuzziness: "AUTO", boost: 1 } } }, + { match: { subjects: { query: term, fuzziness: "AUTO", boost: 1 } } }, // In SOLR is "subject"! { wildcard: { title: { value: `${term}*`, boost: 3 } } }, { wildcard: { author: { value: `${term}*`, boost: 2 } } }, - { wildcard: { subject: { value: `${term}*`, boost: 1 } } } + { wildcard: { subjects: { value: `${term}*`, boost: 1 } } } // In SOLR is "subject"! ], minimum_should_match: 1 } }, size: 10, from: 0, - sort: [{ server_date_published: { order: "desc" } }], + // sort: [{ server_date_published: { order: "desc" } }], + sort: [{ _score: { order: "desc" } }], // Sort by _score in descending order + track_scores: true, // This ensures "_score" is included even when sorting by other criteria. Otherwise the relevance score is not calculated aggs: { language: { terms: { field: "language.keyword" } }, - subject: { terms: { field: "subjects.keyword", size: 10 } } + subjects: { terms: { field: "subjects.keyword", size: 10 } } // In SOLR is "subject"! + }, + highlight: { + fields: { + title: {}, + author: {}, + subjects: {} + } } }; + // Make API call to OpenSearch and return the result + /** + * When a POST request is made to the OpenSearch server using the api.post<OpenSearchResponse> method, the response received from OpenSearch is an object that includes various details about the search results. + * One of the key properties of this response object is _source, which is an array of documents (datasets) that match the search criteria. + * It is used the pipe method to chain RxJS operators to the Observable returned by api.get. The map operator is used to transform the emitted items of the Observable. + */ return api.post<OpenSearchResponse>(this.openSearchUrl, body).pipe( + // tap(response => console.log("OpenSearchResponse:", response)), // Log the complete response + // tap(response => console.log("Aggre:", response.aggregations?.subjects.buckets[0])), // log the first subject of the array of subjects returned + // tap(response => console.log("Hits:", response.hits)), // log the first subject of the array of subjects returned + map(response => response.hits.hits.map(hit => hit._source)) + + // map(response => response.hits.hits.map(hit => { + // const source = hit._source; + // const highlights = hit._highlight || {}; + // return { + // ...source, + // highlights + // }; + // })) ); } @@ -219,7 +247,7 @@ class DatasetService { * It is used the pipe method to chain RxJS operators to the Observable returned by api.get. The map operator is used to transform the emitted items of the Observable. */ const stations = api.get<SolrResponse>(base, q_params).pipe(map((res: SolrResponse) => res.response.docs)); - + return stations; }