// import Logger from '@ioc:Adonis/Core/Logger'; import { XMLBuilder } from 'xmlbuilder2/lib/interfaces.js'; import { create } from 'xmlbuilder2'; import Dataset from '#app/Models/Dataset'; import XmlModel from '#app/Library/XmlModel'; import { readFileSync } from 'fs'; import SaxonJS from 'saxon-js'; import { Client } from '@opensearch-project/opensearch'; import { getDomain } from '#app/Utils/utility-functions'; import { BaseCommand, flags } from '@adonisjs/core/ace'; import { CommandOptions } from '@adonisjs/core/types/ace'; const opensearchNode = process.env.OPENSEARCH_HOST || 'localhost'; const client = new Client({ node: `http://${opensearchNode}` }); // replace with your OpenSearch endpoint export default class IndexDatasets extends BaseCommand { static commandName = 'index:datasets'; static description = 'Index datasets based on publish_id'; @flags.number({ alias: 'p' }) public publish_id: number; static options: CommandOptions = { loadApp: true, staysAlive: false, }; async run() { this.logger.info('Hello world!'); const { default: Dataset } = await import('#app/Models/Dataset'); const datasets = await Dataset.query().where('server_state', 'published').exec(); //this.getDatasets(); const proc = readFileSync('public/assets2/solr.sef.json'); const index_name = 'tethys-records'; for (var dataset of datasets) { // Logger.info(`File publish_id ${dataset.publish_id}`); // const jsonString = await this.getJsonString(dataset, proc); // console.log(jsonString); await this.indexDocument(dataset, index_name, proc); } } // private async getDatasets(): Promise { // // const { default: Dataset } = await import('#app/Models/Dataset'); // // const Dataset = (await import('#app/Models/Dataset')).default // const query = Dataset.query().where('server_state', 'published'); // if (this.publish_id) { // query.where('publish_id', this.publish_id); // } // return await query; // } private async indexDocument(dataset: Dataset, index_name: string, proc: Buffer): Promise { try { const doc = await this.getJsonString(dataset, proc); let document = JSON.parse(doc); await client.index({ id: dataset.publish_id?.toString(), index: index_name, body: document, refresh: true, }); this.logger.info(`dataset with publish_id ${dataset.publish_id} successfully indexed`); } catch (error) { this.logger.error(`An error occurred while indexing datsaet with publish_id ${dataset.publish_id}.`); } } private async getJsonString(dataset: Dataset, proc: Buffer) { let xml = create({ version: '1.0', encoding: 'UTF-8', standalone: true }, ''); const datasetNode = xml.root().ele('Dataset'); await this.createXmlRecord(dataset, datasetNode); const xmlString = xml.end({ prettyPrint: false }); try { const result = await SaxonJS.transform({ stylesheetText: proc, destination: 'serialized', sourceText: xmlString, }); return result.principalResult; } catch (error) { this.logger.error(`An error occurred while creating the user, error: ${error.message},`); return ''; } } private async createXmlRecord(dataset: Dataset, datasetNode: XMLBuilder): Promise { const domNode = await this.getDatasetXmlDomNode(dataset); if (domNode) { dataset.publish_id && this.addLandingPageAttribute(domNode, dataset.publish_id.toString()); this.addSpecInformation(domNode, 'data-type:' + dataset.type); datasetNode.import(domNode); } } private async getDatasetXmlDomNode(dataset: Dataset): Promise { const xmlModel = new XmlModel(dataset); // xmlModel.setModel(dataset); xmlModel.excludeEmptyFields(); xmlModel.caching = true; // const cache = dataset.xmlCache ? dataset.xmlCache : null; // dataset.load('xmlCache'); if (dataset.xmlCache) { xmlModel.xmlCache = dataset.xmlCache; } // return cache.getDomDocument(); const domDocument: XMLBuilder | null = await xmlModel.getDomDocument(); return domDocument; } private addSpecInformation(domNode: XMLBuilder, information: string) { domNode.ele('SetSpec').att('Value', information); } private addLandingPageAttribute(domNode: XMLBuilder, dataid: string) { const baseDomain = process.env.OAI_BASE_DOMAIN || 'localhost'; const url = 'https://' + getDomain(baseDomain) + '/dataset/' + dataid; // add attribute du dataset xml element domNode.att('landingpage', url); } }