tethys.backend/app/Controllers/Http/Editor/DatasetController.ts
2024-09-26 13:51:35 +02:00

683 lines
27 KiB
TypeScript

import type { HttpContext } from '@adonisjs/core/http';
import { Client } from '@opensearch-project/opensearch';
import User from '#models/user';
import Dataset from '#models/dataset';
import DatasetIdentifier from '#models/dataset_identifier';
import XmlModel from '#app/Library/XmlModel';
import { XMLBuilder } from 'xmlbuilder2/lib/interfaces.js';
import { create } from 'xmlbuilder2';
import { readFileSync } from 'fs';
import SaxonJS from 'saxon-js';
import { DateTime } from 'luxon';
import Index from '#app/Library/Utils/Index';
import { getDomain } from '#app/utils/utility-functions';
import { DoiClient } from '#app/Library/Doi/DoiClient';
import DoiClientException from '#app/exceptions/DoiClientException';
import logger from '@adonisjs/core/services/logger';
import { HttpException } from 'node-exceptions';
import { ModelQueryBuilderContract } from '@adonisjs/lucid/types/model';
import vine, { SimpleMessagesProvider } from '@vinejs/vine';
import mail from '@adonisjs/mail/services/main';
// import { resolveMx } from 'dns/promises';
// import * as net from 'net';
import { validate } from 'deep-email-validator';
// Create a new instance of the client
const client = new Client({ node: 'http://localhost:9200' }); // replace with your OpenSearch endpoint
export default class DatasetsController {
private proc;
public messages = {
// 'required': '{{ field }} is required',
// 'licenses.minLength': 'at least {{ options.minLength }} permission must be defined',
'reviewer_id.required': 'reviewer_id must be defined',
'publisher_name.required': 'publisher name must be defined',
};
constructor() {
this.proc = readFileSync('public/assets2/solr.sef.json');
// Load the XSLT file
// this.proc = readFileSync('public/assets2/datasetxml2oai.sef.json');
}
// public async index({}: HttpContextContract) {}
public async index({ auth, request, inertia }: HttpContext) {
const user = (await User.find(auth.user?.id)) as User;
const page = request.input('page', 1);
let datasets: ModelQueryBuilderContract<typeof Dataset, Dataset> = Dataset.query();
// if (request.input('search')) {
// // users = users.whereRaw('name like %?%', [request.input('search')])
// const searchTerm = request.input('search');
// datasets.where('name', 'ilike', `%${searchTerm}%`);
// }
if (request.input('sort')) {
type SortOrder = 'asc' | 'desc' | undefined;
let attribute = request.input('sort');
let sortOrder: SortOrder = 'asc';
if (attribute.substr(0, 1) === '-') {
sortOrder = 'desc';
// attribute = substr(attribute, 1);
attribute = attribute.substr(1);
}
datasets.orderBy(attribute, sortOrder);
} else {
// users.orderBy('created_at', 'desc');
datasets.orderBy('id', 'asc');
}
// const users = await User.query().orderBy('login').paginate(page, limit);
const myDatasets = await datasets
.where('server_state', 'released')
.orWhere((dQuery) => {
dQuery
.whereIn('server_state', ['editor_accepted', 'rejected_reviewer', 'reviewed', 'published'])
.where('editor_id', user.id)
.doesntHave('identifier', 'and');
})
// .preload('identifier')
.preload('titles')
.preload('user', (query) => query.select('id', 'login'))
.preload('editor', (query) => query.select('id', 'login'))
.paginate(page, 10);
return inertia.render('Editor/Dataset/Index', {
datasets: myDatasets.serialize(),
filters: request.all(),
can: {
receive: await auth.user?.can(['dataset-receive']),
approve: await auth.user?.can(['dataset-approve']),
reject: await auth.user?.can(['dataset-editor-reject']),
edit: await auth.user?.can(['dataset-editor-update']),
delete: await auth.user?.can(['dataset-editor-delete']),
publish: await auth.user?.can(['dataset-publish']),
},
});
}
public async receive({ request, inertia, response }: HttpContext) {
const id = request.param('id');
const dataset = await Dataset.query()
.where('id', id)
.preload('titles')
.preload('descriptions')
.preload('user', (builder) => {
builder.select('id', 'login');
})
.firstOrFail();
const validStates = ['released'];
if (!validStates.includes(dataset.server_state)) {
// session.flash('errors', 'Invalid server state!');
return response
.flash(
'warning',
`Invalid server state. Dataset with id ${id} cannot be received. Datset has server state ${dataset.server_state}.`,
)
.redirect()
.back();
}
return inertia.render('Editor/Dataset/Receive', {
dataset,
});
}
public async receiveUpdate({ auth, request, response }: HttpContext) {
const id = request.param('id');
// const { id } = params;
const dataset = await Dataset.findOrFail(id);
const validStates = ['released'];
if (!validStates.includes(dataset.server_state)) {
// throw new Error('Invalid server state!');
// return response.flash('warning', 'Invalid server state. Dataset cannot be released to editor').redirect().back();
return response
.flash(
'warning',
`Invalid server state. Dataset with id ${id} cannot be received by editor. Datset has server state ${dataset.server_state}.`,
)
.redirect()
.toRoute('editor.dataset.list');
}
dataset.server_state = 'editor_accepted';
const user = (await User.find(auth.user?.id)) as User;
// dataset.editor().associate(user).save();
try {
await dataset.related('editor').associate(user); // speichert schon ab
// await dataset.save();
return response.toRoute('editor.dataset.list').flash(`You have accepted dataset ${dataset.id}!`, 'message');
} catch (error) {
// Handle any errors
console.error(error);
return response.status(500).json({ error: 'An error occurred while accepting the data.' });
}
}
public async approve({ request, inertia, response }: HttpContext) {
const id = request.param('id');
// $dataset = Dataset::with('user:id,login')->findOrFail($id);
const dataset = await Dataset.findOrFail(id);
const validStates = ['editor_accepted', 'rejected_reviewer'];
if (!validStates.includes(dataset.server_state)) {
// session.flash('errors', 'Invalid server state!');
return response
.flash(
'warning',
`Invalid server state. Dataset with id ${id} cannot be approved. Datset has server state ${dataset.server_state}.`,
)
.redirect()
.back();
}
const reviewers = await User.query()
.whereHas('roles', (builder) => {
builder.where('name', 'reviewer');
})
.pluck('login', 'id');
return inertia.render('Editor/Dataset/Approve', {
dataset,
reviewers,
});
}
public async approveUpdate({ request, response }: HttpContext) {
const approveDatasetSchema = vine.object({
reviewer_id: vine.number(),
});
try {
// await request.validate({ schema: approveDatasetSchema, messages: this.messages });
const validator = vine.compile(approveDatasetSchema);
await request.validateUsing(validator, { messagesProvider: new SimpleMessagesProvider(this.messages) });
} catch (error) {
// return response.badRequest(error.messages);
throw error;
}
const id = request.param('id');
const dataset = await Dataset.findOrFail(id);
const validStates = ['editor_accepted', 'rejected_reviewer'];
if (!validStates.includes(dataset.server_state)) {
// session.flash('errors', 'Invalid server state!');
return response
.flash(
'warning',
`Invalid server state. Dataset with id ${id} cannot be approved. Datset has server state ${dataset.server_state}.`,
)
.redirect()
.back();
}
dataset.server_state = 'approved';
if (dataset.reject_reviewer_note != null) {
dataset.reject_reviewer_note = null;
}
//save main and additional titles
const reviewer_id = request.input('reviewer_id', null);
dataset.reviewer_id = reviewer_id;
if (await dataset.save()) {
return response.toRoute('editor.dataset.list').flash('message', 'You have approved one dataset!');
}
}
public async reject({ request, inertia, response }: HttpContext) {
const id = request.param('id');
const dataset = await Dataset.query()
.where('id', id)
// .preload('titles')
// .preload('descriptions')
.preload('user', (builder) => {
builder.select('id', 'login', 'email');
})
.firstOrFail();
const validStates = ['editor_accepted', 'rejected_reviewer'];
if (!validStates.includes(dataset.server_state)) {
// session.flash('errors', 'Invalid server state!');
return response
.flash(
'warning',
`Invalid server state. Dataset with id ${id} cannot be rejected. Datset has server state ${dataset.server_state}.`,
)
.redirect()
.toRoute('editor.dataset.list');
}
return inertia.render('Editor/Dataset/Reject', {
dataset,
});
}
// private async checkEmailDomain(email: string): Promise<boolean> {
// const domain = email.split('@')[1];
// try {
// // Step 1: Check MX records for the domain
// const mxRecords = await resolveMx(domain);
// if (mxRecords.length === 0) {
// return false; // No MX records, can't send email
// }
// // Sort MX records by priority
// mxRecords.sort((a, b) => a.priority - b.priority);
// // Step 2: Attempt SMTP connection to the first available mail server
// const smtpServer = mxRecords[0].exchange;
// return await this.checkMailboxExists(smtpServer, email);
// } catch (error) {
// console.error('Error during MX lookup or SMTP validation:', error);
// return false;
// }
// }
//// Helper function to check if the mailbox exists using SMTP
// private async checkMailboxExists(smtpServer: string, email: string): Promise<boolean> {
// return new Promise((resolve, reject) => {
// const socket = net.createConnection(25, smtpServer);
// socket.on('connect', () => {
// socket.write(`HELO ${smtpServer}\r\n`);
// socket.write(`MAIL FROM: <test@example.com>\r\n`);
// socket.write(`RCPT TO: <${email}>\r\n`);
// });
// socket.on('data', (data) => {
// const response = data.toString();
// if (response.includes('250')) {
// // 250 is an SMTP success code
// socket.end();
// resolve(true); // Email exists
// } else if (response.includes('550')) {
// // 550 means the mailbox doesn't exist
// socket.end();
// resolve(false); // Email doesn't exist
// }
// });
// socket.on('error', (error) => {
// console.error('SMTP connection error:', error);
// socket.end();
// resolve(false);
// });
// socket.on('end', () => {
// // SMTP connection closed
// });
// socket.setTimeout(5000, () => {
// // Timeout after 5 seconds
// socket.end();
// resolve(false); // Assume email doesn't exist if no response
// });
// });
// }
public async rejectUpdate({ request, response, auth }: HttpContext) {
const authUser = auth.user!;
const id = request.param('id');
const dataset = await Dataset.query()
.where('id', id)
.preload('user', (builder) => {
builder.select('id', 'login', 'email');
})
.firstOrFail();
const newSchema = vine.object({
server_state: vine.string().trim(),
reject_editor_note: vine.string().trim().minLength(10).maxLength(500),
send_mail: vine.boolean().optional(),
});
try {
// await request.validate({ schema: newSchema });
const validator = vine.compile(newSchema);
await request.validateUsing(validator);
} catch (error) {
// return response.badRequest(error.messages);
throw error;
}
const validStates = ['editor_accepted', 'rejected_reviewer'];
if (!validStates.includes(dataset.server_state)) {
// throw new Error('Invalid server state!');
// return response.flash('warning', 'Invalid server state. Dataset cannot be released to editor').redirect().back();
return response
.flash(
`Invalid server state. Dataset with id ${id} cannot be rejected. Datset has server state ${dataset.server_state}.`,
'warning'
)
.redirect()
.toRoute('editor.dataset.list');
}
dataset.server_state = 'rejected_editor';
const rejectEditorNote = request.input('reject_editor_note', '');
dataset.reject_editor_note = rejectEditorNote;
// add logic for sending reject message
const sendMail = request.input('send_email', false);
// const validRecipientEmail = await this.checkEmailDomain('arno.kaimbacher@outlook.at');
const validationResult = await validate({
email: dataset.user.email,
validateSMTP: false,
});
const validRecipientEmail: boolean = validationResult.valid;
let emailStatusMessage = '';
if (sendMail == true) {
if (dataset.user.email && validRecipientEmail) {
try {
await mail.send((message) => {
message.to(dataset.user.email).subject('Dataset Rejection Notification').html(`
<p>Dear ${dataset.user.login},</p>
<p>Your dataset with ID ${dataset.id} has been rejected.</p>
<p>Reason for rejection: ${rejectEditorNote}</p>
<p>Best regards,<br>Your Tethys editor: ${authUser.login}</p>
`);
});
emailStatusMessage = ` A rejection email was successfully sent to ${dataset.user.email}.`;
} catch (error) {
logger.error(error);
return response.flash('Dataset has not been rejected due to an email error: ' + error.message, 'error').toRoute('editor.dataset.list');
}
} else {
emailStatusMessage = ` However, the email could not be sent because the submitter's email address (${dataset.user.email}) is not valid.`;
}
}
await dataset.save();
return response
.flash(
`You have successfully rejected dataset ${dataset.id} submitted by ${dataset.user.login}.${emailStatusMessage}`,
'message',
)
.toRoute('editor.dataset.list');
}
public async publish({ request, inertia, response }: HttpContext) {
const id = request.param('id');
const dataset = await Dataset.query()
.where('id', id)
.preload('titles')
.preload('authors')
// .preload('persons', (builder) => {
// builder.wherePivot('role', 'author')
// })
.firstOrFail();
const validStates = ['reviewed'];
if (!validStates.includes(dataset.server_state)) {
// session.flash('errors', 'Invalid server state!');
return response
.flash(
'warning',
`Invalid server state. Dataset with id ${id} cannot be published. Datset has server state ${dataset.server_state}.`,
)
.redirect()
.back();
}
return inertia.render('Editor/Dataset/Publish', {
dataset,
});
}
public async publishUpdate({ request, response }: HttpContext) {
const publishDatasetSchema = vine.object({
publisher_name: vine.string().trim(),
});
try {
// await request.validate({ schema: publishDatasetSchema, messages: this.messages });
const validator = vine.compile(publishDatasetSchema);
await request.validateUsing(validator, { messagesProvider: new SimpleMessagesProvider(this.messages) });
} catch (error) {
throw error;
}
const id = request.param('id');
const dataset = await Dataset.findOrFail(id);
// let test = await Dataset.getMax('publish_id');
// const maxPublishId = await Database.from('documents').max('publish_id as max_publish_id').first();
// const max = maxPublishId.max_publish_id;
const max = await Dataset.getMax('publish_id');
let publish_id = 0;
if (max != null) {
publish_id = max + 1;
} else {
publish_id = publish_id + 1;
}
dataset.publish_id = publish_id;
dataset.server_state = 'published';
dataset.server_date_published = DateTime.now();
const publisherName = request.input('publisher_name', 'Tethys');
dataset.publisher_name = publisherName;
if (await dataset.save()) {
const index_name = 'tethys-records';
await Index.indexDocument(dataset, index_name);
return response.toRoute('editor.dataset.list').flash('message', 'You have successfully published the dataset!');
}
}
public async doiCreate({ request, inertia }: HttpContext) {
const id = request.param('id');
const dataset = await Dataset.query()
.where('id', id)
.preload('titles')
.preload('descriptions')
// .preload('identifier')
.preload('authors')
.firstOrFail();
return inertia.render('Editor/Dataset/Doi', {
dataset,
});
}
public async doiStore({ request, response }: HttpContext) {
const dataId = request.param('publish_id');
const dataset = await Dataset.query()
// .preload('xmlCache')
.where('publish_id', dataId)
.firstOrFail();
const xmlMeta = (await Index.getDoiRegisterString(dataset)) as string;
let prefix = '';
let base_domain = '';
// const datacite_environment = process.env.DATACITE_ENVIRONMENT || 'debug';
prefix = process.env.DATACITE_PREFIX || '';
base_domain = process.env.BASE_DOMAIN || '';
// register DOI:
const doiValue = prefix + '/tethys.' + dataset.publish_id; //'10.21388/tethys.213'
const landingPageUrl = 'https://doi.' + getDomain(base_domain) + '/' + prefix + '/tethys.' + dataset.publish_id; //https://doi.dev.tethys.at/10.21388/tethys.213
const doiClient = new DoiClient();
const dataciteResponse = await doiClient.registerDoi(doiValue, xmlMeta, landingPageUrl);
if (dataciteResponse?.status === 201) {
// if response OK 201; save the Identifier value into db
const doiIdentifier = new DatasetIdentifier();
doiIdentifier.value = doiValue;
doiIdentifier.dataset_id = dataset.id;
doiIdentifier.type = 'doi';
doiIdentifier.status = 'findable';
// save modified date of datset for re-caching model in db an update the search index
dataset.server_date_modified = DateTime.now();
// save updated dataset to db an index to OpenSearch
try {
await dataset.related('identifier').save(doiIdentifier);
const index_name = 'tethys-records';
await Index.indexDocument(dataset, index_name);
} catch (error) {
logger.error(`${__filename}: Indexing document ${dataset.id} failed: ${error.message}`);
// Log the error or handle it as needed
throw new HttpException(error.message);
}
return response.toRoute('editor.dataset.list').flash('message', 'You have successfully created a DOI for the dataset!');
} else {
const message = `Unexpected DataCite MDS response code ${dataciteResponse?.status}`;
// Log the error or handle it as needed
throw new DoiClientException(dataciteResponse?.status, message);
}
// return response.toRoute('editor.dataset.list').flash('message', xmlMeta);
}
public async show({}: HttpContext) {}
public async edit({}: HttpContext) {}
// public async update({}: HttpContextContract) {}
public async update({ response }: HttpContext) {
const id = 273; //request.param('id');
const dataset = await Dataset.query().preload('xmlCache').where('id', id).firstOrFail();
// add xml elements
let xml = create({ version: '1.0', encoding: 'UTF-8', standalone: true }, '<root></root>');
const datasetNode = xml.root().ele('Dataset');
await this.createXmlRecord(dataset, datasetNode);
// const domNode = await this.getDatasetXmlDomNode(dataset);
// const xmlString = xml.end({ prettyPrint: true });
// const data = request.only(['field1', 'field2']); // get it from xslt
// Create an index with non-default settings.
var index_name = 'tethys-features';
const xmlString = xml.end({ prettyPrint: false });
let doc = '';
try {
const result = await SaxonJS.transform({
// stylesheetFileName: `${config.TMP_BASE_DIR}/data-quality/rules/iati.sef.json`,
stylesheetText: this.proc,
destination: 'serialized',
// sourceFileName: sourceFile,
sourceText: xmlString,
// stylesheetParams: xsltParameter,
// logLevel: 10,
});
doc = result.principalResult;
} catch (error) {
return response.status(500).json({
message: 'An error occurred while creating the user',
error: error.message,
});
}
// var settings = {
// settings: {
// index: {
// number_of_shards: 4,
// number_of_replicas: 3,
// },
// },
// };
// var test = await client.indices.create({
// index: index_name,
// body: settings,
// });
// var document = {
// title: 'Sample Document',
// authors: [
// {
// first_name: 'John',
// last_name: 'Doe',
// },
// {
// first_name: 'Jane',
// last_name: 'Smith',
// },
// ],
// year: '2018',
// genre: 'Crime fiction',
// };
// http://localhost:9200/datastets/_doc/1
// var id = '1';
try {
// console.log(doc);
let document = JSON.parse(`${doc}`);
// https://opensearch.org/docs/2.1/opensearch/supported-field-types/geo-shape/
// Define the new document
// const document = {
// title: 'Your Document Name',
// id: dataset.publish_id,
// doctype: 'GIS',
// // "location" : {
// // "type" : "point",
// // "coordinates" : [74.00, 40.71]
// // },
// geo_location: {
// type: 'linestring',
// coordinates: [
// [-77.03653, 38.897676],
// [-77.009051, 38.889939],
// ],
// },
// // geo_location: 'BBOX (71.0589, 74.0060, 42.3601, 40.7128)'
// // geo_location: {
// // type: 'envelope',
// // coordinates: [
// // [13.0, 53.0],
// // [14.0, 52.0],
// // ], // Define your BBOX coordinates
// // },
// };
// Update the document
var test = await client.index({
id: dataset.publish_id?.toString(),
index: index_name,
body: document,
refresh: true,
});
// Return the result
return response.json(test.body);
} catch (error) {
// Handle any errors
console.error(error);
return response.status(500).json({ error: 'An error occurred while updating the data.' });
}
}
public async destroy({}: HttpContext) {}
private async createXmlRecord(dataset: Dataset, datasetNode: XMLBuilder) {
const domNode = await this.getDatasetXmlDomNode(dataset);
if (domNode) {
datasetNode.import(domNode);
}
}
private async getDatasetXmlDomNode(dataset: Dataset) {
const xmlModel = new XmlModel(dataset);
// xmlModel.setModel(dataset);
xmlModel.excludeEmptyFields();
xmlModel.caching = true;
// const cache = dataset.xmlCache ? dataset.xmlCache : null;
// dataset.load('xmlCache');
if (dataset.xmlCache) {
xmlModel.xmlCache = dataset.xmlCache;
}
// return cache.getDomDocument();
const domDocument: XMLBuilder | null = await xmlModel.getDomDocument();
return domDocument;
}
}