geolba/tethys.backend
2
0
Fork 1
Code Issues 7 Pull Requests Actions Packages Projects Releases Wiki Activity

- added earliestPublicationDate for App/Models/Dataset.ts
All checks were successful
CI Pipeline / japa-tests (push) Successful in 49s
Details

Browse Source 
- new classes TokenWorkerService.ts, TokenWorker.ts and ResumptionToken.ts for using REDIS with paging OAI results
- deletd public/asstes2/langCodeMap.xml: integrated it directly in datasetxml2oai-pmh.xslt
- added redis npm package
- added TokenWorkerProvider.ts for using singleton of TokenWorkerService inside OaiController.ts
- added config/oai.ts for oai related configs from .env-file
- adapted XmlModel.ts for grting domDocument from database
This commit is contained in:
Kaimbacher 2023-10-03 21:11:02 +02:00
parent 2a7480d2ed
commit 7915f66dd6
16 changed files with 691 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.adonisrc.json
View File

@ -40,7 +40,8 @@
"@adonisjs/lucid",
"@adonisjs/auth",
"@eidellev/adonis-stardust",
"./providers/QueryBuilderProvider"
"./providers/QueryBuilderProvider",
"./providers/TokenWorkerProvider"
],
"metaFiles": [
{

334
app/Controllers/Http/Oai/OaiController.ts
View File

@ -13,8 +13,15 @@ import { OaiErrorCodes, OaiModelError } from 'App/Exceptions/OaiErrorCodes';
import { OaiModelException, BadOaiModelException } from 'App/Exceptions/OaiModelException';
import Dataset from 'App/Models/Dataset';
import Collection from 'App/Models/Collection';
import { getDomain } from 'App/Utils/utility-functions';
import { getDomain, preg_match } from 'App/Utils/utility-functions';
import XmlModel from 'App/Library/XmlModel';
import Logger from '@ioc:Adonis/Core/Logger';
import ResumptionToken from 'App/Library/Oai/ResumptionToken';
import { ModelQueryBuilderContract } from '@ioc:Adonis/Lucid/Orm';
import Config from '@ioc:Adonis/Core/Config';
import { inject } from '@adonisjs/fold';
// import { TokenWorkerContract } from "MyApp/Models/TokenWorker";
import TokenWorkerContract from 'App/Library/Oai/TokenWorker';
interface XslTParameter {
[key: string]: any;
@ -24,12 +31,19 @@ interface Dictionary {
[index: string]: string;
}
interface ListParameter {
cursor: number;
totalIds: number;
start: number;
reldocIds: (number | null)[];
metadataPrefix: string;
}
@inject(['App/Library/Oai/TokenWorkerContract'])
export default class OaiController {
private deliveringDocumentStates = ['published', 'deleted'];
// private sampleRegEx = /^[A-Za-zäüÄÜß0-9\-_.!~]+$/;
private sampleRegEx = /^[A-Za-zäüÄÜß0-9\-_.!~]+$/;
private xsltParameter: XslTParameter;
// private configuration: Configuration;
// private tokenWorker: TokenWorker;
/**
* Holds xml representation of document information to be processed.
@ -39,13 +53,9 @@ export default class OaiController {
private xml: XMLBuilder;
private proc;
constructor() {
constructor(public tokenWorker: TokenWorkerContract) {
// Load the XSLT file
this.proc = readFileSync('public/assets2/datasetxml2oai.sef.json');
// tests
// const xslPath = 'assets/datasetxml2oai-pmh.xslt'; // Replace with the actual path to your XSLT file
// this.proc = readFileSync(xslPath, 'utf-8');
// this.configuration = new Configuration();
dayjs.extend(utc);
dayjs.extend(timezone);
}
@ -66,8 +76,15 @@ export default class OaiController {
xsltParameter['oai_error_code'] = 'unknown';
xsltParameter['oai_error_message'] = 'Only POST and GET methods are allowed for OAI-PMH.';
}
let earliestDateFromDb;
// const oaiRequest: OaiParameter = request.body;
try {
const firstPublishedDataset: Dataset | null = await Dataset.earliestPublicationDate();
firstPublishedDataset != null &&
(earliestDateFromDb = firstPublishedDataset.server_date_published.toFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"));
this.xsltParameter['earliestDatestamp'] = earliestDateFromDb;
// start the request
await this.handleRequest(oaiRequest, request);
} catch (error) {
if (error instanceof OaiModelException) {
@ -87,7 +104,7 @@ export default class OaiController {
const xmlString = this.xml.end({ prettyPrint: true });
let xmlOutput;
let xmlOutput; // = xmlString;
try {
const result = await transform({
// stylesheetFileName: `${config.TMP_BASE_DIR}/data-quality/rules/iati.sef.json`,
@ -123,7 +140,7 @@ export default class OaiController {
this.xsltParameter['unixTimestamp'] = now.unix();
// set OAI base url
const baseDomain = process.env.BASE_DOMAIN || 'localhost';
const baseDomain = process.env.OAI_BASE_DOMAIN || 'localhost';
this.xsltParameter['baseURL'] = baseDomain + '/oai';
this.xsltParameter['repURL'] = request.protocol() + '://' + request.hostname();
this.xsltParameter['downloadLink'] = request.protocol() + '://' + request.hostname() + '/file/download/';
@ -139,13 +156,11 @@ export default class OaiController {
this.handleListMetadataFormats();
} else if (verb == 'GetRecord') {
await this.handleGetRecord(oaiRequest);
}
// else if (verb == "ListRecords") {
// await this.handleListRecords(oaiRequest);
// } else if (verb == "ListIdentifiers") {
// await this.handleListIdentifiers(oaiRequest);
// }
else if (verb == 'ListSets') {
} else if (verb == 'ListRecords') {
await this.handleListRecords(oaiRequest);
} else if (verb == 'ListIdentifiers') {
await this.handleListIdentifiers(oaiRequest);
} else if (verb == 'ListSets') {
await this.handleListSets();
} else {
this.handleIllegalVerb();
@ -197,7 +212,7 @@ export default class OaiController {
const sets: { [key: string]: string } = {
'open_access': 'Set for open access licenses',
'doc-type:ResearchData': 'Set for document type ResearchData',
// ...(await this.getSetsForDatasetTypes()),
...(await this.getSetsForDatasetTypes()),
...(await this.getSetsForCollections()),
// ... await this.getSetsForProjects(),
} as Dictionary;
@ -214,7 +229,13 @@ export default class OaiController {
this.xsltParameter['repIdentifier'] = repIdentifier;
const dataId = this.validateAndGetIdentifier(oaiRequest);
const dataset = await Dataset.query().where('publish_id', dataId).preload('xmlCache').preload('collections').first();
const dataset = await Dataset.query()
.where('publish_id', dataId)
.preload('xmlCache')
.preload('collections', (builder) => {
builder.preload('collectionRole');
})
.first();
if (!dataset || !dataset.publish_id) {
throw new OaiModelException(
@ -234,6 +255,229 @@ export default class OaiController {
await this.createXmlRecord(dataset, datasetNode);
}
protected async handleListIdentifiers(oaiRequest: Dictionary) {
!this.tokenWorker.isConnected && (await this.tokenWorker.connect());
const maxIdentifier: number = Config.get('oai.max.listidentifiers', 100);
await this.handleLists(oaiRequest, maxIdentifier);
}
protected async handleListRecords(oaiRequest) {
!this.tokenWorker.isConnected && (await this.tokenWorker.connect());
const maxRecords: number = Config.get('oai.max.listrecords', 100);
await this.handleLists(oaiRequest, maxRecords);
}
private async handleLists(oaiRequest: Dictionary, maxRecords: number) {
maxRecords = maxRecords || 100;
const repIdentifier = 'tethys.at';
this.xsltParameter['repIdentifier'] = repIdentifier;
const datasetNode = this.xml.root().ele('Datasets');
// list initialisation
const numWrapper: ListParameter = {
cursor: 0,
totalIds: 0,
start: maxRecords + 1,
reldocIds: [],
metadataPrefix: '',
};
// resumptionToken is defined
if ('resumptionToken' in oaiRequest) {
await this.handleResumptionToken(oaiRequest, maxRecords, numWrapper);
} else {
// no resumptionToken is given
await this.handleNoResumptionToken(oaiRequest, numWrapper);
}
// handling of document ids
const restIds = numWrapper.reldocIds as number[];
const workIds = restIds.splice(0, maxRecords) as number[]; // array_splice(restIds, 0, maxRecords);
// no records returned
if (workIds.length == 0) {
throw new OaiModelException(
StatusCodes.INTERNAL_SERVER_ERROR,
'The combination of the given values results in an empty list.',
OaiErrorCodes.NORECORDSMATCH,
);
}
const datasets: Dataset[] = await Dataset.query()
.whereIn('publish_id', workIds)
.preload('xmlCache')
.preload('collections', (builder) => {
builder.preload('collectionRole');
})
.orderBy('publish_id');
for (const dataset of datasets) {
await this.createXmlRecord(dataset, datasetNode);
}
// store the further Ids in a resumption-file
const countRestIds = restIds.length; //84
if (countRestIds > 0) {
const token = new ResumptionToken();
token.startPosition = numWrapper.start; //101
token.totalIds = numWrapper.totalIds; //184
token.documentIds = restIds; //101 -184
token.metadataPrefix = numWrapper.metadataPrefix;
// $tokenWorker->storeResumptionToken($token);
const res: string = await this.tokenWorker.set(token);
// set parameters for the resumptionToken-node
// const res = token.ResumptionId;
this.setParamResumption(res, numWrapper.cursor, numWrapper.totalIds);
}
}
private async handleResumptionToken(oaiRequest: Dictionary, maxRecords: number, numWrapper) {
const resParam = oaiRequest['resumptionToken']; //e.g. "158886496600000"
const token = await this.tokenWorker.get(resParam);
if (!token) {
throw new OaiModelException(StatusCodes.INTERNAL_SERVER_ERROR, 'cache is outdated.', OaiErrorCodes.BADRESUMPTIONTOKEN);
}
numWrapper.cursor = token.startPosition - 1; //startet dann bei Index 10
numWrapper.start = token.startPosition + maxRecords;
numWrapper.totalIds = token.totalIds;
numWrapper.reldocIds = token.documentIds;
numWrapper.metadataPrefix = token.metadataPrefix;
this.xsltParameter['oai_metadataPrefix'] = numWrapper.metadataPrefix;
}
private async handleNoResumptionToken(oaiRequest: Dictionary, numWrapper) {
// no resumptionToken is given
if ('metadataPrefix' in oaiRequest) {
numWrapper.metadataPrefix = oaiRequest['metadataPrefix'];
} else {
throw new OaiModelException(
StatusCodes.INTERNAL_SERVER_ERROR,
'The prefix of the metadata argument is unknown.',
OaiErrorCodes.BADARGUMENT,
);
}
this.xsltParameter['oai_metadataPrefix'] = numWrapper.metadataPrefix;
let finder: ModelQueryBuilderContract<typeof Dataset, Dataset> = Dataset.query();
// add server state restrictions
finder.whereIn('server_state', this.deliveringDocumentStates);
if ('set' in oaiRequest) {
const set = oaiRequest['set'] as string;
const setArray = set.split(':');
if (setArray[0] == 'data-type') {
if (setArray.length == 2 && setArray[1]) {
finder.where('type', setArray[1]);
}
} else if (setArray[0] == 'open_access') {
const openAccessLicences = ['CC-BY-4.0', 'CC-BY-SA-4.0'];
finder.andWhereHas('licenses', (query) => {
query.whereIn('name', openAccessLicences);
});
} else if (setArray[0] == 'ddc') {
if (setArray.length == 2 && setArray[1] != '') {
finder.andWhereHas('collections', (query) => {
query.where('number', setArray[1]);
});
}
}
}
// const timeZone = "Europe/Vienna"; // Canonical time zone name
// &from=2020-09-03&until2020-09-03
// &from=2020-09-11&until=2021-05-11
if ('from' in oaiRequest && 'until' in oaiRequest) {
const from = oaiRequest['from'] as string;
let fromDate = dayjs(from); //.tz(timeZone);
const until = oaiRequest['until'] as string;
let untilDate = dayjs(until); //.tz(timeZone);
if (!fromDate.isValid() || !untilDate.isValid()) {
throw new OaiModelException(StatusCodes.INTERNAL_SERVER_ERROR, 'Date Parameter is not valid.', OaiErrorCodes.BADARGUMENT);
}
fromDate = dayjs.tz(from, 'Europe/Vienna');
untilDate = dayjs.tz(until, 'Europe/Vienna');
if (from.length != until.length) {
throw new OaiModelException(
StatusCodes.INTERNAL_SERVER_ERROR,
'The request has different granularities for the from and until parameters.',
OaiErrorCodes.BADARGUMENT,
);
}
fromDate.hour() == 0 && (fromDate = fromDate.startOf('day'));
untilDate.hour() == 0 && (untilDate = untilDate.endOf('day'));
finder.whereBetween('server_date_published', [fromDate.format('YYYY-MM-DD HH:mm:ss'), untilDate.format('YYYY-MM-DD HH:mm:ss')]);
} else if ('from' in oaiRequest && !('until' in oaiRequest)) {
const from = oaiRequest['from'] as string;
let fromDate = dayjs(from);
if (!fromDate.isValid()) {
throw new OaiModelException(
StatusCodes.INTERNAL_SERVER_ERROR,
'From date parameter is not valid.',
OaiErrorCodes.BADARGUMENT,
);
}
fromDate = dayjs.tz(from, 'Europe/Vienna');
fromDate.hour() == 0 && (fromDate = fromDate.startOf('day'));
const now = dayjs();
if (fromDate.isAfter(now)) {
throw new OaiModelException(
StatusCodes.INTERNAL_SERVER_ERROR,
'Given from date is greater than now. The given values results in an empty list.',
OaiErrorCodes.NORECORDSMATCH,
);
} else {
finder.andWhere('server_date_published', '>=', fromDate.format('YYYY-MM-DD HH:mm:ss'));
}
} else if (!('from' in oaiRequest) && 'until' in oaiRequest) {
const until = oaiRequest['until'] as string;
let untilDate = dayjs(until);
if (!untilDate.isValid()) {
throw new OaiModelException(
StatusCodes.INTERNAL_SERVER_ERROR,
'Until date parameter is not valid.',
OaiErrorCodes.BADARGUMENT,
);
}
untilDate = dayjs.tz(until, 'Europe/Vienna');
untilDate.hour() == 0 && (untilDate = untilDate.endOf('day'));
const firstPublishedDataset: Dataset = (await Dataset.earliestPublicationDate()) as Dataset;
const earliestPublicationDate = dayjs(firstPublishedDataset.server_date_published.toISO()); //format("YYYY-MM-DDThh:mm:ss[Z]"));
if (earliestPublicationDate.isAfter(untilDate)) {
throw new OaiModelException(
StatusCodes.INTERNAL_SERVER_ERROR,
`earliestDatestamp is greater than given until date.
The given values results in an empty list.`,
OaiErrorCodes.NORECORDSMATCH,
);
} else {
finder.andWhere('server_date_published', '<=', untilDate.format('YYYY-MM-DD HH:mm:ss'));
}
}
let reldocIdsDocs = await finder.select('publish_id').orderBy('publish_id');
numWrapper.reldocIds = reldocIdsDocs.map((dat) => dat.publish_id);
numWrapper.totalIds = numWrapper.reldocIds.length; //212
}
private setParamResumption(res: string, cursor: number, totalIds: number) {
const tomorrow = dayjs().add(1, 'day').format('YYYY-MM-DDThh:mm:ss[Z]');
this.xsltParameter['dateDelete'] = tomorrow;
this.xsltParameter['res'] = res;
this.xsltParameter['cursor'] = cursor;
this.xsltParameter['totalIds'] = totalIds;
}
private validateAndGetIdentifier(oaiRequest: Dictionary): number {
// Identifier references metadata Urn, not plain Id!
// Currently implemented as 'oai:foo.bar.de:{docId}' or 'urn:nbn...-123'
@ -283,12 +527,12 @@ export default class OaiController {
dataset.publish_id && this.addLandingPageAttribute(domNode, dataset.publish_id.toString());
this.addSpecInformation(domNode, 'data-type:' + dataset.type);
// if (dataset.collections) {
// for (const coll of dataset.collections) {
// const collRole = await coll.getCollectionRole();
// this.addSpecInformation(domNode, collRole.oai_name + ':' + coll.number);
// }
// }
if (dataset.collections) {
for (const coll of dataset.collections) {
const collRole = coll.collectionRole;
this.addSpecInformation(domNode, collRole.oai_name + ':' + coll.number);
}
}
datasetNode.import(domNode);
}
@ -315,7 +559,7 @@ export default class OaiController {
}
private addLandingPageAttribute(domNode: XMLBuilder, dataid: string) {
const baseDomain = process.env.BASE_DOMAIN || 'localhost';
const baseDomain = process.env.OAI_BASE_DOMAIN || 'localhost';
const url = 'https://' + getDomain(baseDomain) + '/dataset/' + dataid;
// add attribute du dataset xml element
domNode.att('landingpage', url);
@ -368,26 +612,24 @@ export default class OaiController {
return sets;
}
// private async getSetsForDatasetTypes(): Promise<IDictionary> {
// const sets: { [key: string]: string } = {} as IDictionary;
private async getSetsForDatasetTypes(): Promise<Dictionary> {
const sets: { [key: string]: string } = {} as Dictionary;
// const datasets: Array<Dataset> = await Dataset.findAll({
// attributes: ["type"],
// where: { server_state: { [Sequelize.Op.eq]: "published" } },
// });
// datasets.forEach((dataset) => {
// if (dataset.type && false == preg_match(this.sampleRegEx, dataset.type)) {
// const msg = `Invalid SetSpec (data-type='${dataset.type}').
// Allowed characters are [${this.sampleRegEx}].`;
// Logger.err(`OAI: ${msg}`);
// // Log::error("OAI-PMH: $msg");
// return;
// }
// const setSpec = "data-type:" + dataset.type;
// sets[setSpec] = `Set for document type '${dataset.type}'`;
// });
// return sets;
// }
const datasets: Array<Dataset> = await Dataset.query().select('type').where('server_state', 'published');
datasets.forEach((dataset) => {
if (dataset.type && false == preg_match(this.sampleRegEx, dataset.type)) {
const msg = `Invalid SetSpec (data-type='${dataset.type}').
Allowed characters are [${this.sampleRegEx}].`;
// Log::error("OAI-PMH: $msg");
Logger.error(`OAI-PMH: ${msg}`);
return;
}
const setSpec = 'data-type:' + dataset.type;
sets[setSpec] = `Set for document type '${dataset.type}'`;
});
return sets;
}
private handleIllegalVerb() {
this.xsltParameter['oai_error_code'] = 'badVerb';

51
app/Library/Oai/ResumptionToken.ts Normal file
View File

@ -0,0 +1,51 @@
export default class ResumptionToken {
private _documentIds: number[] = [];
private _metadataPrefix = '';
private _resumptionId = '';
private _startPosition = 0;
private _totalIds = 0;
get key(): string {
return this.metadataPrefix + this.startPosition + this.totalIds;
}
get documentIds(): number[] {
return this._documentIds;
}
set documentIds(idsToStore: number | number[]) {
this._documentIds = Array.isArray(idsToStore) ? idsToStore : [idsToStore];
}
get metadataPrefix(): string {
return this._metadataPrefix;
}
set metadataPrefix(value: string) {
this._metadataPrefix = value;
}
get resumptionId(): string {
return this._resumptionId;
}
set resumptionId(resumptionId: string) {
this._resumptionId = resumptionId;
}
get startPosition(): number {
return this._startPosition;
}
set startPosition(startPosition: number) {
this._startPosition = startPosition;
}
get totalIds(): number {
return this._totalIds;
}
set totalIds(totalIds: number) {
this._totalIds = totalIds;
}
}

10
app/Library/Oai/TokenWorker.ts Normal file
View File

@ -0,0 +1,10 @@
import ResumptionToken from './ResumptionToken';
export default interface TokenWorkerContract {
ttl: number;
isConnected: boolean;
connect();
close();
get(key: string): Promise<ResumptionToken | null>;
set(token: ResumptionToken): Promise<string>;
}

95
app/Library/Oai/TokenWorkerSerice.ts Normal file
View File

@ -0,0 +1,95 @@
import ResumptionToken from './ResumptionToken';
import { createClient, RedisClientType } from 'redis';
import InternalServerErrorException from 'App/Exceptions/InternalServerException';
import { sprintf } from 'sprintf-js';
import dayjs from 'dayjs';
import TokenWorkerContract from './TokenWorker';
export default class TokenWorkerService implements TokenWorkerContract {
protected filePrefix = 'rs_';
protected fileExtension = 'txt';
private cache: RedisClientType;
public ttl: number;
private url: string;
private connected = false;
constructor(ttl: number) {
this.ttl = ttl; // time to live
this.url = process.env.REDIS_URL || 'redis://127.0.0.1:6379';
}
public async connect() {
this.cache = createClient({ url: this.url });
this.cache.on('error', (err) => {
this.connected = false;
console.log('[Redis] Redis Client Error: ', err);
});
this.cache.on('connect', () => {
this.connected = true;
});
await this.cache.connect();
}
public get isConnected(): boolean {
return this.connected;
}
public async has(key: string): Promise<boolean> {
const result = await this.cache.get(key);
return result !== undefined && result !== null;
}
public async set(token: ResumptionToken): Promise<string> {
const uniqueName = await this.generateUniqueName();
const serialToken = JSON.stringify(token);
await this.cache.setEx(uniqueName, this.ttl, serialToken);
return uniqueName;
}
private async generateUniqueName(): Promise<string> {
let fc = 0;
const uniqueId = dayjs().unix().toString();
let uniqueName: string;
let cacheKeyExists: boolean;
do {
// format values
// %s - String
// %d - Signed decimal number (negative, zero or positive)
// [0-9] (Specifies the minimum width held of to the variable value)
uniqueName = sprintf('%s%05d', uniqueId, fc++);
cacheKeyExists = await this.has(uniqueName);
} while (cacheKeyExists);
return uniqueName;
}
public async get(key: string): Promise<ResumptionToken | null> {
if (!this.cache) {
throw new InternalServerErrorException('Dataset is not available for OAI export!');
}
const result = await this.cache.get(key);
return result ? this.parseToken(result) : null;
}
private parseToken(result: string): ResumptionToken {
const rToken: ResumptionToken = new ResumptionToken();
const parsed = JSON.parse(result);
Object.assign(rToken, parsed);
return rToken;
}
public del(key: string) {
this.cache.del(key);
}
public flush() {
this.cache.flushAll();
}
public async close() {
await this.cache.disconnect();
this.connected = false;
}
}

14
app/Library/XmlModel.ts
View File

@ -3,6 +3,7 @@ import { XMLBuilder } from 'xmlbuilder2/lib/interfaces';
import Dataset from 'App/Models/Dataset';
import Strategy from './Strategy';
import { DateTime } from 'luxon';
import { builder } from 'xmlbuilder2';
/**
* This is the description of the interface
@ -84,10 +85,21 @@ export default class XmlModel {
this.cache = this.cache || new DocumentXmlCache();
this.cache.document_id = dataset.id;
this.cache.xml_version = 1; // (int)$this->strategy->getVersion();
// this.cache.server_date_modified = dataset.server_date_modified.toFormat("yyyy-MM-dd HH:mm:ss");
this.cache.server_date_modified = dataset.server_date_modified.toFormat("yyyy-MM-dd HH:mm:ss");
this.cache.xml_data = domDocument.end();
await this.cache.save();
}
const node = domDocument.find(
(n) => {
const test = n.node.nodeName == 'Rdr_Dataset';
return test;
},
false,
true,
)?.node;
if(node != undefined) {
domDocument = builder({ version: '1.0', encoding: 'UTF-8', standalone: true }, node);
}
}
return domDocument;
}

8
app/Models/Dataset.ts
View File

@ -214,4 +214,12 @@ export default class Dataset extends DatasetExtension {
foreignKey: 'document_id',
})
public xmlCache: HasOne<typeof DocumentXmlCache>;
static async earliestPublicationDate(): Promise<Dataset | null> {
const serverState = 'published';
const model = await this.query().where('server_state', serverState).orderBy('server_date_published', 'asc').first();
return model || null;
}
}

5
app/Utils/utility-functions.ts
View File

@ -19,3 +19,8 @@ export function getDomain(host: string): string {
myHost = myHost.replace(new RegExp(/^.*:\/\//i, 'g'), '');
return myHost;
}
export function preg_match(regex: RegExp, str: string) {
const result: boolean = regex.test(str);
return result;
}

18
config/oai.ts Normal file
View File

@ -0,0 +1,18 @@
import Env from '@ioc:Adonis/Core/Env';
interface OaiConfig {
max: { listidentifiers: number; listrecords: number };
workspacePath: string;
redis: { ttl: number };
}
const config: OaiConfig = {
max: {
listidentifiers: parseInt(Env.get('OAI_LIST_SIZE', 100), 10),
listrecords: parseInt(Env.get('OAI_LIST_SIZE', 100), 10),
},
workspacePath: 'workspace',
redis: {
ttl: 86400, //sec 1 day
},
};
export default config;

138
package-lock.json generated
View File

@ -31,6 +31,7 @@
"notiwind": "^2.0.0",
"pg": "^8.9.0",
"proxy-addr": "^2.0.7",
"redis": "^4.6.10",
"reflect-metadata": "^0.1.13",
"saxon-js": "^2.5.0",
"source-map-support": "^0.5.21",
@ -2778,9 +2779,9 @@
}
},
"node_modules/@fontsource/archivo-black": {
"version": "5.0.12",
"resolved": "https://registry.npmjs.org/@fontsource/archivo-black/-/archivo-black-5.0.12.tgz",
"integrity": "sha512-zHsdUkz1ax+OEGAJadV5Jglzd7qhCZevOGjWPeuYo8Oh1JhHwKPocPXYU32rVBRWu1rRuvHNYg82+1tYJYZXbA=="
"version": "5.0.13",
"resolved": "https://registry.npmjs.org/@fontsource/archivo-black/-/archivo-black-5.0.13.tgz",
"integrity": "sha512-zFAxd0j3XrVBqSHMLdA0A7ZExGhgM7EO0Sx7MewQdRcYGJ45PBojGIp22f5Deb8DuZB9nESz+qAHABUF1XKpFA=="
},
"node_modules/@fontsource/inter": {
"version": "5.0.8",
@ -3524,6 +3525,64 @@
"truncatise": "0.0.8"
}
},
"node_modules/@redis/bloom": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz",
"integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/client": {
"version": "1.5.11",
"resolved": "https://registry.npmjs.org/@redis/client/-/client-1.5.11.tgz",
"integrity": "sha512-cV7yHcOAtNQ5x/yQl7Yw1xf53kO0FNDTdDU6bFIMbW6ljB7U7ns0YRM+QIkpoqTAt6zK5k9Fq0QWlUbLcq9AvA==",
"dependencies": {
"cluster-key-slot": "1.1.2",
"generic-pool": "3.9.0",
"yallist": "4.0.0"
},
"engines": {
"node": ">=14"
}
},
"node_modules/@redis/client/node_modules/yallist": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
},
"node_modules/@redis/graph": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.0.tgz",
"integrity": "sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg==",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/json": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.6.tgz",
"integrity": "sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw==",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/search": {
"version": "1.1.5",
"resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.5.tgz",
"integrity": "sha512-hPP8w7GfGsbtYEJdn4n7nXa6xt6hVZnnDktKW4ArMaFQ/m/aR7eFvsLQmG/mn1Upq99btPJk+F27IQ2dYpCoUg==",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/time-series": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.5.tgz",
"integrity": "sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg==",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@sinclair/typebox": {
"version": "0.27.8",
"resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz",
@ -4043,9 +4102,9 @@
"dev": true
},
"node_modules/@types/node": {
"version": "20.7.1",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.7.1.tgz",
"integrity": "sha512-LT+OIXpp2kj4E2S/p91BMe+VgGX2+lfO+XTpfXhh+bCk2LkQtHZSub8ewFBMGP5ClysPjTDFa4sMI8Q3n4T0wg=="
"version": "20.8.0",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.8.0.tgz",
"integrity": "sha512-LzcWltT83s1bthcvjBmiBvGJiiUe84NWRHkw+ZV6Fr41z2FbIzvc815dk2nQ3RAKMuN2fkenM/z3Xv2QzEpYxQ=="
},
"node_modules/@types/pino": {
"version": "6.3.12",
@ -6085,9 +6144,9 @@
}
},
"node_modules/caniuse-lite": {
"version": "1.0.30001541",
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001541.tgz",
"integrity": "sha512-bLOsqxDgTqUBkzxbNlSBt8annkDpQB9NdzdTbO2ooJ+eC/IQcvDspDc058g84ejCelF7vHUx57KIOjEecOHXaw==",
"version": "1.0.30001542",
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001542.tgz",
"integrity": "sha512-UrtAXVcj1mvPBFQ4sKd38daP8dEcXXr5sQe6QNNinaPd0iA/cxg9/l3VrSdL73jgw5sKyuQ6jNgiKO12W3SsVA==",
"dev": true,
"funding": [
{
@ -6472,6 +6531,14 @@
"node": ">=6"
}
},
"node_modules/cluster-key-slot": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
"integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/co-compose": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/co-compose/-/co-compose-7.0.3.tgz",
@ -6739,12 +6806,12 @@
}
},
"node_modules/core-js-compat": {
"version": "3.32.2",
"resolved": "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.32.2.tgz",
"integrity": "sha512-+GjlguTDINOijtVRUxrQOv3kfu9rl+qPNdX2LTbJ/ZyVTuxK+ksVSAGX1nHstu4hrv1En/uPTtWgq2gI5wt4AQ==",
"version": "3.33.0",
"resolved": "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.33.0.tgz",
"integrity": "sha512-0w4LcLXsVEuNkIqwjjf9rjCoPhK8uqA4tMRh4Ge26vfLtUutshn+aRJU21I9LCJlh2QQHfisNToLjw1XEJLTWw==",
"dev": true,
"dependencies": {
"browserslist": "^4.21.10"
"browserslist": "^4.22.1"
},
"funding": {
"type": "opencollective",
@ -7973,9 +8040,9 @@
"integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="
},
"node_modules/electron-to-chromium": {
"version": "1.4.535",
"resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.535.tgz",
"integrity": "sha512-4548PpR4S5X5dlvX8NUIw0njH7btQtBoJWcgzpq7n2F9NQ5gMXOPP/6p6iVx6+YT3FVioNhEGa14WJj1k+2SfA==",
"version": "1.4.537",
"resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.537.tgz",
"integrity": "sha512-W1+g9qs9hviII0HAwOdehGYkr+zt7KKdmCcJcjH0mYg6oL8+ioT3Skjmt7BLoAQqXhjf40AXd+HlR4oAWMlXjA==",
"dev": true
},
"node_modules/emittery": {
@ -9206,6 +9273,14 @@
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
"integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
},
"node_modules/generic-pool": {
"version": "3.9.0",
"resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz",
"integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==",
"engines": {
"node": ">= 4"
}
},
"node_modules/gensync": {
"version": "1.0.0-beta.2",
"resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz",
@ -10805,9 +10880,9 @@
}
},
"node_modules/jest-util/node_modules/@types/yargs": {
"version": "17.0.25",
"resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.25.tgz",
"integrity": "sha512-gy7iPgwnzNvxgAEi2bXOHWCVOG6f7xsprVJH4MjlAWeBmJ7vh/Y1kwMtUrs64ztf24zVIRCpr3n/z6gm9QIkgg==",
"version": "17.0.26",
"resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.26.tgz",
"integrity": "sha512-Y3vDy2X6zw/ZCumcwLpdhM5L7jmyGpmBCTYMHDLqT2IKVMYRRLdv6ZakA+wxhra6Z/3bwhNbNl9bDGXaFU+6rw==",
"dev": true,
"dependencies": {
"@types/yargs-parser": "*"
@ -11365,9 +11440,9 @@
"integrity": "sha512-QS9p+Q20YBxpE0dJBnF6CPURP7p1GUsxnhTxTWH5nG3A1F5w8Rg3T4Xyh5UlrFSbHp88oOciVP/0agsNLhkHdQ=="
},
"node_modules/magic-string": {
"version": "0.30.3",
"resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.3.tgz",
"integrity": "sha512-B7xGbll2fG/VjP+SWg4sX3JynwIU0mjoTc6MPpKNuIvftk6u6vqhDnk1R80b8C2GBR6ywqy+1DcKBrevBg+bmw==",
"version": "0.30.4",
"resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.4.tgz",
"integrity": "sha512-Q/TKtsC5BPm0kGqgBIF9oXAs/xEf2vRKiIB4wCRQTJOQIByZ1d+NnUOotvJOvNpi5RNIgVOMC3pOuaP1ZTDlVg==",
"dependencies": {
"@jridgewell/sourcemap-codec": "^1.4.15"
},
@ -13176,9 +13251,9 @@
}
},
"node_modules/postcss": {
"version": "8.4.30",
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.30.tgz",
"integrity": "sha512-7ZEao1g4kd68l97aWG/etQKPKq07us0ieSZ2TnFDk11i0ZfDW2AwKHYU8qv4MZKqN2fdBfg+7q0ES06UA73C1g==",
"version": "8.4.31",
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.31.tgz",
"integrity": "sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ==",
"funding": [
{
"type": "opencollective",
@ -14193,6 +14268,19 @@
"esprima": "~4.0.0"
}
},
"node_modules/redis": {
"version": "4.6.10",
"resolved": "https://registry.npmjs.org/redis/-/redis-4.6.10.tgz",
"integrity": "sha512-mmbyhuKgDiJ5TWUhiKhBssz+mjsuSI/lSZNPI9QvZOYzWvYGejtb+W3RlDDf8LD6Bdl5/mZeG8O1feUGhXTxEg==",
"dependencies": {
"@redis/bloom": "1.2.0",
"@redis/client": "1.5.11",
"@redis/graph": "1.1.0",
"@redis/json": "1.0.6",
"@redis/search": "1.1.5",
"@redis/time-series": "1.0.5"
}
},
"node_modules/reflect-metadata": {
"version": "0.1.13",
"resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz",

2
package.json
View File

@ -5,6 +5,7 @@
"scripts": {
"type-check": "tsc --noEmit",
"dev": "node ace serve --watch",
"compress:xslt": "./node_modules/xslt3/xslt3.js -xsl:public/assets2/datasetxml2oai-pmh.xslt -export:public/assets2/datasetxml2oai.sef.json -t -nogo '-ns:##html5'",
"build": "node ace build --production",
"start": "node server.js",
"lint": "eslint . --ext=.ts",
@ -86,6 +87,7 @@
"notiwind": "^2.0.0",
"pg": "^8.9.0",
"proxy-addr": "^2.0.7",
"redis": "^4.6.10",
"reflect-metadata": "^0.1.13",
"saxon-js": "^2.5.0",
"source-map-support": "^0.5.21",

61
providers/TokenWorkerProvider.ts Normal file
View File

@ -0,0 +1,61 @@
import type { ApplicationContract } from '@ioc:Adonis/Core/Application';
/*
|--------------------------------------------------------------------------
| Provider
|--------------------------------------------------------------------------
|
| Your application is not ready when this file is loaded by the framework.
| Hence, the top level imports relying on the IoC container will not work.
| You must import them inside the life-cycle methods defined inside
| the provider class.
|
| @example:
|
| public async ready () {
| const Database = this.app.container.resolveBinding('Adonis/Lucid/Database')
| const Event = this.app.container.resolveBinding('Adonis/Core/Event')
| Event.on('db:query', Database.prettyPrint)
| }
|
*/
export default class TokenWorkerProvider {
public static needsApplication = true;
private tokenWorkerInstance; //: TokenWorkerService | null = null;
constructor(protected app: ApplicationContract) {}
public register() {
// Register your own bindings
// Bind TokenWorker to the IoC container
this.app.container.singleton('App/Library/Oai/TokenWorkerContract', () => {
// 1. import the oai configuration
const ttl: number = 86400;
// 2. import our REDIS wrapper class
const TokenWorkerService = require('App/Library/Oai/TokenWorkerSerice').default;
this.tokenWorkerInstance = new TokenWorkerService(ttl);
// 3. return a new instance
return this.tokenWorkerInstance;
});
}
// public async boot() {
// // All bindings are ready, feel free to use them
// // optionally do some initial setup
// }
// public async ready() {
// // App is ready
// }
public async shutdown() {
console.log('TokenServerProvider shutdown()');
// Cleanup, since app is going down
if (this.tokenWorkerInstance) {
// Call the disconnect method when the application is shutting down
await this.tokenWorkerInstance.close();
}
}
}

4
public/assets/manifest.json
View File

@ -8,10 +8,10 @@
"assets/fonts/inter-latin-400-normal.woff2": "http://localhost:8080/assets/fonts/inter-latin-400-normal.be7cb18d.woff2",
"assets/fonts/archivo-black-latin-ext-400-normal.woff2": "http://localhost:8080/assets/fonts/archivo-black-latin-ext-400-normal.21761451.woff2",
"assets/fonts/inter-cyrillic-ext-400-normal.woff": "http://localhost:8080/assets/fonts/inter-cyrillic-ext-400-normal.3c63e274.woff",
"assets/fonts/archivo-black-latin-400-normal.woff": "http://localhost:8080/assets/fonts/archivo-black-latin-400-normal.58a301a6.woff",
"assets/fonts/archivo-black-latin-400-normal.woff": "http://localhost:8080/assets/fonts/archivo-black-latin-400-normal.583e4fc9.woff",
"assets/fonts/inter-greek-400-normal.woff": "http://localhost:8080/assets/fonts/inter-greek-400-normal.b31b8612.woff",
"assets/fonts/inter-cyrillic-ext-400-normal.woff2": "http://localhost:8080/assets/fonts/inter-cyrillic-ext-400-normal.fcc125c4.woff2",
"assets/fonts/archivo-black-latin-ext-400-normal.woff": "http://localhost:8080/assets/fonts/archivo-black-latin-ext-400-normal.5ab5ba92.woff",
"assets/fonts/archivo-black-latin-ext-400-normal.woff": "http://localhost:8080/assets/fonts/archivo-black-latin-ext-400-normal.ce39b04f.woff",
"assets/fonts/inter-cyrillic-400-normal.woff": "http://localhost:8080/assets/fonts/inter-cyrillic-400-normal.3862a5ab.woff",
"assets/fonts/inter-greek-400-normal.woff2": "http://localhost:8080/assets/fonts/inter-greek-400-normal.0278a49f.woff2",
"assets/fonts/inter-greek-ext-400-normal.woff": "http://localhost:8080/assets/fonts/inter-greek-ext-400-normal.61350b97.woff",

25
public/assets2/datasetxml2oai-pmh.xslt
View File

@ -43,7 +43,6 @@
<xsl:param name="oai_until" />
<xsl:param name="oai_set" /> -->
<xsl:variable name="langCodes" select="document('langCodeMap.xml')/langCodeMap/langCode"/>
<!-- Characters we'll support.
We could add control chars 0-31 and 127-159, but we won't. -->
@ -739,13 +738,31 @@
</xsl:template>
<xsl:template match="@Language" mode="oai_dc">
<xsl:variable name="language" select="string(.)"/>
<dc:language>
<!-- <xsl:value-of select="." /> -->
<xsl:value-of select="$langCodes[@iso639-1 = $language]/@iso639-2" />
<xsl:choose>
<xsl:when test="string(.)='de'"> <xsl:value-of select="'ger'"/></xsl:when>
<xsl:when test="string(.)='en'"> <xsl:value-of select="'eng'"/></xsl:when>
<xsl:when test="string(.)='es'"> <xsl:value-of select="'spa'"/></xsl:when>
<xsl:when test="string(.)='it'"> <xsl:value-of select="'ita'"/></xsl:when>
<xsl:otherwise>
<xsl:value-of select="string(.)"/>
</xsl:otherwise>
</xsl:choose>
</dc:language>
</xsl:template>
<!-- <xsl:template match="@Language" mode="oai_dc">
<dc:language>
<xsl:variable name="langMap">
<entry input="de" output="ger"/>
<entry input="en" output="eng"/>
<entry input="es" output="spa"/>
<entry input="it" output="ita"/>
</xsl:variable>
<xsl:value-of select="$langMap/entry[@input = string(.)]/@output" />
</dc:language>
</xsl:template> -->
<xsl:template match="Licence" mode="oai_dc">
<dc:rights>
<xsl:value-of select="@NameLong" />

2
public/assets2/datasetxml2oai.sef.json
View File

File diff suppressed because one or more lines are too long

8
public/assets2/langCodeMap.xml
View File

@ -1,8 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- langCodeMap.xml -->
<langCodeMap>
<langCode iso639-1="de" iso639-2="ger"/>
<langCode iso639-1="en" iso639-2="eng"/>
<langCode iso639-1="es" iso639-2="spa"/>
<langCode iso639-1="it" iso639-2="ita"/>
</langCodeMap>