geolba/tethys.api
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- npm updates, delete cd 0.3.3

Browse Source 
- aslo add collection for GetRecord request
- add src/utils/utility-functions.ts
- don't create empty xml caches
This commit is contained in:
Arno Kaimbacher 2022-12-06 13:52:55 +01:00
parent ea568123f1
commit 10b1fb0c86
10 changed files with 133 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
assets/oai2_style.xslt
View File

@ -65,7 +65,7 @@
<!-- root template --> <!-- root template -->
<xsl:template match="/"> <xsl:template match="/">
<html> <html>
<head> <head>
<title>TETHYS OAI 2.0 Request Results</title> <title>TETHYS OAI 2.0 Request Results</title>
<!-- <style><xsl:call-template name="style"/></style> --> <!-- <style><xsl:call-template name="style"/></style> -->
<link rel="stylesheet" type="text/css" href="assets/style.css" /> <link rel="stylesheet" type="text/css" href="assets/style.css" />
@ -76,7 +76,7 @@
<link rel="icon" type="image/png" sizes="16x16" href="/images/favicon/favicon-16x16.png"/> <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon/favicon-16x16.png"/>
<link rel="manifest" href="/images/favicon/site.webmanifest"/> <link rel="manifest" href="/images/favicon/site.webmanifest"/>
<link rel="mask-icon" href="/images/favicon/safari-pinned-tab.svg" color="#5bbad5"/> --> <link rel="mask-icon" href="/images/favicon/safari-pinned-tab.svg" color="#5bbad5"/> -->
<link rel="shortcut icon" href="/images/favicon/favicon.ico"/> <link rel="shortcut icon" href="/images/favicon/favicon.ico"/>
<!-- <style>h2{font-size:22pt;text-decoration:underline}para{color:green}</style> --> <!-- <style>h2{font-size:22pt;text-decoration:underline}para{color:green}</style> -->
</head> </head>

73
datasetxml2oai-pmh.xslt
View File

@ -8,8 +8,6 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd"> xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
<!--<xsl:param name="urnResolverUrl" />-->
<!-- add include here for each new metadata format --> <!-- add include here for each new metadata format -->
<xsl:include href="assets/oai_datacite.xslt" /> <xsl:include href="assets/oai_datacite.xslt" />
<xsl:include href="assets/oai_2_iso19139.xslt" /> <xsl:include href="assets/oai_2_iso19139.xslt" />
@ -47,11 +45,56 @@
<xsl:variable name="langCodes" select="document('assets/langCodeMap.xml')/langCodeMap/langCode"/> <xsl:variable name="langCodes" select="document('assets/langCodeMap.xml')/langCodeMap/langCode"/>
<!-- Characters we'll support.
We could add control chars 0-31 and 127-159, but we won't. -->
<xsl:variable name="ascii"> !"#$%&amp;'()*+,-./0123456789:;&lt;=&gt;?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~</xsl:variable>
<xsl:variable name="latin1">&#160;&#161;&#162;&#163;&#164;&#165;&#166;&#167;&#168;&#169;&#170;&#171;&#172;&#173;&#174;&#175;&#176;&#177;&#178;&#179;&#180;&#181;&#182;&#183;&#184;&#185;&#186;&#187;&#188;&#189;&#190;&#191;&#192;&#193;&#194;&#195;&#196;&#197;&#198;&#199;&#200;&#201;&#202;&#203;&#204;&#205;&#206;&#207;&#208;&#209;&#210;&#211;&#212;&#213;&#214;&#215;&#216;&#217;&#218;&#219;&#220;&#221;&#222;&#223;&#224;&#225;&#226;&#227;&#228;&#229;&#230;&#231;&#232;&#233;&#234;&#235;&#236;&#237;&#238;&#239;&#240;&#241;&#242;&#243;&#244;&#245;&#246;&#247;&#248;&#249;&#250;&#251;&#252;&#253;&#254;&#255;</xsl:variable>
<!-- Characters that usually don't need to be escaped -->
<xsl:variable name="safe">!'()*-.0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz~</xsl:variable>
<xsl:variable name="hex" >0123456789ABCDEF</xsl:variable>
<xsl:template name="url-encode">
<xsl:param name="str"/>
<xsl:if test="$str">
<xsl:variable name="first-char" select="substring($str,1,1)"/>
<xsl:choose>
<xsl:when test="contains($safe,$first-char)">
<xsl:value-of select="$first-char"/>
</xsl:when>
<xsl:otherwise>
<xsl:variable name="codepoint">
<xsl:choose>
<xsl:when test="contains($ascii,$first-char)">
<xsl:value-of select="string-length(substring-before($ascii,$first-char)) + 32"/>
</xsl:when>
<xsl:when test="contains($latin1,$first-char)">
<xsl:value-of select="string-length(substring-before($latin1,$first-char)) + 160"/>
</xsl:when>
<xsl:otherwise>
<xsl:message terminate="no">Warning: string contains a character that is out of range! Substituting "?".</xsl:message>
<xsl:text>63</xsl:text>
</xsl:otherwise>
</xsl:choose>
</xsl:variable>
<xsl:variable name="hex-digit1" select="substring($hex,floor($codepoint div 16) + 1,1)"/>
<xsl:variable name="hex-digit2" select="substring($hex,$codepoint mod 16 + 1,1)"/>
<xsl:value-of select="concat('%',$hex-digit1,$hex-digit2)"/>
</xsl:otherwise>
</xsl:choose>
<xsl:if test="string-length($str) &gt; 1">
<xsl:call-template name="url-encode">
<xsl:with-param name="str" select="substring($str,2)"/>
</xsl:call-template>
</xsl:if>
</xsl:if>
</xsl:template>
<!--create the head of oai response --> <!--create the head of oai response -->
<xsl:template match="/root"> <xsl:template match="/root">
<!-- stylesheet for browser --> <!-- stylesheet for browser -->
<xsl:processing-instruction name="xml-stylesheet"> <xsl:processing-instruction name="xml-stylesheet">
<xsl:text>type="text/xsl" href="../assets/oai2_style.xslt" title="Defaultstyle"</xsl:text> <xsl:text>type="text/xsl" href="assets/oai2_style.xslt"</xsl:text>
</xsl:processing-instruction> </xsl:processing-instruction>
<OAI-PMH xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd"> <OAI-PMH xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
@ -416,7 +459,10 @@
<xsl:apply-templates select="Identifier" mode="oai_dc" /> <xsl:apply-templates select="Identifier" mode="oai_dc" />
</xsl:if> --> </xsl:if> -->
<dc:identifier> <dc:identifier>
<xsl:value-of select="@landingpage" /> <xsl:value-of select="string(@landingpage)" />
<!-- <xsl:call-template name="url-encode">
<xsl:with-param name="str" select="@landingpage"/>
</xsl:call-template> -->
</dc:identifier> </dc:identifier>
<!-- dc:language --> <!-- dc:language -->
<xsl:apply-templates select="@Language" mode="oai_dc" /> <xsl:apply-templates select="@Language" mode="oai_dc" />
@ -553,21 +599,23 @@
<xsl:value-of select="@Value" /> <xsl:value-of select="@Value" />
</dc:subject> </dc:subject>
</xsl:template> </xsl:template>
<xsl:template match="Collection" mode="oai_dc"> <xsl:template match="Collection" mode="oai_dc">
<dc:subject> <dc:subject>
<!-- <xsl:if test="@Language != ''"> <!-- <xsl:if test="@Language != ''">
<xsl:attribute name="xml:lang"> <xsl:attribute name="xml:lang">
<xsl:value-of select="@Language" /> <xsl:value-of select="@Language" />
</xsl:attribute> </xsl:attribute>
</xsl:if> --> </xsl:if> -->
<xsl:value-of select="concat(string(@Collectionrole.Name), ':', string(@Number))" /> <xsl:value-of select="concat(string(@Collectionrole.Name), ':', string(@Number))" />
</dc:subject> </dc:subject>
</xsl:template> </xsl:template>
<xsl:template match="Reference" mode="oai_dc"> <xsl:template match="Reference" mode="oai_dc">
<dc:relation> <dc:relation>
<xsl:value-of select="@Value" /> <xsl:value-of select="string(@Value)" />
<!-- <xsl:call-template name="url-encode">
<xsl:with-param name="str" select="@Value"/>
</xsl:call-template> -->
</dc:relation> </dc:relation>
</xsl:template> </xsl:template>
@ -673,7 +721,10 @@
<xsl:template match="Identifier" mode="oai_dc"> <xsl:template match="Identifier" mode="oai_dc">
<dc:relation> <dc:relation>
<!-- <xsl:value-of select="concat($doiLink, @Value)" /> --> <!-- <xsl:value-of select="concat($doiLink, @Value)" /> -->
<xsl:value-of select="concat($doiPrefix, @Value)" /> <xsl:value-of select="concat($doiPrefix, string(@Value))" />
<!-- <xsl:call-template name="url-encode">
<xsl:with-param name="str" select="concat($doiPrefix, @Value)"/>
</xsl:call-template> -->
</dc:relation> </dc:relation>
</xsl:template> </xsl:template>

1
package.json
View File

@ -27,7 +27,6 @@
"dependencies": { "dependencies": {
"@overnightjs/core": "^1.7.6", "@overnightjs/core": "^1.7.6",
"body-parser": "^1.20.0", "body-parser": "^1.20.0",
"cd": "^0.3.3",
"core-js": "^3.25.5", "core-js": "^3.25.5",
"cors": "^2.8.5", "cors": "^2.8.5",
"dayjs": "^1.11.5", "dayjs": "^1.11.5",

6
src/app.ts
View File

@ -40,14 +40,14 @@ export class App extends Server {
next(); next();
}); });
// this.app.use(bodyParser.json()); // this.app.use(bodyParser.json());
// for parsing application/json // for parsing application/json
this.app.use(bodyParser.json({ limit: "100mb" })); this.app.use(bodyParser.json({ limit: "100mb" }));
// for parsing application/xwww- // for parsing application/xwww-
this.app.use(bodyParser.urlencoded({ limit: "50mb", extended: true })); this.app.use(bodyParser.urlencoded({ limit: "50mb", extended: true }));
// this.app.use(forms.array()); // this.app.use(forms.array());
// for parsing multipart/form-data = uploads // for parsing multipart/form-data = uploads
// this.app.use(upload.array()); // this.app.use(upload.array());
} }
private boostrap(): void { private boostrap(): void {

2
src/controllers/datasetxml2oai.sef.json
View File

File diff suppressed because one or more lines are too long

50
src/controllers/oai.controller.ts
View File

@ -11,7 +11,6 @@ import dayjs, { Dayjs } from "dayjs";
import { Dataset, Project, License, Collection, CollectionRole } from "../models/init-models"; import { Dataset, Project, License, Collection, CollectionRole } from "../models/init-models";
import Logger from "jet-logger"; import Logger from "jet-logger";
import { BadOaiModelException, OaiModelException } from "../exceptions/OaiModelException"; import { BadOaiModelException, OaiModelException } from "../exceptions/OaiModelException";
import PageNotFoundException from "../exceptions/PageNotFoundException";
import { OaiErrorCodes, OaiModelError } from "../exceptions/OaiErrorCodes"; import { OaiErrorCodes, OaiModelError } from "../exceptions/OaiErrorCodes";
import XmlModel from "../library/XmlModel"; import XmlModel from "../library/XmlModel";
import Configuration from "../library/oai/OaiConfiguration"; import Configuration from "../library/oai/OaiConfiguration";
@ -19,6 +18,7 @@ import ResumptionToken from "../library/oai/ResumptionToken";
import TokenWorker from "../library/oai/TokenWorker"; import TokenWorker from "../library/oai/TokenWorker";
import utc from "dayjs/plugin/utc"; import utc from "dayjs/plugin/utc";
import timezone from "dayjs/plugin/timezone"; import timezone from "dayjs/plugin/timezone";
import { getDomain } from "../utils/utility-functions";
interface XslTParameter { interface XslTParameter {
[key: string]: any; [key: string]: any;
@ -102,7 +102,7 @@ export class OaiController {
await this.handleRequest(oaiRequest, request); await this.handleRequest(oaiRequest, request);
} catch (error) { } catch (error) {
if (error instanceof OaiModelException) { if (error instanceof OaiModelException) {
let code = error.oaiCode; const code = error.oaiCode;
let oaiErrorCode: string | undefined = "Unknown oai error code " + code; let oaiErrorCode: string | undefined = "Unknown oai error code " + code;
if (OaiModelError.has(error.oaiCode) && OaiModelError.get(code) != undefined) { if (OaiModelError.has(error.oaiCode) && OaiModelError.get(code) != undefined) {
oaiErrorCode = OaiModelError.get(error.oaiCode); oaiErrorCode = OaiModelError.get(error.oaiCode);
@ -291,7 +291,7 @@ export class OaiController {
const dataset = await Dataset.findOne({ const dataset = await Dataset.findOne({
where: { publish_id: dataId }, where: { publish_id: dataId },
include: ["xmlCache"], include: ["xmlCache", "collections"],
// order: ['server_date_published'], // order: ['server_date_published'],
}); });
if (!dataset || !dataset.publish_id) { if (!dataset || !dataset.publish_id) {
@ -683,29 +683,11 @@ export class OaiController {
private addLandingPageAttribute(domNode: XMLBuilder, dataid: string) { private addLandingPageAttribute(domNode: XMLBuilder, dataid: string) {
const baseDomain = process.env.BASE_DOMAIN || "localhost"; const baseDomain = process.env.BASE_DOMAIN || "localhost";
const url = "https://" + this.getDomain(baseDomain) + "/dataset/" + dataid; const url = "https://" + getDomain(baseDomain) + "/dataset/" + dataid;
// add attribute du dataset xml element // add attribute du dataset xml element
domNode.att("landingpage", url); domNode.att("landingpage", url);
} }
private getDomain(host: string): string {
// $myhost = strtolower(trim($host));
let myHost: string = host.trim().toLocaleLowerCase();
// $count = substr_count($myhost, '.');
const count: number = myHost.split(",").length - 1;
if (count == 2) {
const words = myHost.split(".");
if (words[1].length > 3) {
myHost = myHost.split(".", 2)[1];
}
} else if (count > 2) {
myHost = this.getDomain(myHost.split(".", 2)[1]);
}
myHost = myHost.replace(new RegExp(/^.*:\/\//i, "g"), "");
return myHost;
}
private getDocumentIdByIdentifier(oaiIdentifier: string): string { private getDocumentIdByIdentifier(oaiIdentifier: string): string {
const identifierParts: string[] = oaiIdentifier.split(":"); // explode(":", $oaiIdentifier); const identifierParts: string[] = oaiIdentifier.split(":"); // explode(":", $oaiIdentifier);
const dataId: string = identifierParts[2]; const dataId: string = identifierParts[2];
@ -735,20 +717,22 @@ export class OaiController {
private async createXmlRecord(dataset: Dataset, datasetNode: XMLBuilder) { private async createXmlRecord(dataset: Dataset, datasetNode: XMLBuilder) {
const domNode = await this.getDatasetXmlDomNode(dataset); const domNode = await this.getDatasetXmlDomNode(dataset);
// add frontdoor url and data-type if (domNode) {
// if (dataset.publish_id) { // add frontdoor url and data-type
dataset.publish_id && this.addLandingPageAttribute(domNode, dataset.publish_id.toString()); // if (dataset.publish_id) {
// } dataset.publish_id && this.addLandingPageAttribute(domNode, dataset.publish_id.toString());
this.addSpecInformation(domNode, "data-type:" + dataset.type); // }
this.addSpecInformation(domNode, "data-type:" + dataset.type);
if (dataset.collections) { if (dataset.collections) {
for (const coll of dataset.collections) { for (const coll of dataset.collections) {
const collRole = await coll.getCollectionRole(); const collRole = await coll.getCollectionRole();
this.addSpecInformation(domNode, collRole.oai_name + ":" + coll.number); this.addSpecInformation(domNode, collRole.oai_name + ":" + coll.number);
}
} }
}
datasetNode.import(domNode); datasetNode.import(domNode);
}
} }
private async getDatasetXmlDomNode(dataset: Dataset) { private async getDatasetXmlDomNode(dataset: Dataset) {

47
src/library/XmlModel.ts
View File

@ -115,32 +115,33 @@ export default class XmlModel {
} else { } else {
//create domDocument during runtime //create domDocument during runtime
// domDocument = $this->strategy->getDomDocument(); // domDocument = $this->strategy->getDomDocument();
domDocument = create({ version: "1.0", encoding: "UTF-8", standalone: true }, "<root></root>"); // domDocument = create({ version: "1.0", encoding: "UTF-8", standalone: true }, "<root></root>");
return null;
} }
//if caching isn't wanted return only dom Document // //if caching isn't wanted return only dom Document
if (this._caching != true) { // if (this._caching != true) {
return domDocument; // return domDocument;
//otherwise caching is desired: // //otherwise caching is desired:
// save xml cache to db and return domDocument // // save xml cache to db and return domDocument
} else { // } else {
// save new DocumentXmlCache // // save new DocumentXmlCache
if (!this.cache) { // if (!this.cache) {
this.cache = new DocumentXmlCache(); // this.cache = new DocumentXmlCache();
this.cache.document_id = dataset.id; // this.cache.document_id = dataset.id;
} // }
// if (!this.cache.document_id) { // // if (!this.cache.document_id) {
// this.cache.document_id = dataset.id; // // this.cache.document_id = dataset.id;
// } // // }
this.cache.xml_version = 1; // (int)$this->strategy->getVersion(); // this.cache.xml_version = 1; // (int)$this->strategy->getVersion();
this.cache.server_date_modified = dayjs(dataset.server_date_modified).format("YYYY-MM-DD HH:mm:ss"); // this.cache.server_date_modified = dayjs(dataset.server_date_modified).format("YYYY-MM-DD HH:mm:ss");
this.cache.xml_data = domDocument.end(); // this.cache.xml_data = domDocument.end();
//save xml cache // //save xml cache
this.cache.save(); // this.cache.save();
// return newly created xml cache // // return newly created xml cache
return domDocument; // return domDocument;
} // }
} }
private async getDomDocumentFromXmlCache(): Promise<XMLBuilder | null> { private async getDomDocumentFromXmlCache(): Promise<XMLBuilder | null> {

2
src/models/Dataset.ts
View File

@ -79,7 +79,7 @@ class Dataset extends Model<InferAttributes<Dataset>, InferCreationAttributes<Da
declare collections?: NonAttribute<Collection[]>; declare collections?: NonAttribute<Collection[]>;
// declare static associations: { // declare static associations: {
// }; // };
// getters that are not attributes should be tagged using NonAttribute // getters that are not attributes should be tagged using NonAttribute

4
src/models/init-models.js
View File

@ -229,8 +229,8 @@ export function initModels() {
}); });
// dataset and collections // dataset and collections
//licences //licences
const DatasetCollection = sequelizeConnection.define( const DatasetCollection = sequelizeConnection.define(
"link_documents_collections", "link_documents_collections",
{}, {},
{ {

21
src/utils/utility-functions.ts Normal file
View File

@ -0,0 +1,21 @@
export function sum(a: number, b: number): number {
return a + b;
}
export function getDomain(host: string): string {
// $myhost = strtolower(trim($host));
let myHost: string = host.trim().toLocaleLowerCase();
// $count = substr_count($myhost, '.');
const count: number = myHost.split(",").length - 1;
if (count == 2) {
const words = myHost.split(".");
if (words[1].length > 3) {
myHost = myHost.split(".", 2)[1];
}
} else if (count > 2) {
myHost = getDomain(myHost.split(".", 2)[1]);
}
myHost = myHost.replace(new RegExp(/^.*:\/\//i, "g"), "");
return myHost;
}