- add classes inside app/library for creting Tethys xml: Field.ts, Strategy.ts, XmlModel.ts

- added model DocumentXmlCache.ts
- npm updates
- changed all models inside app/Models to use corrected BaseModel.ts
- added extra extension class DatasetExtension.ts for app/dataset.ts for caching internal and external fields

app/Controllers/Http/Submitter/DatasetController.ts

@@ -33,6 +33,8 @@ import ClamScan from 'clamscan';
 import { ValidationException } from '@ioc:Adonis/Core/Validator';
 import Drive from '@ioc:Adonis/Core/Drive';
 import { Exception } from '@adonisjs/core/build/standalone';
+import XmlModel from 'App/Library/XmlModel';
+import { XMLBuilder } from 'xmlbuilder2/lib/interfaces';
 
 export default class DatasetController {
     public async index({ auth, request, inertia }: HttpContextContract) {
@@ -342,9 +344,9 @@ export default class DatasetController {
         // Create a new instance of the Dataset model:
         const dataset = new Dataset();
         dataset.type = request.input('type');
-        dataset.creatingCorporation = request.input('creating_corporation');
+        dataset.creating_corporation = request.input('creating_corporation');
         dataset.language = request.input('language');
-        dataset.embargoDate = request.input('embargo_date');
+        dataset.embargo_date = request.input('embargo_date');
         //await dataset.related('user').associate(user); // speichert schon ab
         // Dataset.$getRelation('user').boot();
         // Dataset.$getRelation('user').setRelated(dataset, user);
@@ -577,6 +579,23 @@ export default class DatasetController {
         'files.extnames': 'file extension is not supported',
     };
 
+    private async getDatasetXmlDomNode(dataset: Dataset) {
+        // dataset.fetchValues();
+        const xmlModel = new XmlModel(dataset);
+        // xmlModel.setModel(dataset);
+        xmlModel.excludeEmptyFields();
+        xmlModel.caching = true;
+        // const cache = dataset.xmlCache ? dataset.xmlCache : null;
+        // dataset.load('xmlCache');
+        if (dataset.xmlCache) {
+            xmlModel.xmlCache = dataset.xmlCache;
+        }
+
+        // return cache.getDomDocument();
+        const domDocument: XMLBuilder | null = await xmlModel.getDomDocument();
+        return domDocument;
+    }
+
     // public async release({ params, view }) {
     public async release({ request, inertia, response }: HttpContextContract) {
         const id = request.param('id');

app/Library/Field.ts

@@ -0,0 +1,69 @@
+export default class Field {
+    // private _multiplicity: number | string = 1;
+    private _hasMultipleValues: boolean = false;
+    private _valueModelClass: string | null = null;
+    private _linkModelClass: string | null = null;
+    // private _owningModelClass: string | null = null;
+    private _value: any;
+    private _name: string;
+
+    constructor(name: string) {
+        this._name = name;
+        this._value = null;
+    }
+
+    getValueModelClass(): any {
+        return this._valueModelClass;
+    }
+
+    setValueModelClass(classname: any): Field {
+        this._valueModelClass = classname;
+        return this;
+    }
+
+    getName(): string {
+        return this._name;
+    }
+
+    // setOwningModelClass(classname: string): Field {
+    //     this._owningModelClass = classname;
+    //     return this;
+    // }
+
+    public setValue(value: string | string[] | number | boolean): Field {
+        if (value === null || value === this._value) {
+            return this;
+        }
+
+        if (Array.isArray(value) && value.length === 0) {
+            value = [];
+        } else if (typeof value === 'boolean') {
+            value = value ? 1 : 0;
+        } else {
+            this._value = value;
+        }
+        return this;
+    }
+    public getValue() {
+        return this._value;
+    }
+
+    hasMultipleValues(): boolean {
+        return this._hasMultipleValues;
+    }
+
+    setMultiplicity(max: number | '*'): Field {
+        if (max !== '*') {
+            if (typeof max !== 'number' || max < 1) {
+                throw new Error('Only integer values > 1 or "*" allowed.');
+            }
+        }
+        // this._multiplicity = max;
+        this._hasMultipleValues = (typeof max == 'number' && max > 1) || max === '*';
+        return this;
+    }
+
+    getLinkModelClass(): string | null {
+        return this._linkModelClass;
+    }
+}

app/Library/Strategy.ts

@@ -0,0 +1,177 @@
+import { XMLBuilder } from 'xmlbuilder2/lib/interfaces';
+import { create } from 'xmlbuilder2';
+import Dataset from 'App/Models/Dataset';
+import Field from './Field';
+import BaseModel from 'App/Models/BaseModel';
+import { DateTime } from 'luxon';
+
+export default class Strategy {
+    private version: number;
+    private config;
+    private xml: XMLBuilder;
+
+    constructor(config) {
+        this.version = 1.0;
+        this.config = config;
+    }
+    public async createDomDocument(): Promise<XMLBuilder> {
+        if (this.config.model === null) {
+            throw new Error('No Model given for serialization.');
+        }
+        // domDocument = create({ version: '1.0', encoding: 'UTF-8', standalone: true }, '<root></root>');
+
+        this.xml = create({ version: '1.0', encoding: 'UTF-8' }, '<Opus></Opus>');
+        this.xml.root().att('version', this.version.toString());
+        // this.xml.root().att('version', this.getVersion());
+        this.xml.root().att('xmlns:xlink', 'http://www.w3.org/1999/xlink');
+
+        await this._mapModel(this.config.model);
+        return this.xml; //.end({ prettyPrint: true });
+    }
+
+    private async _mapModel(model: Dataset) {
+        const fields: Array<string> = await model.describe();
+        const excludeFields = this.getConfig().excludeFields;
+        let fieldsDiff;
+
+        if (excludeFields.length > 0) {
+            fieldsDiff = fields.filter((fieldname) => !excludeFields.includes(fieldname));
+        } else {
+            fieldsDiff = fields;
+        }
+
+        const modelNode: XMLBuilder = this.createModelNode(model);
+        // rootNode.appendChild(childNode);
+
+        for (const fieldname of fieldsDiff) {
+            const field = model.getField(fieldname);
+            this.mapField(field, modelNode);
+        }
+    }
+
+    private mapField(field, modelNode: XMLBuilder) {
+        const modelClass = field.getValueModelClass();
+        let fieldValues = field.getValue();
+
+        if (this.config.excludeEmpty) {
+            if (
+                fieldValues === null ||
+                (typeof fieldValues === 'string' && fieldValues.trim() === '') ||
+                (Array.isArray(fieldValues) && fieldValues.length === 0)
+            ) {
+                return;
+            }
+        }
+
+        if (modelClass === null) {
+            this.mapSimpleField(modelNode, field);
+        } else {
+            // map related models with values:
+            // let modelInstance = new modelClass();
+            const fieldName = field.getName();
+
+            if (!Array.isArray(fieldValues)) {
+                fieldValues = [fieldValues];
+            }
+
+            for (const value of fieldValues) {
+                const childNode = modelNode.ele(fieldName);
+                // rootNode.appendChild(childNode);
+                // if a field has no value then there is nothing more to do
+                // TODO maybe there must be another solution
+                if (value === null) {
+                    continue;
+                }
+                if (modelClass.prototype instanceof BaseModel) {
+                    this.mapModelAttributes(value, childNode);
+                } else if (modelClass instanceof DateTime) {
+                    // console.log('Value is a luxon date');
+                    this.mapDateAttributes(value, childNode);
+                } else if (Array.isArray(value)) {
+                    console.log('Value is an array');
+                    // this.mapArrayAttributes(value, childNode);
+                }
+            }
+        }
+    }
+
+    private mapDateAttributes(model: DateTime, childNode: XMLBuilder) {
+        childNode.att('Year', model.year.toString());
+        childNode.att('Month', model.month.toString());
+        childNode.att('Day', model.day.toString());
+        childNode.att('Hour', model.hour.toString());
+        childNode.att('Minute', model.minute.toString());
+        childNode.att('Second', model.second.toString());
+        childNode.att('UnixTimestamp', model.toUnixInteger().toString());
+        let zoneName = model.zoneName ? model.zoneName : '';
+        childNode.att('Timezone', zoneName);
+    }
+
+    private mapModelAttributes(myObject, childNode: XMLBuilder) {
+        Object.keys(myObject).forEach((prop) => {
+            let value = myObject[prop];
+            console.log(`${prop}: ${value}`);
+            if (value != null) {
+                if (value instanceof DateTime) {
+                    value = value.toFormat('yyyy-MM-dd HH:mm:ss').trim();
+                } else {
+                    value = value.toString().trim();
+                }
+
+                // Replace invalid XML-1.0-Characters by UTF-8 replacement character.
+                let fieldValues = value.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '\xEF\xBF\xBD ');
+
+                // Create an attribute with the field name and field values
+                // const attr = { [fieldName]: fieldValues };
+
+                // Add the attribute to the root element
+                childNode.att(prop, fieldValues);
+            }
+        });
+    }
+
+    private mapSimpleField(modelNode: XMLBuilder, field: Field) {
+        const fieldName = field.getName();
+        let fieldValues = this.getFieldValues(field);
+
+        if (fieldValues != null) {
+            // Replace invalid XML-1.0-Characters by UTF-8 replacement character.
+            fieldValues = fieldValues.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '\xEF\xBF\xBD ');
+
+            // Create an attribute with the field name and field values
+            // const attr = { [fieldName]: fieldValues };
+
+            // Add the attribute to the root element
+            modelNode.att(fieldName, fieldValues);
+        }
+    }
+
+    private getFieldValues(field: any): string {
+        let fieldValues: number | string | Array<string> = field.getValue(); //275
+
+        // workaround for simple fields with multiple values
+        if (field.hasMultipleValues() === true && Array.isArray(fieldValues)) {
+            fieldValues = fieldValues.join(',');
+        }
+        // Uncomment if needed
+        // if (fieldValues instanceof DateTimeZone) {
+        //     fieldValues = fieldValues.getName();
+        // }
+
+        return fieldValues?.toString().trim();
+    }
+
+    private createModelNode(model) {
+        const className = 'Rdr_' + model.constructor.name.split('\\').pop(); //Rdr_Dataset
+        // return dom.createElement(className);
+        return this.xml.root().ele(className);
+    }
+
+    // private getVersion() {
+    //     return Math.floor(this.version);
+    // }
+
+    private getConfig() {
+        return this.config;
+    }
+}

app/Library/XmlModel.ts

@@ -0,0 +1,117 @@
+import DocumentXmlCache from 'App/Models/DocumentXmlCache';
+import { XMLBuilder } from 'xmlbuilder2/lib/interfaces';
+import Dataset from 'App/Models/Dataset';
+import Strategy from './Strategy';
+import { DateTime } from 'luxon';
+
+/**
+ * This is the description of the interface
+ *
+ * @interface Conf
+ * @member {Model} model holds the current dataset model
+ * @member {XMLBuilder} dom holds the current DOM representation
+ * @member {Array<string>} excludeFields List of fields to skip on serialization.
+ * @member {boolean} excludeEmpty True, if empty fields get excluded from serialization.
+ * @member {string} baseUri Base URI for xlink:ref elements
+ */
+export interface Conf {
+    model: Dataset;
+    dom?: XMLBuilder;
+    excludeFields: Array<string>;
+    excludeEmpty: boolean;
+    baseUri: string;
+}
+
+export default class XmlModel {
+    private config: Conf;
+    // private strategy = null;
+    private cache: DocumentXmlCache | null = null;
+    private _caching = false;
+    private strategy: Strategy;
+
+    constructor(dataset: Dataset) {
+        // $this->strategy = new Strategy();// Opus_Model_Xml_Version1;
+        // $this->config = new Conf();
+        // $this->strategy->setup($this->config);
+
+        this.config = {
+            excludeEmpty: false,
+            baseUri: '',
+            excludeFields: [],
+            model: dataset,
+        };
+
+        this.strategy = new Strategy({
+            excludeEmpty: true,
+            baseUri: '',
+            excludeFields: [],
+            model: dataset,
+        });
+    }
+
+    set model(model: Dataset) {
+        this.config.model = model;
+    }
+
+    public excludeEmptyFields(): void {
+        this.config.excludeEmpty = true;
+    }
+
+    get xmlCache(): DocumentXmlCache | null {
+        return this.cache;
+    }
+
+    set xmlCache(cache: DocumentXmlCache) {
+        this.cache = cache;
+    }
+
+    get caching(): boolean {
+        return this._caching;
+    }
+    set caching(caching: boolean) {
+        this._caching = caching;
+    }
+
+    public async getDomDocument(): Promise<XMLBuilder | null> {
+        const dataset = this.config.model;
+
+        let domDocument: XMLBuilder | null = await this.getDomDocumentFromXmlCache();
+        if (domDocument == null) {
+            domDocument = await this.strategy.createDomDocument();
+            //     domDocument = create({ version: '1.0', encoding: 'UTF-8', standalone: true }, '<root></root>');
+            if (this._caching) {
+                // caching is desired:
+                this.cache = this.cache || new DocumentXmlCache();
+                this.cache.document_id = dataset.id;
+                this.cache.xml_version = 1; // (int)$this->strategy->getVersion();
+                // this.cache.server_date_modified = dataset.server_date_modified.toFormat("yyyy-MM-dd HH:mm:ss");
+                this.cache.xml_data = domDocument.end();
+                await this.cache.save();
+            }
+        }
+        return domDocument;
+    }
+
+    private async getDomDocumentFromXmlCache(): Promise<XMLBuilder | null> {
+        const dataset: Dataset = this.config.model;
+        if (!this.cache) {
+            return null;
+        }
+        //.toFormat('YYYY-MM-DD HH:mm:ss');
+        let date: DateTime = dataset.server_date_modified;
+        const actuallyCached: boolean = await DocumentXmlCache.hasValidEntry(dataset.id, date);
+        if (!actuallyCached) {
+            return null;
+        }
+        //cache is actual return it for oai:
+        try {
+            if (this.cache) {
+                return this.cache.getDomDocument();
+            } else {
+                return null;
+            }
+        } catch (error) {
+            return null;
+        }
+    }
+}

app/Models/BaseModel.ts

@@ -13,6 +13,7 @@ import { BaseModel as LucidBaseModel } from '@ioc:Adonis/Lucid/Orm';
 //   }
 // }
 
+
 /**
  * Helper to find if value is a valid Object or
  * not
@@ -21,7 +22,7 @@ export function isObject(value: any): boolean {
     return value !== null && typeof value === 'object' && !Array.isArray(value);
 }
 
-export default class BaseModel extends LucidBaseModel {
+export default class BaseModel extends LucidBaseModel {  
     /**
      * When `fill` method is called, then we may have a situation where it
      * removed the values which exists in `original` and hence the dirty
@@ -116,4 +117,12 @@ export default class BaseModel extends LucidBaseModel {
 
         return this;
     }
+
 }
+
+// export class DatasetRelatedBaseModel extends LucidBaseModel {
+//     public dataset: BelongsTo<typeof Dataset>;
+// }
+// export interface DatasetRelatedBaseModel {
+//     dataset: BelongsTo<typeof Dataset>;
+// }

app/Models/Collection.ts

@@ -1,5 +1,6 @@
-import { column, BaseModel, SnakeCaseNamingStrategy, manyToMany, ManyToMany } from '@ioc:Adonis/Lucid/Orm';
+import { column, SnakeCaseNamingStrategy, manyToMany, ManyToMany } from '@ioc:Adonis/Lucid/Orm';
 import Dataset from './Dataset';
+import BaseModel from './BaseModel';
 
 export default class Collection extends BaseModel {
     public static namingStrategy = new SnakeCaseNamingStrategy();

app/Models/Coverage.ts

@@ -1,6 +1,7 @@
-import { column, BaseModel, SnakeCaseNamingStrategy, belongsTo, BelongsTo } from '@ioc:Adonis/Lucid/Orm';
+import { column, SnakeCaseNamingStrategy, belongsTo, BelongsTo } from '@ioc:Adonis/Lucid/Orm';
 import { DateTime } from 'luxon';
 import Dataset from './Dataset';
+import BaseModel from './BaseModel';
 
 export default class Coverage extends BaseModel {
     public static namingStrategy = new SnakeCaseNamingStrategy();

app/Models/Dataset.ts

@@ -1,6 +1,5 @@
 import {
     column,
-    BaseModel,
     SnakeCaseNamingStrategy,
     manyToMany,
     ManyToMany,
@@ -26,8 +25,10 @@ import DatasetReference from './DatasetReference';
 import Collection from './Collection';
 import DatasetIdentifier from './DatasetIdentifier';
 import Project from './Project';
+import DocumentXmlCache from './DocumentXmlCache';
+import DatasetExtension from 'App/Models/Traits/DatasetExtension'; // Adjust the import path
 
-export default class Dataset extends BaseModel {
+export default class Dataset extends DatasetExtension {
     public static namingStrategy = new SnakeCaseNamingStrategy();
     public static primaryKey = 'id';
     public static table = 'documents';
@@ -40,13 +41,13 @@ export default class Dataset extends BaseModel {
     public server_state: string;
 
     @column({})
-    public publisherName: string;
+    public publisher_name: string;
 
     @column({ columnName: 'creating_corporation' })
-    public creatingCorporation: string;
+    public creating_corporation: string;
 
     @column.dateTime({ columnName: 'embargo_date' })
-    public embargoDate: DateTime;
+    public embargo_date: DateTime;
 
     @column({})
     public type: string;
@@ -82,7 +83,7 @@ export default class Dataset extends BaseModel {
     public reject_reviewer_note: string | null;
 
     @column.dateTime({ columnName: 'server_date_published' })
-    public serverDatePublished: DateTime;
+    public server_date_published: DateTime;
 
     // @column.dateTime({ autoCreate: true, columnName: 'created_at' })
     @column.dateTime({
@@ -92,10 +93,10 @@ export default class Dataset extends BaseModel {
         autoCreate: true,
         columnName: 'created_at',
     })
-    public createdAt: DateTime;
+    public created_at: DateTime;
 
     @column.dateTime({ autoCreate: true, autoUpdate: true, columnName: 'server_date_modified' })
-    public updatedAt: DateTime;
+    public server_date_modified: DateTime;
 
     @manyToMany(() => Person, {
         pivotForeignKey: 'document_id',
@@ -208,4 +209,9 @@ export default class Dataset extends BaseModel {
         },
     })
     public contributors: ManyToMany<typeof Person>;
+
+    @hasOne(() => DocumentXmlCache, {
+        foreignKey: 'document_id',
+    })
+    public xmlCache: HasOne<typeof DocumentXmlCache>;
 }

app/Models/DatasetIdentifier.ts

@@ -1,6 +1,7 @@
-import { column, BaseModel, SnakeCaseNamingStrategy, belongsTo, BelongsTo } from '@ioc:Adonis/Lucid/Orm';
+import { column, SnakeCaseNamingStrategy, belongsTo, BelongsTo } from '@ioc:Adonis/Lucid/Orm';
 import { DateTime } from 'luxon';
 import Dataset from './Dataset';
+import BaseModel from './BaseModel';
 
 export default class DatasetIdentifier extends BaseModel {
     public static namingStrategy = new SnakeCaseNamingStrategy();

app/Models/DatasetReference.ts

@@ -1,6 +1,7 @@
-import { column, BaseModel, SnakeCaseNamingStrategy, belongsTo, BelongsTo } from '@ioc:Adonis/Lucid/Orm';
+import { column, SnakeCaseNamingStrategy, belongsTo, BelongsTo } from '@ioc:Adonis/Lucid/Orm';
 import { DateTime } from 'luxon';
 import Dataset from './Dataset';
+import BaseModel from './BaseModel';
 
 export default class DatasetReference extends BaseModel {
     public static namingStrategy = new SnakeCaseNamingStrategy();

app/Models/DocumentXmlCache.ts

@@ -0,0 +1,98 @@
+import { column, BaseModel, SnakeCaseNamingStrategy, belongsTo, BelongsTo } from '@ioc:Adonis/Lucid/Orm';
+import Dataset from './Dataset';
+import { builder, create } from 'xmlbuilder2';
+import { XMLBuilder } from 'xmlbuilder2/lib/interfaces';
+import Database from '@ioc:Adonis/Lucid/Database';
+import dayjs from 'dayjs';
+import { DateTime } from 'luxon';
+
+export default class DocumentXmlCache extends BaseModel {
+    public static namingStrategy = new SnakeCaseNamingStrategy();
+    public static table = 'document_xml_cache';
+    // public static fillable: string[] = ['value', 'label', 'type', 'relation'];
+    // public static primaryKey = false;
+    static primaryKey = ''; // Set primaryKey to null to indicate there is no primary key
+
+    @column({
+        isPrimary: true,
+    })
+    public document_id: number;
+
+    @column({})
+    public xml_version: number;
+
+    @column()
+    public server_date_modified?: string;
+
+    // @column.dateTime({
+    //     autoCreate: true,
+    //     autoUpdate: true,
+    // })
+    // public updated_at?: DateTime;
+
+    @column({})
+    public xml_data: string;
+
+    @belongsTo(() => Dataset, {
+        foreignKey: 'document_id',
+    })
+    public dataset: BelongsTo<typeof Dataset>;
+
+    /**
+     * Get dom document of 'xml_data' string
+     *
+     * @returns {XMLBuilder}
+     */
+    public getDomDocument(): XMLBuilder {
+        // const dom = xmlbuilder.create({ version: "1.0", encoding: "UTF-8", standalone: true });
+        let dom: XMLBuilder = create({ version: '1.0', encoding: 'UTF-8', standalone: true }, this.xml_data);
+        // return dom.first();
+
+        const rdrDataset = dom.find(
+            (n) => {
+                const test = n.node.nodeName == 'Rdr_Dataset';
+                return test;
+            },
+            false,
+            true,
+        )?.node;
+
+        if (rdrDataset == undefined) {
+            return dom.first();
+        } else {
+            dom = builder({ version: '1.0', encoding: 'UTF-8', standalone: true }, rdrDataset);
+            return dom;
+        }
+    }
+
+    /**
+     * Check if a dataset in a specific xml version is already cached or not.
+     *
+     * @param mixed datasetId
+     * @param mixed serverDateModified
+     * @returns {Promise<boolean>} Returns true on cached hit else false.
+     */
+    // public static async hasValidEntry(datasetId: number, datasetServerDateModified: DateTime): Promise<boolean> {
+    //     // const formattedDate = dayjs(datasetServerDateModified).format('YYYY-MM-DD HH:mm:ss');
+
+    //     const query = Database.from(this.table)
+    //         .where('document_id', datasetId)
+    //         .where('server_date_modified', '2023-08-17 16:51:03')
+    //         .first();
+
+    //     const row = await query;
+    //     return !!row;
+    // }
+
+    // Assuming 'DocumentXmlCache' has a table with a 'server_date_modified' column in your database
+    public static async hasValidEntry(datasetId: number, datasetServerDateModified: DateTime): Promise<boolean> {
+        const serverDateModifiedString: string = datasetServerDateModified.toFormat('yyyy-MM-dd HH:mm:ss'); // Convert DateTime to ISO string
+        const query = Database.from(this.table)
+            .where('document_id', datasetId)
+            .where('server_date_modified', '>=', serverDateModifiedString) // Check if server_date_modified is newer or equal
+            .first();
+
+        const row = await query;
+        return !!row;
+    }
+}

app/Models/File.ts

@@ -1,7 +1,6 @@
 import { DateTime } from 'luxon';
 import {
     column,
-    BaseModel,
     hasMany,
     HasMany,
     belongsTo,
@@ -12,6 +11,7 @@ import {
 } from '@ioc:Adonis/Lucid/Orm';
 import HashValue from './HashValue';
 import Dataset from './Dataset';
+import BaseModel from './BaseModel';
 
 export default class File extends BaseModel {
     public static namingStrategy = new SnakeCaseNamingStrategy();

app/Models/Language.ts

@@ -1,4 +1,5 @@
-import { column, BaseModel, SnakeCaseNamingStrategy } from '@ioc:Adonis/Lucid/Orm';
+import { column, SnakeCaseNamingStrategy } from '@ioc:Adonis/Lucid/Orm';
+import BaseModel from './BaseModel';
 // import { DateTime } from 'luxon';
 
 export default class Language extends BaseModel {

app/Models/License.ts

@@ -1,4 +1,5 @@
-import { column, BaseModel, SnakeCaseNamingStrategy } from '@ioc:Adonis/Lucid/Orm';
+import { column, SnakeCaseNamingStrategy } from '@ioc:Adonis/Lucid/Orm';
+import BaseModel from './BaseModel';
 
 export default class License extends BaseModel {
     public static namingStrategy = new SnakeCaseNamingStrategy();

app/Models/Permission.ts

@@ -1,7 +1,8 @@
-import { column, BaseModel, manyToMany, ManyToMany, SnakeCaseNamingStrategy, beforeUpdate, beforeCreate } from '@ioc:Adonis/Lucid/Orm';
+import { column, manyToMany, ManyToMany, SnakeCaseNamingStrategy, beforeUpdate, beforeCreate } from '@ioc:Adonis/Lucid/Orm';
 import { DateTime } from 'luxon';
 import dayjs from 'dayjs';
 import Role from 'App/Models/Role';
+import BaseModel from './BaseModel';
 
 export default class Permission extends BaseModel {
     public static namingStrategy = new SnakeCaseNamingStrategy();

app/Models/Project.ts

@@ -1,5 +1,6 @@
-import { column, BaseModel, SnakeCaseNamingStrategy } from '@ioc:Adonis/Lucid/Orm';
+import { column, SnakeCaseNamingStrategy } from '@ioc:Adonis/Lucid/Orm';
 import { DateTime } from 'luxon';
+import BaseModel from './BaseModel';
 
 export default class Project extends BaseModel {
     public static namingStrategy = new SnakeCaseNamingStrategy();

app/Models/Role.ts

@@ -1,5 +1,5 @@
-import { column, BaseModel, SnakeCaseNamingStrategy, manyToMany, ManyToMany, beforeCreate, beforeUpdate } from '@ioc:Adonis/Lucid/Orm';
-
+import { column, SnakeCaseNamingStrategy, manyToMany, ManyToMany, beforeCreate, beforeUpdate } from '@ioc:Adonis/Lucid/Orm';
+import BaseModel from './BaseModel';
 import { DateTime } from 'luxon';
 // import moment from 'moment';
 import dayjs from 'dayjs';

app/Models/Subject.ts

@@ -1,4 +1,5 @@
-import { column, BaseModel, SnakeCaseNamingStrategy, manyToMany, ManyToMany, beforeCreate, beforeUpdate } from '@ioc:Adonis/Lucid/Orm';
+import { column, SnakeCaseNamingStrategy, manyToMany, ManyToMany, beforeCreate, beforeUpdate } from '@ioc:Adonis/Lucid/Orm';
+import BaseModel from './BaseModel';
 
 import { DateTime } from 'luxon';
 import dayjs from 'dayjs';

app/Models/Title.ts

@@ -1,6 +1,7 @@
 import { column, belongsTo, BelongsTo } from '@ioc:Adonis/Lucid/Orm';
 import Dataset from './Dataset';
 import BaseModel from './BaseModel';
+// import { DatasetRelatedBaseModel } from './BaseModel';
 
 export default class Title extends BaseModel {
     public static primaryKey = 'id';

app/Models/Traits/DatasetExtension.ts

@@ -0,0 +1,329 @@
+import Title from 'App/Models/Title';
+import Description from 'App/Models/Description';
+import License from 'App/Models/License';
+import Person from 'App/Models/Person';
+import DatasetReference from 'App/Models/DatasetReference';
+import DatasetIdentifier from 'App/Models/DatasetIdentifier';
+import Subject from 'App/Models/Subject';
+import File from 'App/Models/File';
+import Coverage from 'App/Models/Coverage';
+import Collection from 'App/Models/Collection';
+import { BaseModel as LucidBaseModel } from '@ioc:Adonis/Lucid/Orm';
+import Field from 'App/Library/Field';
+import { DateTime } from 'luxon';
+
+// @StaticImplements<LucidModel>()
+// class LucidDatasetModel extends BaseModel{
+//     @belongsTo(() => Dataset, {
+//         foreignKey: 'dataset_id',
+//     })
+//     dataset: BelongsTo<typeof Dataset>;
+// }
+
+export type DatasetRelatedModel =
+    | typeof Title
+    | typeof Description
+    | typeof Coverage
+    | typeof DatasetIdentifier
+    | typeof DatasetReference
+    | typeof Description
+    | typeof DatasetIdentifier
+    | typeof File;
+
+
+export default abstract class DatasetExtension extends LucidBaseModel {
+    public abstract id;
+    public externalFields: Record<string, any> = this.getExternalFields();
+    // which fields shoud#t be published
+    protected internalFields: Record<string, any> = {};
+    protected fields: Record<string, any> = {};
+
+    private getExternalFields(): Record<string, any> {
+        // External fields definition
+        return {
+            TitleMain: {
+                model: Title,
+                options: { type: ['Main'] },
+                fetch: 'eager',
+            },
+            TitleAdditional: {
+                model: Title,
+                options: { type: ['Alternative', 'Sub', 'Translated', 'Other'] },
+                fetch: 'eager',
+            },
+            TitleAbstract: {
+                model: Description,
+                options: { type: ['Abstract', 'Translated'] },
+                fetch: 'eager',
+            },
+            TitleAbstractAdditional: {
+                model: Description,
+                options: { type: ['Methods', 'Technical_info', 'Series_information', 'Other'] },
+                fetch: 'eager',
+            },
+            Licence: {
+                model: License,
+                through: 'link_documents_licences',
+                relation: 'licenses',
+                fetch: 'eager',
+            },
+            PersonAuthor: {
+                model: Person,
+                through: 'link_documents_persons',
+                pivot: { role: 'author', sort_order: 'sort_order', allow_email_contact: 'allow_email_contact' },
+                relation: 'persons',
+                fetch: 'eager',
+            },
+            PersonContributor: {
+                model: Person,
+                through: 'link_documents_persons',
+                pivot: {
+                    role: 'contributor',
+                    contributor_type: 'contributor_type',
+                    sort_order: 'sort_order',
+                    allow_email_contact: 'allow_email_contact',
+                },
+                relation: 'persons',
+                fetch: 'eager',
+            },
+            Reference: {
+                model: DatasetReference,
+                relation: 'references',
+                fetch: 'eager',
+            },
+            Identifier: {
+                model: DatasetIdentifier,
+                relation: 'identifier',
+                fetch: 'eager',
+            },
+            Subject: {
+                model: Subject,
+                through: 'link_dataset_subjects',
+                relation: 'subjects',
+                fetch: 'eager',
+            },
+            File: {
+                model: File,
+                relation: 'files',
+                fetch: 'eager',
+            },
+            Coverage: {
+                model: Coverage,
+                relation: 'coverage',
+                fetch: 'eager',
+            },
+            Collection: {
+                model: Collection,
+                through: 'link_documents_collections',
+                relation: 'collections',
+                fetch: 'eager',
+                //   'include': { 'model': CollectionRole, 'relation': 'collectionrole' }
+            },
+        };
+    }
+
+    public initFields(): void {
+        // Initialize internal fields
+        let fields = new Array<string>(
+            'Id',
+            'PublisherName',
+            'PublishId',
+            'ContributingCorporation',
+            'CreatingCorporation',
+            'Language',
+            'PublishedDate',
+            // 'PublishedYear',
+            'PublisherName',
+            // 'PublisherPlace',
+            'PublicationState',
+            'EmbargoDate',
+            'CreatedAt',
+            'ServerDateModified',
+            'ServerDatePublished',
+            'ServerDateDeleted',
+            'ServerState',
+            'Type',
+            'BelongsToBibliography',
+        );
+        fields.forEach((fieldname) => {
+            let field = new Field(fieldname);
+            this.addField(field);
+        });
+        // Initialize external fields
+        const fieldNames = Object.keys(this.externalFields);
+        for (const fieldName of fieldNames) {
+            // const field = this.externalFields[fieldName];
+            let field = new Field(fieldName);
+            field.setMultiplicity('*');
+            this.addField(field);
+        }
+
+        //      // Initialize available date fields and set up date validator
+        //     // if the particular field is present
+        let dateFields = new Array<string>('EmbargoDate', 'CreatedAt', 'ServerDatePublished', 'ServerDateDeleted');
+        dateFields.forEach((fieldname) => {
+            let dateField = this.getField(fieldname);
+            dateField instanceof Field && dateField.setValueModelClass(DateTime.now());
+        });
+    }
+
+    public async describe(): Promise<Array<string>> {
+        let length: number = Object.keys(this.fields).length;
+        if (length == 0) {
+            await this.fetchValues();
+        }
+        // Get an array of all field names in the 'fields' object
+        const allFields = Object.keys(this.fields);
+        // Get an array of all field names in the 'internalFields' array
+        const internalFields = Object.keys(this.internalFields);
+        // Use the `filter` method to find field names that are not in 'internalFields'
+        const filteredFields = allFields.filter((fieldName) => !internalFields.includes(fieldName));
+        return filteredFields;
+    }
+
+    private addField(field: Field): void {
+        // Add field
+        const fieldName = field.getName();
+        if (fieldName && this.externalFields[fieldName]) {
+            const options = this.externalFields[fieldName];
+
+            // Set ValueModelClass if a model option is given
+            if (options.model) {
+                field.setValueModelClass(options.model);
+            }
+        }
+
+        this.fields[field.getName()] = field;
+        // field.setOwningModelClass(this.constructor.name);
+    }
+
+    public getField(name: string): Field | null {
+        // Get field
+        return this.fields[name] !== undefined ? this.fields[name] : null;
+    }
+
+    public async fetchValues(): Promise<void> {
+        this.initFields();
+        await this.loadFieldValues();
+    }
+
+    private async loadFieldValues(): Promise<void> {
+        for (const [fieldname, field] of Object.entries(this.fields)) {
+            // extern fields via model relation
+            if (this.externalFields.hasOwnProperty(fieldname)) {
+                await this.loadExternal(fieldname);
+                // dataset attributes itself
+            } else {
+                // Field is not external and gets handled by simply reading. to snake_case
+                const property_name = this.convertFieldnameToColumn(fieldname); //id
+                const fieldVal = this[property_name]; //276
+
+                // Explicitly set null if the field represents a model except for dates.
+                if (field.getValueModelClass() !== null) {
+                    field.setValue(fieldVal === undefined || fieldVal === null ? null : fieldVal);
+                } else {
+                    field.setValue(fieldVal);
+                }
+            }
+        }
+    }
+
+    private async loadExternal(fieldname: string): Promise<void> {
+        const field = this.fields[fieldname];
+
+        // let modelclass: typeof Title | typeof Description;
+        let modelclass: DatasetRelatedModel = field.getValueModelClass();
+        let modelInstance = new modelclass();
+
+        // Create a query builder
+        const select = modelclass.query();
+
+        // If any declared constraints, add them to the query
+        if (this.externalFields[fieldname]?.options) {
+            const options: Array<string> = this.externalFields[fieldname].options;
+            for (const [column, value] of Object.entries(options)) {
+                if (Array.isArray(value)) {
+                    select.whereIn(column, value);
+                } else {
+                    select.where(column, value);
+                }
+            }
+        }
+
+        // Get dependent rows
+        const result: Record<string, any>[] = [];
+        const datasetId = this.id;
+        let rows: any[] = [];
+
+        if (this.externalFields[fieldname]?.through) {
+            const relation = this.externalFields[fieldname].relation;
+            // rows = this[relation];
+            rows = await this.related(relation).query();
+
+            if (this.externalFields[fieldname].pivot) {
+                const pivotArray = this.externalFields[fieldname].pivot;
+                const pivotValue = pivotArray.role;
+                // rows = this[relation]().wherePivot('role', pivotValue).get();
+                rows = await this.related(relation).query().wherePivot('role', pivotValue);
+            }
+        } else if (modelInstance.hasOwnProperty('dataset')) {
+            rows = await select
+                .whereHas('dataset', (q) => {
+                    q.where('id', datasetId);
+                })
+                .orderBy('id')
+                .select();
+        }
+
+        // 1 ..n relations
+        for (const row of rows) {
+            const attributes = Object.keys(row.$attributes);
+
+            if (this.externalFields[fieldname]?.pivot) {
+                const pivotArray = this.externalFields[fieldname].pivot;
+                const arrayKeys = Object.keys(pivotArray);
+                const extendedArrayKeys = arrayKeys.map((pivotAttribute) => {
+                    return `pivot_${pivotAttribute}`;
+                });
+                attributes.push(...extendedArrayKeys);
+            }
+
+            const objArray: Record<string, any> = {};
+
+            for (const property_name of attributes) {
+                let fieldName = this.convertColumnToFieldname(property_name);
+                let fieldval = '';
+
+                if (property_name.startsWith('pivot_')) {
+                    const str = property_name.replace('pivot_', '');
+                    fieldName = this.convertColumnToFieldname(str);
+                    // fieldval = row.$pivot[str];
+                    fieldval = row.$extras[property_name];
+                } else if (fieldName === 'Type') {
+                    fieldval = row[property_name]?.charAt(0).toUpperCase() + row[property_name]?.slice(1);
+                } else {
+                    fieldval = row[property_name];
+                }
+
+                objArray[fieldName] = fieldval;
+            }
+
+            result.push(objArray);
+        }
+
+        // Set the field value
+        field.setValue(result);
+    }
+
+    // to snakeCase
+    private convertFieldnameToColumn(fieldname: string): string {
+        return fieldname.replace(/([a-z\d])([A-Z])/g, '$1_$2').toLowerCase();
+    }
+
+    private convertColumnToFieldname(columnName: string): string {
+        return columnName
+            .split(/[-_]/)
+            .map((word) => (word.charAt(0).toUpperCase() + word.slice(1)))
+            .join('');
+    }
+}

app/Models/User.ts

@@ -1,10 +1,11 @@
 import { DateTime } from 'luxon';
-import { BaseModel, column, beforeSave, manyToMany, ManyToMany, hasMany, HasMany } from '@ioc:Adonis/Lucid/Orm';
+import { column, beforeSave, manyToMany, ManyToMany, hasMany, HasMany } from '@ioc:Adonis/Lucid/Orm';
 import Hash from '@ioc:Adonis/Core/Hash';
 import Role from './Role';
 import Database from '@ioc:Adonis/Lucid/Database';
 import Config from '@ioc:Adonis/Core/Config';
 import Dataset from './Dataset';
+import BaseModel from './BaseModel';
 
 // export default interface IUser {
 // 	id: number;

config/database.ts

@@ -48,6 +48,8 @@ const databaseConfig: DatabaseConfig = {
             },
             healthCheck: false,
             debug: false,
+            pool: { min: 1, max: 100 },           
+                        
         },
     },
 };

resources/js/Pages/Submitter/Dataset/Create.vue

@@ -1039,7 +1039,7 @@ Removes a selected keyword
                                 Next
                             </button>
 
-                            <button v-if="formStep == 4" :disabled="form.processing"
+                            <button v-if="formStep == 4" :disabled="form.processing" :class="{ 'opacity-25': form.processing }"
                                 class="text-base hover:scale-110 focus:outline-none flex justify-center px-4 py-2 rounded font-bold cursor-pointer hover:bg-teal-200 bg-teal-100 text-teal-700 border duration-200 ease-in-out border-teal-600 transition"
                                 @click.stop="submit">
                                 Save