From 4b5a8b06a016cf0245f0d84f9668773db0aceb86 Mon Sep 17 00:00:00 2001 From: Arno Kaimbacher Date: Tue, 22 Aug 2023 17:11:15 +0200 Subject: [PATCH] - stop eager loading hash values ( too big sql view) --- src/controllers/dataset.controller.ts | 29 ++++---- src/controllers/exampleQuery.sql | 96 +++++++++++++++++++++++++++ src/models/Dataset.ts | 3 +- 3 files changed, 115 insertions(+), 13 deletions(-) create mode 100644 src/controllers/exampleQuery.sql diff --git a/src/controllers/dataset.controller.ts b/src/controllers/dataset.controller.ts index 0b3d439..d2102b9 100644 --- a/src/controllers/dataset.controller.ts +++ b/src/controllers/dataset.controller.ts @@ -75,12 +75,12 @@ export class DatasetController { }, ], }, - // "files", - { - model: dbContext.File, - as: "files", - include: ["hashvalues"], - }, + "files", + // { + // model: dbContext.File, + // as: "files", + // include: ["hashvalues"], + // }, "identifier", ], order: [ @@ -89,8 +89,13 @@ export class DatasetController { ], // order: ['server_date_published'], // order: ['server_date_published'], + // logging: (sql, queryObject) => { + // const test = sql; + // console.log(sql); + // }, }); if (dataset) { + // dataset.files = await dataset.getFiles(); res.status(StatusCodes.OK).send(dataset); } else { res.status(StatusCodes.NOT_FOUND).send({ @@ -176,12 +181,12 @@ export class DatasetController { }, ], }, - // "files", - { - model: dbContext.File, - as: "files", - include: ["hashvalues"], - }, + "files", + // { + // model: dbContext.File, + // as: "files", + // include: ["hashvalues"], + // }, "identifier", ], order: [ diff --git a/src/controllers/exampleQuery.sql b/src/controllers/exampleQuery.sql new file mode 100644 index 0000000..ecc5452 --- /dev/null +++ b/src/controllers/exampleQuery.sql @@ -0,0 +1,96 @@ +SELECT "Dataset".*, "titles"."id" AS "titles.id", "titles"."type" AS "titles.type", "titles"."value" AS "titles.value", +"titles"."language" AS "titles.language", "titles"."document_id" AS "titles.document_id", "abstracts"."id" AS "abstracts.id", +"abstracts"."type" AS "abstracts.type", "abstracts"."value" AS "abstracts.value", "abstracts"."language" AS "abstracts.language", +"abstracts"."document_id" AS "abstracts.document_id", "user"."id" AS "user.id", "user"."login" AS "user.login", +"user"."email" AS "user.email", "user"."first_name" AS "user.first_name", "user"."last_name" AS "user.last_name", +"user"."created_at" AS "user.created_at", "user"."updated_at" AS "user.updated_at", "authors"."id" AS "authors.id", +"authors"."academic_title" AS "authors.academic_title", "authors"."email" AS "authors.email", +"authors"."first_name" AS "authors.first_name", "authors"."last_name" AS "authors.last_name", +"authors"."place_of_birth" AS "authors.place_of_birth", "authors"."identifier_orcid" AS "authors.identifier_orcid", +"authors"."identifier_gnd" AS "authors.identifier_gnd", "authors"."identifier_misc" AS "authors.identifier_misc", +"authors"."status" AS "authors.status", "authors"."name_type" AS "authors.name_type", +"authors->link_documents_persons"."person_id" AS "authors.link_documents_persons.person_id", +"authors->link_documents_persons"."document_id" AS "authors.link_documents_persons.document_id", +"authors->link_documents_persons"."role" AS "authors.link_documents_persons.role", +"authors->link_documents_persons"."sort_order" AS "authors.link_documents_persons.sort_order", +"authors->link_documents_persons"."allow_email_contact" AS "authors.link_documents_persons.allow_email_contact", +"authors->link_documents_persons"."contributor_type" AS "authors.link_documents_persons.contributor_type", +"contributors"."id" AS "contributors.id", "contributors"."academic_title" AS "contributors.academic_title", +"contributors"."email" AS "contributors.email", "contributors"."first_name" AS "contributors.first_name", +"contributors"."last_name" AS "contributors.last_name", "contributors"."place_of_birth" AS "contributors.place_of_birth", +"contributors"."identifier_orcid" AS "contributors.identifier_orcid", "contributors"."identifier_gnd" AS "contributors.identifier_gnd", +"contributors"."identifier_misc" AS "contributors.identifier_misc", "contributors"."status" AS "contributors.status", +"contributors"."name_type" AS "contributors.name_type", "contributors->link_documents_persons"."person_id" AS "contributors.link_documents_persons.person_id", +"contributors->link_documents_persons"."document_id" AS "contributors.link_documents_persons.document_id", +"contributors->link_documents_persons"."role" AS "contributors.link_documents_persons.role", +"contributors->link_documents_persons"."sort_order" AS "contributors.link_documents_persons.sort_order", +"contributors->link_documents_persons"."allow_email_contact" AS "contributors.link_documents_persons.allow_email_contact", +"contributors->link_documents_persons"."contributor_type" AS "contributors.link_documents_persons.contributor_type", +"subjects"."id" AS "subjects.id", "subjects"."type" AS "subjects.type", "subjects"."value" AS "subjects.value", +"subjects"."external_key" AS "subjects.external_key", "subjects"."language" AS "subjects.language", +"subjects"."created_at" AS "subjects.created_at", "subjects"."updated_at" AS "subjects.updated_at", +"subjects->link_dataset_subjects"."document_id" AS "subjects.link_dataset_subjects.document_id", +"subjects->link_dataset_subjects"."subject_id" AS "subjects.link_dataset_subjects.subject_id", "coverage"."id" AS "coverage.id", +"coverage"."elevation_min" AS "coverage.elevation_min", "coverage"."elevation_max" AS "coverage.elevation_max", +"coverage"."elevation_absolut" AS "coverage.elevation_absolut", "coverage"."depth_min" AS "coverage.depth_min", +"coverage"."depth_max" AS "coverage.depth_max", "coverage"."depth_absolut" AS "coverage.depth_absolut", +"coverage"."time_min" AS "coverage.time_min", "coverage"."time_max" AS "coverage.time_max", "coverage"."time_absolut" AS "coverage.time_absolut", +"coverage"."x_min" AS "coverage.x_min", "coverage"."x_max" AS "coverage.x_max", "coverage"."y_min" AS "coverage.y_min", +"coverage"."y_max" AS "coverage.y_max", "coverage"."created_at" AS "coverage.created_at", +"coverage"."updated_at" AS "coverage.updated_at", "coverage"."dataset_id" AS "coverage.dataset_id", "licenses"."id" AS "licenses.id", +"licenses"."active" AS "licenses.active", "licenses"."comment_internal" AS "licenses.comment_internal", +"licenses"."desc_markup" AS "licenses.desc_markup", "licenses"."desc_text" AS "licenses.desc_text", +"licenses"."language" AS "licenses.language", "licenses"."link_licence" AS "licenses.link_licence", "licenses"."link_logo" AS "licenses.link_logo", +"licenses"."link_sign" AS "licenses.link_sign", "licenses"."mime_type" AS "licenses.mime_type", +"licenses"."name_long" AS "licenses.name_long", "licenses"."name" AS "licenses.name", +"licenses"."pod_allowed" AS "licenses.pod_allowed", "licenses"."sort_order" AS "licenses.sort_order", +"licenses->link_documents_licences"."document_id" AS "licenses.link_documents_licences.document_id", +"licenses->link_documents_licences"."licence_id" AS "licenses.link_documents_licences.licence_id", +"references"."id" AS "references.id", "references"."document_id" AS "references.document_id", +"references"."type" AS "references.type", "references"."relation" AS "references.relation", "references"."value" AS "references.value", +"references"."label" AS "references.label", "references"."created_at" AS "references.created_at", +"references"."updated_at" AS "references.updated_at", "references"."related_document_id" AS "references.related_document_id", +"project"."id" AS "project.id", "project"."label" AS "project.label", "project"."name" AS "project.name", +"project"."created_at" AS "project.created_at", "project"."updated_at" AS "project.updated_at", +"referenced_by"."id" AS "referenced_by.id", "referenced_by"."document_id" AS "referenced_by.document_id", +"referenced_by"."type" AS "referenced_by.type", "referenced_by"."relation" AS "referenced_by.relation", +"referenced_by"."value" AS "referenced_by.value", "referenced_by"."label" AS "referenced_by.label", +"referenced_by"."created_at" AS "referenced_by.created_at", "referenced_by"."updated_at" AS "referenced_by.updated_at", +"referenced_by"."related_document_id" AS "referenced_by.related_document_id", "referenced_by->dataset"."id" AS "referenced_by.dataset.id", +"referenced_by->dataset"."contributing_corporation" AS "referenced_by.dataset.contributing_corporation", +"referenced_by->dataset"."creating_corporation" AS "referenced_by.dataset.creating_corporation", +"referenced_by->dataset"."publisher_name" AS "referenced_by.dataset.publisher_name", +"referenced_by->dataset"."embargo_date" AS "referenced_by.dataset.embargo_date", +"referenced_by->dataset"."publish_id" AS "referenced_by.dataset.publish_id", +"referenced_by->dataset"."type" AS "referenced_by.dataset.type", "referenced_by->dataset"."language" AS "referenced_by.dataset.language", "referenced_by->dataset"."server_state" AS "referenced_by.dataset.server_state", "referenced_by->dataset"."server_date_published" AS "referenced_by.dataset.server_date_published", "referenced_by->dataset"."created_at" AS "referenced_by.dataset.created_at", "referenced_by->dataset"."server_date_modified" AS "referenced_by.dataset.server_date_modified", "referenced_by->dataset"."account_id" AS "referenced_by.dataset.account_id", "referenced_by->dataset"."project_id" AS "referenced_by.dataset.project_id", +"files"."id" AS "files.id", "files"."path_name" AS "files.path_name", "files"."label" AS "files.label", +"files"."comment" AS "files.comment", "files"."mime_type" AS "files.mime_type", "files"."language" AS "files.language", +"files"."file_size" AS "files.file_size", "files"."visible_in_frontdoor" AS "files.visible_in_frontdoor", +"files"."visible_in_oai" AS "files.visible_in_oai", "files"."sort_order" AS "files.sort_order", "files"."created_at" AS +"files.created_at", "files"."updated_at" AS "files.updated_at", "files"."document_id" AS "files.document_id", +-- "files->hashvalues"."file_id" AS "files.hashvalues.file_id", "files->hashvalues"."type" AS "files.hashvalues.type", +-- "files->hashvalues"."value" AS "files.hashvalues.value", +"identifier"."id" AS "identifier.id", "identifier"."value" AS "identifier.value", "identifier"."type" AS "identifier.type", "identifier"."status" AS "identifier.status", "identifier"."created_at" AS "identifier.created_at", "identifier"."updated_at" AS "identifier.updated_at", "identifier"."dataset_id" AS "identifier.dataset_id" + +FROM (SELECT "Dataset"."id", "Dataset"."contributing_corporation", "Dataset"."creating_corporation", "Dataset"."publisher_name", +"Dataset"."embargo_date", "Dataset"."publish_id", "Dataset"."type", "Dataset"."language", "Dataset"."server_state", + "Dataset"."server_date_published", "Dataset"."created_at", "Dataset"."server_date_modified", "Dataset"."account_id", + "Dataset"."project_id" FROM "gba"."documents" AS "Dataset" WHERE "Dataset"."publish_id" = '212' LIMIT 1) AS "Dataset" + +LEFT OUTER JOIN "gba"."dataset_titles" AS "titles" ON "Dataset"."id" = "titles"."document_id" +LEFT OUTER JOIN "gba"."dataset_abstracts" AS "abstracts" ON "Dataset"."id" = "abstracts"."document_id" +LEFT OUTER JOIN "gba"."accounts" AS "user" ON "Dataset"."account_id" = "user"."id" +LEFT OUTER JOIN ( "gba"."link_documents_persons" AS "authors->link_documents_persons" INNER JOIN "gba"."persons" AS "authors" ON "authors"."id" = "authors->link_documents_persons"."person_id" AND "authors->link_documents_persons"."role" = 'author') ON "Dataset"."id" = "authors->link_documents_persons"."document_id" +LEFT OUTER JOIN ( "gba"."link_documents_persons" AS "contributors->link_documents_persons" INNER JOIN "gba"."persons" AS "contributors" ON "contributors"."id" = "contributors->link_documents_persons"."person_id" AND "contributors->link_documents_persons"."role" = 'contributor') ON "Dataset"."id" = "contributors->link_documents_persons"."document_id" +LEFT OUTER JOIN ( "gba"."link_dataset_subjects" AS "subjects->link_dataset_subjects" INNER JOIN "gba"."dataset_subjects" AS "subjects" ON "subjects"."id" = "subjects->link_dataset_subjects"."subject_id") ON "Dataset"."id" = "subjects->link_dataset_subjects"."document_id" +LEFT OUTER JOIN "gba"."coverage" AS "coverage" ON "Dataset"."id" = "coverage"."dataset_id" +LEFT OUTER JOIN ( "gba"."link_documents_licences" AS "licenses->link_documents_licences" INNER JOIN "gba"."document_licences" AS "licenses" ON "licenses"."id" = "licenses->link_documents_licences"."licence_id") ON "Dataset"."id" = "licenses->link_documents_licences"."document_id" +LEFT OUTER JOIN "gba"."document_references" AS "references" ON "Dataset"."id" = "references"."document_id" +LEFT OUTER JOIN "gba"."projects" AS "project" ON "Dataset"."project_id" = "project"."id" +LEFT OUTER JOIN "gba"."document_references" AS "referenced_by" ON "Dataset"."id" = "referenced_by"."related_document_id" +LEFT OUTER JOIN "gba"."documents" AS "referenced_by->dataset" ON "referenced_by"."document_id" = "referenced_by->dataset"."id" +LEFT OUTER JOIN "gba"."document_files" AS "files" ON "Dataset"."id" = "files"."document_id" +-- LEFT OUTER JOIN "gba"."file_hashvalues" AS "files->hashvalues" ON "files"."id" = "files->hashvalues"."file_id" +LEFT OUTER JOIN "gba"."dataset_identifiers" AS "identifier" ON "Dataset"."id" = "identifier"."dataset_id" + +ORDER BY "authors->link_documents_persons"."sort_order" ASC, "contributors->link_documents_persons"."sort_order" ASC \ No newline at end of file diff --git a/src/models/Dataset.ts b/src/models/Dataset.ts index 90b1bb5..cbf8359 100644 --- a/src/models/Dataset.ts +++ b/src/models/Dataset.ts @@ -49,7 +49,7 @@ // public readonly abstracts?: Abstract[]; // } -import { Op, Model, DataTypes, InferAttributes, InferCreationAttributes, CreationOptional, NonAttribute, Association } from "sequelize"; +import { Op, Model, DataTypes, InferAttributes, InferCreationAttributes, CreationOptional, NonAttribute, Association, HasManyGetAssociationsMixin } from "sequelize"; import sequelizeConnection from "../config/db.config"; import DocumentXmlCache from "./DocumentXmlCache"; import Collection from "./Collection"; @@ -85,6 +85,7 @@ class Dataset extends Model, InferCreationAttributes; // getters that are not attributes should be tagged using NonAttribute // to remove them from the model's Attribute Typings.