From 79b51e93e76078777aaef0857a2448560feee3dd Mon Sep 17 00:00:00 2001 From: Arno Kaimbacher Date: Thu, 7 May 2020 19:29:18 +0200 Subject: [PATCH] - add ResumptionToken for OAI-ListIdentifiers - max identifiers and reciords via config - add constants.php - add clear-expired command for deleting obsolete cache --- app/Constants/constants.php | 5 + .../Controllers/Oai/RequestController.php | 78 ++++++++- app/Models/Oai/Configuration.php | 68 ++++++++ app/Models/Oai/ResumptionToken.php | 31 ++-- app/Models/Oai/ResumptionTokenException.php | 8 + app/Models/Oai/ResumptionTokens.php | 148 ++++++++++++++++++ composer.json | 4 +- composer.lock | 52 +++++- config/app.php | 2 + config/oai.php | 22 +++ public/datasetxml2oai-pmh.xslt | 4 +- 11 files changed, 395 insertions(+), 27 deletions(-) create mode 100644 app/Constants/constants.php create mode 100644 app/Models/Oai/Configuration.php create mode 100644 app/Models/Oai/ResumptionTokenException.php create mode 100644 app/Models/Oai/ResumptionTokens.php create mode 100644 config/oai.php diff --git a/app/Constants/constants.php b/app/Constants/constants.php new file mode 100644 index 0000000..8c59384 --- /dev/null +++ b/app/Constants/constants.php @@ -0,0 +1,5 @@ +xml = new \DomDocument; $this->proc = new \XSLTProcessor; + + $this->configuration = new OaiModelConfiguration(); } public function index(Request $request) @@ -114,6 +119,8 @@ class RequestController extends Controller $uri = explode('?', $_SERVER['REQUEST_URI'], 2); $this->proc->setParameter('', 'baseURL', url('/') . $uri[0]); + // $resumptionPath = $this->configuration->getResumptionTokenPath(); + if (isset($oaiRequest['verb'])) { $this->proc->setParameter('', 'oai_verb', $oaiRequest['verb']); if ($oaiRequest['verb'] == 'Identify') { @@ -197,7 +204,7 @@ class RequestController extends Controller //or (false === in_array($dataset->getServerState(), $this->_deliveringDocumentStates)) or (false === $dataset->whereIn('server_state', $this->deliveringDocumentStates)) //or (false === $dataset->hasEmbargoPassed()) - ) { + ) { throw new OaiModelException('Document is not available for OAI export!', OaiModelError::NORECORDSMATCH); } @@ -259,7 +266,8 @@ class RequestController extends Controller */ private function handleListRecords($oaiRequest) { - $maxRecords = 30; //$this->_configuration->getMaxListRecords(); + //$maxRecords = 30; //$this->_configuration->getMaxListRecords(); + $maxRecords = $this->configuration->getMaxListRecords(); $this->handlingOfLists($oaiRequest, $maxRecords); } @@ -271,7 +279,8 @@ class RequestController extends Controller */ private function handleListIdentifiers(array &$oaiRequest) { - $maxIdentifier = 5; //$this->_configuration->getMaxListIdentifiers(); + //$maxIdentifier = 5; //$this->_configuration->getMaxListIdentifiers(); + $maxIdentifier = $this->configuration->getMaxListIdentifiers(); $this->handlingOfLists($oaiRequest, $maxIdentifier); } @@ -333,7 +342,7 @@ class RequestController extends Controller } $repIdentifier = "tethys.at"; - $tokenTempPath = storage_path('app/resumption'); //$this->_configuration->getResumptionTokenPath(); + $tokenTempPath = storage_path('app' . DIRECTORY_SEPARATOR . 'resumption'); //$this->_configuration->getResumptionTokenPath(); $this->proc->setParameter('', 'repIdentifier', $repIdentifier); $this->xml->appendChild($this->xml->createElement('Datasets')); @@ -348,12 +357,28 @@ class RequestController extends Controller if (true === array_key_exists('metadataPrefix', $oaiRequest)) { $metadataPrefix = $oaiRequest['metadataPrefix']; } - $this->proc->setParameter('', 'oai_metadataPrefix', $metadataPrefix); + //$this->proc->setParameter('', 'oai_metadataPrefix', $metadataPrefix); + + $tokenWorker = new ResumptionTokens(); + $tokenWorker->setResumptionPath($tokenTempPath); // parameter resumptionToken is given if (false === empty($oaiRequest['resumptionToken'])) { - $tokenWorker = new ResumptionToken(); - $resParam = $oaiRequest['resumptionToken']; + $resParam = $oaiRequest['resumptionToken']; //e.g. "158886496600000" + $token = $tokenWorker->getResumptionToken($resParam); + + if (true === is_null($token)) { + throw new OaiModelException("cache is outdated.", OaiModelError::BADRESUMPTIONTOKEN); + } + $cursor = $token->getStartPosition() - 1;//startet dann bei Index 10 + $start = $token->getStartPosition() + $maxRecords; + $totalIds = $token->getTotalIds(); + $reldocIds = $token->getDocumentIds(); + $metadataPrefix = $token->getMetadataPrefix(); + + $this->proc->setParameter('', 'oai_metadataPrefix', $metadataPrefix); + + // else no resumptionToken is given } else { // no resumptionToken is given $finder = Dataset::query(); @@ -379,6 +404,45 @@ class RequestController extends Controller $dataset = Dataset::findOrFail($dataId); $this->createXmlRecord($dataset); } + + // no records returned + if (true === empty($workIds)) { + throw new OaiModelException( + "The combination of the given values results in an empty list.", + OaiModelError::NORECORDSMATCH + ); + } + + // store the further Ids in a resumption-file + $countRestIds = count($restIds); + if ($countRestIds > 0) { + $token = new ResumptionToken(); + $token->setStartPosition($start); + $token->setTotalIds($totalIds); + $token->setDocumentIds($restIds); + $token->setMetadataPrefix($metadataPrefix); + + $tokenWorker->storeResumptionToken($token); + + // set parameters for the resumptionToken-node + $res = $token->getResumptionId(); + $this->setParamResumption($res, $cursor, $totalIds); + } + } + /** + * Set parameters for resumptionToken-line. + * + * @param string $res value of the resumptionToken + * @param int $cursor value of the cursor + * @param int $totalIds value of the total Ids + */ + private function setParamResumption($res, $cursor, $totalIds) + { + $tomorrow = str_replace('+00:00', 'Z', Carbon::now()->addHour(1)->setTimeZone('UTC')); + $this->proc->setParameter('', 'dateDelete', $tomorrow); + $this->proc->setParameter('', 'res', $res); + $this->proc->setParameter('', 'cursor', $cursor); + $this->proc->setParameter('', 'totalIds', $totalIds); } private function createXmlRecord(Dataset $dataset) diff --git a/app/Models/Oai/Configuration.php b/app/Models/Oai/Configuration.php new file mode 100644 index 0000000..897fee1 --- /dev/null +++ b/app/Models/Oai/Configuration.php @@ -0,0 +1,68 @@ +maxListIds = config('oai.max.listidentifiers'); + + $this->maxListRecs = config('oai.max.listrecords'); + + $this->pathTokens = config('app.workspacePath') + . DIRECTORY_SEPARATOR .'tmp' + . DIRECTORY_SEPARATOR . 'resumption'; + } + + /** + * Return temporary path for resumption tokens. + * + * @return string Path. + */ + public function getResumptionTokenPath() + { + return $this->pathTokens; + } + + /** + * Return maximum number of listable identifiers per request. + * + * @return int Maximum number of listable identifiers per request. + */ + public function getMaxListIdentifiers() + { + return $this->maxListIds; + } + + /** + * Return maximum number of listable records per request. + * + * @return int Maximum number of listable records per request. + */ + public function getMaxListRecords() + { + return $this->maxListRecs; + } +} diff --git a/app/Models/Oai/ResumptionToken.php b/app/Models/Oai/ResumptionToken.php index 4585020..bfbe2a7 100644 --- a/app/Models/Oai/ResumptionToken.php +++ b/app/Models/Oai/ResumptionToken.php @@ -7,13 +7,12 @@ namespace App\Models\Oai; */ class ResumptionToken { - - /** - * Holds dataset ids + /** + * Holds dcoument ids * * @var array */ - private $datasetIds = array(); + private $documentIds = array(); /** * Holds metadata prefix information @@ -34,7 +33,7 @@ class ResumptionToken * * @var integer */ - private $startPostition = 0; + private $startPosition = 0; /** * Holds total amount of document ids @@ -50,7 +49,7 @@ class ResumptionToken */ public function getDocumentIds() { - return $this->_documentIds; + return $this->documentIds; } /** @@ -60,17 +59,17 @@ class ResumptionToken */ public function getMetadataPrefix() { - return $this->_metadataPrefix; + return $this->metadataPrefix; } /** * Return setted resumption id after successful storing of resumption token. * - * @return string Returns resumption id + * @return string */ public function getResumptionId() { - return $this->_resumptionId; + return $this->resumptionId; } /** @@ -80,7 +79,7 @@ class ResumptionToken */ public function getStartPosition() { - return $this->_startPosition; + return $this->startPosition; } /** @@ -90,7 +89,7 @@ class ResumptionToken */ public function getTotalIds() { - return $this->_totalIds; + return $this->totalIds; } /** @@ -105,7 +104,7 @@ class ResumptionToken $idsToStore = array($idsToStore); } - $this->_documentIds = $idsToStore; + $this->documentIds = $idsToStore; } /** @@ -116,7 +115,7 @@ class ResumptionToken */ public function setMetadataPrefix($prefix) { - $this->_metadataPrefix = $prefix; + $this->metadataPrefix = $prefix; } /** @@ -126,7 +125,7 @@ class ResumptionToken */ public function setResumptionId($resumptionId) { - $this->_resumptionId = $resumptionId; + $this->resumptionId = $resumptionId; } /** @@ -137,7 +136,7 @@ class ResumptionToken */ public function setStartPosition($startPosition) { - $this->_startPosition = (int) $startPosition; + $this->startPosition = (int) $startPosition; } /** @@ -147,6 +146,6 @@ class ResumptionToken */ public function setTotalIds($totalIds) { - $this->_totalIds = (int) $totalIds; + $this->totalIds = (int) $totalIds; } } diff --git a/app/Models/Oai/ResumptionTokenException.php b/app/Models/Oai/ResumptionTokenException.php new file mode 100644 index 0000000..2934dcb --- /dev/null +++ b/app/Models/Oai/ResumptionTokenException.php @@ -0,0 +1,8 @@ +setResumptionPath($resPath); + } + } + + /** + * Generate a unique file name and resumption id for storing resumption token. + * Double action because file name 8without prefix and file extension) + * and resumption id should be equal. + * + * @return filename Generated filename including path and file extension. + */ + private function generateResumptionName() + { + $fc = 0; + + // generate a unique partial name + // return value of time() should be enough + $uniqueId = time(); + + $fileExtension = $this->fileExtension; + if (false === empty($fileExtension)) { + $fileExtension = '.' . $fileExtension; + } + + do { + $uniqueName = sprintf('%s%05d', $uniqueId, $fc++); + $file = $this->resumptionPath . DIRECTORY_SEPARATOR . $this->filePrefix . $uniqueName . $fileExtension; + } while (true === file_exists($file)); + + $this->resumptionId = $uniqueName; + return $uniqueName; + //return $file; + } + + public function getResumptionToken($resId) + { + + $token = null; + + // $fileName = $this->resumptionPath . DIRECTORY_SEPARATOR . $this->filePrefix . $resId; + // if (false === empty($this->fileExtension)) { + // $fileName .= '.' . $this->fileExtension; + // } + + // if (true === file_exists($fileName)) { + if (Cache::has($resId)) { + //$fileContents = file_get_contents($fileName); + $fileContents = Cache::get($resId); + + // if data is not unserializueabke an E_NOTICE will be triggerd and false returned + // avoid this E_NOTICE + $token = @unserialize($fileContents); + if (false === ($token instanceof ResumptionToken)) { + $token = null; + } + } + return $token; + } + + /** + * Set resumption path where the resumption token files are stored. + * + * @throws Oai_Model_ResumptionTokenException Thrown if directory operations failed. + * @return void + */ + public function setResumptionPath($resPath) + { + // expanding all symbolic links and resolving references + $realPath = realpath($resPath); + + // if (empty($realPath) or false === is_dir($realPath)) { + // throw new Oai_Model_ResumptionTokenException( + // 'Given resumption path "' . $resPath . '" (real path: "' . $realPath . '") is not a directory.' + // ); + // } + + // if (false === is_writable($realPath)) { + // throw new Oai_Model_ResumptionTokenException( + // 'Given resumption path "' . $resPath . '" (real path: "' . $realPath . '") is not writeable.' + // ); + // } + $this->resumptionPath = $realPath; + } + + /** + * Store a resumption token + * + * @param Oai_Model_Resumptiontoken $token Token to store. + * @throws Oai_Model_ResumptionTokenException Thrown on file operation error. + * @return void + */ + public function storeResumptionToken(ResumptionToken $token) + { + + // $fileName = $this->generateResumptionName(); + $uniqueName = $this->generateResumptionName(); + + + // $file = fopen($fileName, 'w+'); + // if (false === $file) { + // throw new ResumptionTokenException('Could not open file "' . $fileName . '" for writing!'); + // } + + $serialToken = serialize($token); + // if (false === fwrite($file, $serialToken)) { + // throw new ResumptionTokenException('Could not write file "' . $fileName . '"!'); + // } + // if (false === fclose($file)) { + // throw new ResumptionTokenException('Could not close file "' . $fileName . '"!'); + // } + Cache::put($uniqueName, $serialToken, now()->addMinutes(60)); + + $token->setResumptionId($this->resumptionId); + } +} diff --git a/composer.json b/composer.json index 8485b6a..403125e 100755 --- a/composer.json +++ b/composer.json @@ -9,6 +9,7 @@ "type": "project", "require": { "php": "^7.1.3", + "arifhp86/laravel-clear-expired-cache-file": "^0.0.4", "astrotomic/laravel-translatable": "^11.1", "davejamesmiller/laravel-breadcrumbs": "5.x", "felixkiss/uniquewith-validator": "^3.1", @@ -27,7 +28,8 @@ }, "autoload": { "files": [ - "app/Helpers/utils.php" + "app/Helpers/utils.php", + "app/Constants/constants.php" ], "classmap": [ "database/seeds", diff --git a/composer.lock b/composer.lock index afc0293..642eaf5 100755 --- a/composer.lock +++ b/composer.lock @@ -4,8 +4,58 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "8bb6d0c35adaea7dda9a8f822bafa019", + "content-hash": "afea7cda488160b5ceee2f305b6a3a53", "packages": [ + { + "name": "arifhp86/laravel-clear-expired-cache-file", + "version": "v0.0.4", + "source": { + "type": "git", + "url": "https://github.com/arifhp86/laravel-clear-expired-cache-file.git", + "reference": "263167d6596db92a472e18782f8b4434fcf1a09f" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/arifhp86/laravel-clear-expired-cache-file/zipball/263167d6596db92a472e18782f8b4434fcf1a09f", + "reference": "263167d6596db92a472e18782f8b4434fcf1a09f", + "shasum": "" + }, + "require": { + "illuminate/support": ">=5.4.0", + "php": ">=5.4.0" + }, + "type": "library", + "extra": { + "laravel": { + "providers": [ + "Arifhp86\\ClearExpiredCacheFile\\Providers\\CacheClearServiceProvider" + ] + } + }, + "autoload": { + "psr-4": { + "Arifhp86\\ClearExpiredCacheFile\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Arifur Rahman", + "email": "arifhp86@gmail.com" + } + ], + "description": "Remove laravel expired cache file/folder", + "keywords": [ + "cache", + "clear", + "expired", + "laravel" + ], + "time": "2020-04-18T05:02:06+00:00" + }, { "name": "astrotomic/laravel-translatable", "version": "v11.8.1", diff --git a/config/app.php b/config/app.php index 27aaed8..b1d84d3 100755 --- a/config/app.php +++ b/config/app.php @@ -13,6 +13,8 @@ return [ | */ + 'workspacePath' => storage_path() . DIRECTORY_SEPARATOR . "workspace", + 'name' => env('APP_NAME', 'App'), /* diff --git a/config/oai.php b/config/oai.php new file mode 100644 index 0000000..ee1ec9a --- /dev/null +++ b/config/oai.php @@ -0,0 +1,22 @@ + public_path() . DIRECTORY_SEPARATOR . "workspace", + + 'max' => [ + 'listidentifiers' => 15, + 'listrecords' => 15 + ], +]; diff --git a/public/datasetxml2oai-pmh.xslt b/public/datasetxml2oai-pmh.xslt index 6dad683..216c854 100644 --- a/public/datasetxml2oai-pmh.xslt +++ b/public/datasetxml2oai-pmh.xslt @@ -193,7 +193,7 @@ - +