geolba/tethys
6
0
Fork 0
Code Issues 2 Pull Requests Packages Projects 1 Releases Wiki Activity

- add ResumptionToken for OAI-ListIdentifiers

Browse Source 
- max identifiers and reciords via config
- add constants.php
- add clear-expired command for deleting obsolete cache
This commit is contained in:
Arno Kaimbacher 2020-05-07 19:29:18 +02:00
parent 9d8625225f
commit 79b51e93e7
11 changed files with 395 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
app/Constants/constants.php Normal file
View File

@ -0,0 +1,5 @@
<?php
// Define path to application directory
defined('APPLICATION_PATH')
|| define('APPLICATION_PATH', realpath(dirname(dirname(__FILE__))));

76
app/Http/Controllers/Oai/RequestController.php
View File

@ -4,12 +4,15 @@ namespace App\Http\Controllers\Oai;
use App\Exceptions\OaiModelException;
use App\Http\Controllers\Controller;
use App\Models\Dataset;
use App\Models\Oai\Configuration as OaiModelConfiguration;
use App\Models\Oai\OaiModelError;
use App\Models\Oai\ResumptionTokens;
use App\Models\Oai\ResumptionToken;
use Illuminate\Database\Eloquent\ModelNotFoundException;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Log;
use \Exception;
use Carbon\Carbon;
class RequestController extends Controller
{
@ -66,6 +69,8 @@ class RequestController extends Controller
// Initialize member variables.
$this->xml = new \DomDocument;
$this->proc = new \XSLTProcessor;
$this->configuration = new OaiModelConfiguration();
}
public function index(Request $request)
@ -114,6 +119,8 @@ class RequestController extends Controller
$uri = explode('?', $_SERVER['REQUEST_URI'], 2);
$this->proc->setParameter('', 'baseURL', url('/') . $uri[0]);
// $resumptionPath = $this->configuration->getResumptionTokenPath();
if (isset($oaiRequest['verb'])) {
$this->proc->setParameter('', 'oai_verb', $oaiRequest['verb']);
if ($oaiRequest['verb'] == 'Identify') {
@ -259,7 +266,8 @@ class RequestController extends Controller
*/
private function handleListRecords($oaiRequest)
{
$maxRecords = 30; //$this->_configuration->getMaxListRecords();
//$maxRecords = 30; //$this->_configuration->getMaxListRecords();
$maxRecords = $this->configuration->getMaxListRecords();
$this->handlingOfLists($oaiRequest, $maxRecords);
}
@ -271,7 +279,8 @@ class RequestController extends Controller
*/
private function handleListIdentifiers(array &$oaiRequest)
{
$maxIdentifier = 5; //$this->_configuration->getMaxListIdentifiers();
//$maxIdentifier = 5; //$this->_configuration->getMaxListIdentifiers();
$maxIdentifier = $this->configuration->getMaxListIdentifiers();
$this->handlingOfLists($oaiRequest, $maxIdentifier);
}
@ -333,7 +342,7 @@ class RequestController extends Controller
}
$repIdentifier = "tethys.at";
$tokenTempPath = storage_path('app/resumption'); //$this->_configuration->getResumptionTokenPath();
$tokenTempPath = storage_path('app' . DIRECTORY_SEPARATOR . 'resumption'); //$this->_configuration->getResumptionTokenPath();
$this->proc->setParameter('', 'repIdentifier', $repIdentifier);
$this->xml->appendChild($this->xml->createElement('Datasets'));
@ -348,12 +357,28 @@ class RequestController extends Controller
if (true === array_key_exists('metadataPrefix', $oaiRequest)) {
$metadataPrefix = $oaiRequest['metadataPrefix'];
}
$this->proc->setParameter('', 'oai_metadataPrefix', $metadataPrefix);
//$this->proc->setParameter('', 'oai_metadataPrefix', $metadataPrefix);
$tokenWorker = new ResumptionTokens();
$tokenWorker->setResumptionPath($tokenTempPath);
// parameter resumptionToken is given
if (false === empty($oaiRequest['resumptionToken'])) {
$tokenWorker = new ResumptionToken();
$resParam = $oaiRequest['resumptionToken'];
$resParam = $oaiRequest['resumptionToken']; //e.g. "158886496600000"
$token = $tokenWorker->getResumptionToken($resParam);
if (true === is_null($token)) {
throw new OaiModelException("cache is outdated.", OaiModelError::BADRESUMPTIONTOKEN);
}
$cursor = $token->getStartPosition() - 1;//startet dann bei Index 10
$start = $token->getStartPosition() + $maxRecords;
$totalIds = $token->getTotalIds();
$reldocIds = $token->getDocumentIds();
$metadataPrefix = $token->getMetadataPrefix();
$this->proc->setParameter('', 'oai_metadataPrefix', $metadataPrefix);
// else no resumptionToken is given
} else {
// no resumptionToken is given
$finder = Dataset::query();
@ -379,6 +404,45 @@ class RequestController extends Controller
$dataset = Dataset::findOrFail($dataId);
$this->createXmlRecord($dataset);
}
// no records returned
if (true === empty($workIds)) {
throw new OaiModelException(
"The combination of the given values results in an empty list.",
OaiModelError::NORECORDSMATCH
);
}
// store the further Ids in a resumption-file
$countRestIds = count($restIds);
if ($countRestIds > 0) {
$token = new ResumptionToken();
$token->setStartPosition($start);
$token->setTotalIds($totalIds);
$token->setDocumentIds($restIds);
$token->setMetadataPrefix($metadataPrefix);
$tokenWorker->storeResumptionToken($token);
// set parameters for the resumptionToken-node
$res = $token->getResumptionId();
$this->setParamResumption($res, $cursor, $totalIds);
}
}
/**
* Set parameters for resumptionToken-line.
*
* @param string $res value of the resumptionToken
* @param int $cursor value of the cursor
* @param int $totalIds value of the total Ids
*/
private function setParamResumption($res, $cursor, $totalIds)
{
$tomorrow = str_replace('+00:00', 'Z', Carbon::now()->addHour(1)->setTimeZone('UTC'));
$this->proc->setParameter('', 'dateDelete', $tomorrow);
$this->proc->setParameter('', 'res', $res);
$this->proc->setParameter('', 'cursor', $cursor);
$this->proc->setParameter('', 'totalIds', $totalIds);
}
private function createXmlRecord(Dataset $dataset)

68
app/Models/Oai/Configuration.php Normal file
View File

@ -0,0 +1,68 @@
<?php
namespace App\Models\Oai;
class Configuration
{
/**
* Hold path where to store temporary resumption token files.
*
* @var string
*/
private $pathTokens = '';
/**
* Holds maximum number of identifiers to list per request.
*
* @var int
*/
private $maxListIds = 15;
/**
* Holds maximum number of records to list per request.
*
* @var int
*/
private $maxListRecs = 15;
public function __construct()
{
$this->maxListIds = config('oai.max.listidentifiers');
$this->maxListRecs = config('oai.max.listrecords');
$this->pathTokens = config('app.workspacePath')
. DIRECTORY_SEPARATOR .'tmp'
. DIRECTORY_SEPARATOR . 'resumption';
}
/**
* Return temporary path for resumption tokens.
*
* @return string Path.
*/
public function getResumptionTokenPath()
{
return $this->pathTokens;
}
/**
* Return maximum number of listable identifiers per request.
*
* @return int Maximum number of listable identifiers per request.
*/
public function getMaxListIdentifiers()
{
return $this->maxListIds;
}
/**
* Return maximum number of listable records per request.
*
* @return int Maximum number of listable records per request.
*/
public function getMaxListRecords()
{
return $this->maxListRecs;
}
}

29
app/Models/Oai/ResumptionToken.php
View File

@ -7,13 +7,12 @@ namespace App\Models\Oai;
*/
class ResumptionToken
{
/**
* Holds dataset ids
* Holds dcoument ids
*
* @var array
*/
private $datasetIds = array();
private $documentIds = array();
/**
* Holds metadata prefix information
@ -34,7 +33,7 @@ class ResumptionToken
*
* @var integer
*/
private $startPostition = 0;
private $startPosition = 0;
/**
* Holds total amount of document ids
@ -50,7 +49,7 @@ class ResumptionToken
*/
public function getDocumentIds()
{
return $this->_documentIds;
return $this->documentIds;
}
/**
@ -60,17 +59,17 @@ class ResumptionToken
*/
public function getMetadataPrefix()
{
return $this->_metadataPrefix;
return $this->metadataPrefix;
}
/**
* Return setted resumption id after successful storing of resumption token.
*
* @return string Returns resumption id
* @return string
*/
public function getResumptionId()
{
return $this->_resumptionId;
return $this->resumptionId;
}
/**
@ -80,7 +79,7 @@ class ResumptionToken
*/
public function getStartPosition()
{
return $this->_startPosition;
return $this->startPosition;
}
/**
@ -90,7 +89,7 @@ class ResumptionToken
*/
public function getTotalIds()
{
return $this->_totalIds;
return $this->totalIds;
}
/**
@ -105,7 +104,7 @@ class ResumptionToken
$idsToStore = array($idsToStore);
}
$this->_documentIds = $idsToStore;
$this->documentIds = $idsToStore;
}
/**
@ -116,7 +115,7 @@ class ResumptionToken
*/
public function setMetadataPrefix($prefix)
{
$this->_metadataPrefix = $prefix;
$this->metadataPrefix = $prefix;
}
/**
@ -126,7 +125,7 @@ class ResumptionToken
*/
public function setResumptionId($resumptionId)
{
$this->_resumptionId = $resumptionId;
$this->resumptionId = $resumptionId;
}
/**
@ -137,7 +136,7 @@ class ResumptionToken
*/
public function setStartPosition($startPosition)
{
$this->_startPosition = (int) $startPosition;
$this->startPosition = (int) $startPosition;
}
/**
@ -147,6 +146,6 @@ class ResumptionToken
*/
public function setTotalIds($totalIds)
{
$this->_totalIds = (int) $totalIds;
$this->totalIds = (int) $totalIds;
}
}

8
app/Models/Oai/ResumptionTokenException.php Normal file
View File

@ -0,0 +1,8 @@
<?php
namespace App\Models\Oai;
use Exception;
class ResumptionTokenException extends Exception
{
}

148
app/Models/Oai/ResumptionTokens.php Normal file
View File

@ -0,0 +1,148 @@
<?php
namespace App\Models\Oai;
use App\Models\Oai\ResumptionToken;
use App\Models\Oai\ResumptionTokenException;
use Illuminate\Support\Facades\Cache;
/**
* Handling (read, write) of resumption tokens
*/
class ResumptionTokens
{
/**
* Holds resumption path without trailing slash.
*
* @var string
*/
private $resumptionPath = null;
private $resumptionId = null;
protected $filePrefix = 'rs_';
protected $fileExtension = 'txt';
/**
* Constructor of class
*
* @param $resPath (Optional) Initialise resumption path on create.
*
*/
public function __construct($resPath = null)
{
if (false === empty($resPath)) {
$this->setResumptionPath($resPath);
}
}
/**
* Generate a unique file name and resumption id for storing resumption token.
* Double action because file name 8without prefix and file extension)
* and resumption id should be equal.
*
* @return filename Generated filename including path and file extension.
*/
private function generateResumptionName()
{
$fc = 0;
// generate a unique partial name
// return value of time() should be enough
$uniqueId = time();
$fileExtension = $this->fileExtension;
if (false === empty($fileExtension)) {
$fileExtension = '.' . $fileExtension;
}
do {
$uniqueName = sprintf('%s%05d', $uniqueId, $fc++);
$file = $this->resumptionPath . DIRECTORY_SEPARATOR . $this->filePrefix . $uniqueName . $fileExtension;
} while (true === file_exists($file));
$this->resumptionId = $uniqueName;
return $uniqueName;
//return $file;
}
public function getResumptionToken($resId)
{
$token = null;
// $fileName = $this->resumptionPath . DIRECTORY_SEPARATOR . $this->filePrefix . $resId;
// if (false === empty($this->fileExtension)) {
// $fileName .= '.' . $this->fileExtension;
// }
// if (true === file_exists($fileName)) {
if (Cache::has($resId)) {
//$fileContents = file_get_contents($fileName);
$fileContents = Cache::get($resId);
// if data is not unserializueabke an E_NOTICE will be triggerd and false returned
// avoid this E_NOTICE
$token = @unserialize($fileContents);
if (false === ($token instanceof ResumptionToken)) {
$token = null;
}
}
return $token;
}
/**
* Set resumption path where the resumption token files are stored.
*
* @throws Oai_Model_ResumptionTokenException Thrown if directory operations failed.
* @return void
*/
public function setResumptionPath($resPath)
{
// expanding all symbolic links and resolving references
$realPath = realpath($resPath);
// if (empty($realPath) or false === is_dir($realPath)) {
// throw new Oai_Model_ResumptionTokenException(
// 'Given resumption path "' . $resPath . '" (real path: "' . $realPath . '") is not a directory.'
// );
// }
// if (false === is_writable($realPath)) {
// throw new Oai_Model_ResumptionTokenException(
// 'Given resumption path "' . $resPath . '" (real path: "' . $realPath . '") is not writeable.'
// );
// }
$this->resumptionPath = $realPath;
}
/**
* Store a resumption token
*
* @param Oai_Model_Resumptiontoken $token Token to store.
* @throws Oai_Model_ResumptionTokenException Thrown on file operation error.
* @return void
*/
public function storeResumptionToken(ResumptionToken $token)
{
// $fileName = $this->generateResumptionName();
$uniqueName = $this->generateResumptionName();
// $file = fopen($fileName, 'w+');
// if (false === $file) {
// throw new ResumptionTokenException('Could not open file "' . $fileName . '" for writing!');
// }
$serialToken = serialize($token);
// if (false === fwrite($file, $serialToken)) {
// throw new ResumptionTokenException('Could not write file "' . $fileName . '"!');
// }
// if (false === fclose($file)) {
// throw new ResumptionTokenException('Could not close file "' . $fileName . '"!');
// }
Cache::put($uniqueName, $serialToken, now()->addMinutes(60));
$token->setResumptionId($this->resumptionId);
}
}

4
composer.json
View File

@ -9,6 +9,7 @@
"type": "project",
"require": {
"php": "^7.1.3",
"arifhp86/laravel-clear-expired-cache-file": "^0.0.4",
"astrotomic/laravel-translatable": "^11.1",
"davejamesmiller/laravel-breadcrumbs": "5.x",
"felixkiss/uniquewith-validator": "^3.1",
@ -27,7 +28,8 @@
},
"autoload": {
"files": [
"app/Helpers/utils.php"
"app/Helpers/utils.php",
"app/Constants/constants.php"
],
"classmap": [
"database/seeds",

52
composer.lock generated
View File

@ -4,8 +4,58 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "8bb6d0c35adaea7dda9a8f822bafa019",
"content-hash": "afea7cda488160b5ceee2f305b6a3a53",
"packages": [
{
"name": "arifhp86/laravel-clear-expired-cache-file",
"version": "v0.0.4",
"source": {
"type": "git",
"url": "https://github.com/arifhp86/laravel-clear-expired-cache-file.git",
"reference": "263167d6596db92a472e18782f8b4434fcf1a09f"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/arifhp86/laravel-clear-expired-cache-file/zipball/263167d6596db92a472e18782f8b4434fcf1a09f",
"reference": "263167d6596db92a472e18782f8b4434fcf1a09f",
"shasum": ""
},
"require": {
"illuminate/support": ">=5.4.0",
"php": ">=5.4.0"
},
"type": "library",
"extra": {
"laravel": {
"providers": [
"Arifhp86\\ClearExpiredCacheFile\\Providers\\CacheClearServiceProvider"
]
}
},
"autoload": {
"psr-4": {
"Arifhp86\\ClearExpiredCacheFile\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Arifur Rahman",
"email": "arifhp86@gmail.com"
}
],
"description": "Remove laravel expired cache file/folder",
"keywords": [
"cache",
"clear",
"expired",
"laravel"
],
"time": "2020-04-18T05:02:06+00:00"
},
{
"name": "astrotomic/laravel-translatable",
"version": "v11.8.1",

2
config/app.php
View File

@ -13,6 +13,8 @@ return [
|
*/
'workspacePath' => storage_path() . DIRECTORY_SEPARATOR . "workspace",
'name' => env('APP_NAME', 'App'),
/*

22
config/oai.php Normal file
View File

@ -0,0 +1,22 @@
<?php
return [
/*
|--------------------------------------------------------------------------
| Application Name
|--------------------------------------------------------------------------
|
| This value is the name of your application. This value is used when the
| framework needs to place the application's name in a notification or
| any other location as required by the application or its packages.
|
*/
'workspacePath' => public_path() . DIRECTORY_SEPARATOR . "workspace",
'max' => [
'listidentifiers' => 15,
'listrecords' => 15
],
];

4
public/datasetxml2oai-pmh.xslt
View File

@ -193,7 +193,7 @@
<xsl:if test="count(Rdr_Dataset) > 0">
<ListIdentifiers>
<xsl:apply-templates select="Rdr_Dataset" />
<!--<xsl:if test="$totalIds > 0">
<xsl:if test="$totalIds > 0">
<resumptionToken>
<xsl:attribute name="expirationDate">
<xsl:value-of select="$dateDelete"/>
@ -206,7 +206,7 @@
</xsl:attribute>
<xsl:value-of select="$res"/>
</resumptionToken>
</xsl:if>-->
</xsl:if>
</ListIdentifiers>
</xsl:if>
</xsl:template>