Skip to content

Commit

Permalink
feat(dav): introduce paginate with custom headers
Browse files Browse the repository at this point in the history
Signed-off-by: Benjamin Gaussorgues <[email protected]>
  • Loading branch information
Altahrim committed Oct 11, 2024
1 parent d013a13 commit 6a915b2
Show file tree
Hide file tree
Showing 9 changed files with 328 additions and 1 deletion.
3 changes: 2 additions & 1 deletion apps/dav/appinfo/info.xml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
<name>WebDAV</name>
<summary>WebDAV endpoint</summary>
<description>WebDAV endpoint</description>
<version>1.32.0</version>
<version>1.33.0</version>
<licence>agpl</licence>
<author>owncloud.org</author>
<namespace>DAV</namespace>
Expand All @@ -27,6 +27,7 @@
<job>OCA\DAV\BackgroundJob\CleanupDirectLinksJob</job>
<job>OCA\DAV\BackgroundJob\UpdateCalendarResourcesRoomsBackgroundJob</job>
<job>OCA\DAV\BackgroundJob\CleanupInvitationTokenJob</job>
<job>OCA\DAV\BackgroundJob\CleanupPaginateCacheJob</job>
<job>OCA\DAV\BackgroundJob\EventReminderJob</job>
<job>OCA\DAV\BackgroundJob\CalendarRetentionJob</job>
<job>OCA\DAV\BackgroundJob\PruneOutdatedSyncTokensJob</job>
Expand Down
6 changes: 6 additions & 0 deletions apps/dav/composer/composer/autoload_classmap.php
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
'OCA\\DAV\\BackgroundJob\\CalendarRetentionJob' => $baseDir . '/../lib/BackgroundJob/CalendarRetentionJob.php',
'OCA\\DAV\\BackgroundJob\\CleanupDirectLinksJob' => $baseDir . '/../lib/BackgroundJob/CleanupDirectLinksJob.php',
'OCA\\DAV\\BackgroundJob\\CleanupInvitationTokenJob' => $baseDir . '/../lib/BackgroundJob/CleanupInvitationTokenJob.php',
'OCA\\DAV\\BackgroundJob\\CleanupPaginateCacheJob' => $baseDir . '/../lib/BackgroundJob/CleanupPaginateCacheJob.php',
'OCA\\DAV\\BackgroundJob\\DeleteOutdatedSchedulingObjects' => $baseDir . '/../lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php',
'OCA\\DAV\\BackgroundJob\\EventReminderJob' => $baseDir . '/../lib/BackgroundJob/EventReminderJob.php',
'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => $baseDir . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php',
Expand Down Expand Up @@ -327,6 +328,7 @@
'OCA\\DAV\\Migration\\Version1008Date20181105110300' => $baseDir . '/../lib/Migration/Version1008Date20181105110300.php',
'OCA\\DAV\\Migration\\Version1008Date20181105112049' => $baseDir . '/../lib/Migration/Version1008Date20181105112049.php',
'OCA\\DAV\\Migration\\Version1008Date20181114084440' => $baseDir . '/../lib/Migration/Version1008Date20181114084440.php',
'OCA\\DAV\\Migration\\Version1009Date20181108161232' => $baseDir . '/../lib/Migration/Version1009Date20181108161232.php',
'OCA\\DAV\\Migration\\Version1011Date20190725113607' => $baseDir . '/../lib/Migration/Version1011Date20190725113607.php',
'OCA\\DAV\\Migration\\Version1011Date20190806104428' => $baseDir . '/../lib/Migration/Version1011Date20190806104428.php',
'OCA\\DAV\\Migration\\Version1012Date20190808122342' => $baseDir . '/../lib/Migration/Version1012Date20190808122342.php',
Expand All @@ -340,6 +342,10 @@
'OCA\\DAV\\Migration\\Version1029Date20231004091403' => $baseDir . '/../lib/Migration/Version1029Date20231004091403.php',
'OCA\\DAV\\Migration\\Version1030Date20240205103243' => $baseDir . '/../lib/Migration/Version1030Date20240205103243.php',
'OCA\\DAV\\Migration\\Version1031Date20240610134258' => $baseDir . '/../lib/Migration/Version1031Date20240610134258.php',
'OCA\\DAV\\Migration\\Version1032Date20241011093632' => $baseDir . '/../lib/Migration/Version1032Date20241011093632.php',
'OCA\\DAV\\Paginate\\LimitedCopyIterator' => $baseDir . '/../lib/Paginate/LimitedCopyIterator.php',
'OCA\\DAV\\Paginate\\PaginateCache' => $baseDir . '/../lib/Paginate/PaginateCache.php',
'OCA\\DAV\\Paginate\\PaginatePlugin' => $baseDir . '/../lib/Paginate/PaginatePlugin.php',
'OCA\\DAV\\Profiler\\ProfilerPlugin' => $baseDir . '/../lib/Profiler/ProfilerPlugin.php',
'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningNode' => $baseDir . '/../lib/Provisioning/Apple/AppleProvisioningNode.php',
'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningPlugin' => $baseDir . '/../lib/Provisioning/Apple/AppleProvisioningPlugin.php',
Expand Down
6 changes: 6 additions & 0 deletions apps/dav/composer/composer/autoload_static.php
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ class ComposerStaticInitDAV
'OCA\\DAV\\BackgroundJob\\CalendarRetentionJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CalendarRetentionJob.php',
'OCA\\DAV\\BackgroundJob\\CleanupDirectLinksJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupDirectLinksJob.php',
'OCA\\DAV\\BackgroundJob\\CleanupInvitationTokenJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupInvitationTokenJob.php',
'OCA\\DAV\\BackgroundJob\\CleanupPaginateCacheJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupPaginateCacheJob.php',
'OCA\\DAV\\BackgroundJob\\DeleteOutdatedSchedulingObjects' => __DIR__ . '/..' . '/../lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php',
'OCA\\DAV\\BackgroundJob\\EventReminderJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/EventReminderJob.php',
'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php',
Expand Down Expand Up @@ -342,6 +343,7 @@ class ComposerStaticInitDAV
'OCA\\DAV\\Migration\\Version1008Date20181105110300' => __DIR__ . '/..' . '/../lib/Migration/Version1008Date20181105110300.php',
'OCA\\DAV\\Migration\\Version1008Date20181105112049' => __DIR__ . '/..' . '/../lib/Migration/Version1008Date20181105112049.php',
'OCA\\DAV\\Migration\\Version1008Date20181114084440' => __DIR__ . '/..' . '/../lib/Migration/Version1008Date20181114084440.php',
'OCA\\DAV\\Migration\\Version1009Date20181108161232' => __DIR__ . '/..' . '/../lib/Migration/Version1009Date20181108161232.php',
'OCA\\DAV\\Migration\\Version1011Date20190725113607' => __DIR__ . '/..' . '/../lib/Migration/Version1011Date20190725113607.php',
'OCA\\DAV\\Migration\\Version1011Date20190806104428' => __DIR__ . '/..' . '/../lib/Migration/Version1011Date20190806104428.php',
'OCA\\DAV\\Migration\\Version1012Date20190808122342' => __DIR__ . '/..' . '/../lib/Migration/Version1012Date20190808122342.php',
Expand All @@ -355,6 +357,10 @@ class ComposerStaticInitDAV
'OCA\\DAV\\Migration\\Version1029Date20231004091403' => __DIR__ . '/..' . '/../lib/Migration/Version1029Date20231004091403.php',
'OCA\\DAV\\Migration\\Version1030Date20240205103243' => __DIR__ . '/..' . '/../lib/Migration/Version1030Date20240205103243.php',
'OCA\\DAV\\Migration\\Version1031Date20240610134258' => __DIR__ . '/..' . '/../lib/Migration/Version1031Date20240610134258.php',
'OCA\\DAV\\Migration\\Version1032Date20241011093632' => __DIR__ . '/..' . '/../lib/Migration/Version1032Date20241011093632.php',
'OCA\\DAV\\Paginate\\LimitedCopyIterator' => __DIR__ . '/..' . '/../lib/Paginate/LimitedCopyIterator.php',
'OCA\\DAV\\Paginate\\PaginateCache' => __DIR__ . '/..' . '/../lib/Paginate/PaginateCache.php',
'OCA\\DAV\\Paginate\\PaginatePlugin' => __DIR__ . '/..' . '/../lib/Paginate/PaginatePlugin.php',
'OCA\\DAV\\Profiler\\ProfilerPlugin' => __DIR__ . '/..' . '/../lib/Profiler/ProfilerPlugin.php',
'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningNode' => __DIR__ . '/..' . '/../lib/Provisioning/Apple/AppleProvisioningNode.php',
'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningPlugin' => __DIR__ . '/..' . '/../lib/Provisioning/Apple/AppleProvisioningPlugin.php',
Expand Down
26 changes: 26 additions & 0 deletions apps/dav/lib/BackgroundJob/CleanupPaginateCacheJob.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
<?php

declare(strict_types=1);

/**
* SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/

namespace OCA\DAV\BackgroundJob;

use OCA\DAV\Paginate\PaginateCache;
use OCP\BackgroundJob\Job;

class CleanupPaginateCacheJob extends Job {
/** @var PaginateCache */
private $cache;

public function __construct(PaginateCache $cache) {
$this->cache = $cache;
}

public function run($argument): void {
$this->cache->cleanup();
}
}
63 changes: 63 additions & 0 deletions apps/dav/lib/Migration/Version1032Date20241011093632.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
<?php

declare(strict_types=1);

/**
* SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/

namespace OCA\DAV\Migration;

use OCP\DB\ISchemaWrapper;
use OCP\DB\Types;
use OCP\Migration\IOutput;
use OCP\Migration\SimpleMigrationStep;

class Version1032Date20241011093632 extends SimpleMigrationStep {
public function name(): string {
return 'Add dav_page_cache table';
}

public function description(): string {
return 'Add table to cache webdav multistatus responses for pagination purpose';
}

public function changeSchema(IOutput $output, \Closure $schemaClosure, array $options) {
/** @var ISchemaWrapper $schema */
$schema = $schemaClosure();

if (!$schema->hasTable('dav_page_cache')) {
$table = $schema->createTable('dav_page_cache');

$table->addColumn('id', Types::BIGINT, [
'autoincrement' => true
]);
$table->addColumn('url_hash', Types::STRING, [
'notnull' => true,
'length' => 32,
]);
$table->addColumn('token', Types::STRING, [
'notnull' => true,
'length' => 32
]);
$table->addColumn('result_index', Types::INTEGER, [
'notnull' => true
]);
$table->addColumn('result_value', Types::TEXT, [
'notnull' => false,
]);
$table->addColumn('insert_time', Types::DATETIME, [
'notnull' => true,
]);

$table->setPrimaryKey(['id'], 'dav_page_cache_id_index');
$table->addIndex(['token', 'url_hash'], 'dav_page_cache_token_url');
$table->addUniqueIndex(['token', 'url_hash', 'result_index'], 'dav_page_cache_url_index');
$table->addIndex(['result_index'], 'dav_page_cache_index');
$table->addIndex(['insert_time'], 'dav_page_cache_time');
}

return $schema;
}
}
42 changes: 42 additions & 0 deletions apps/dav/lib/Paginate/LimitedCopyIterator.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
<?php

declare(strict_types=1);

/**
* SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/

namespace OCA\DAV\Paginate;

/**
* Save a copy of the first X items into a separate iterator
*
* this allows us to pass the iterator to the cache while keeping a copy
* of the first X items
*/
class LimitedCopyIterator extends \AppendIterator {

Check failure on line 18 in apps/dav/lib/Paginate/LimitedCopyIterator.php

View workflow job for this annotation

GitHub Actions / static-code-analysis

MissingTemplateParam

apps/dav/lib/Paginate/LimitedCopyIterator.php:18:7: MissingTemplateParam: OCA\DAV\Paginate\LimitedCopyIterator has missing template params when extending AppendIterator, expecting 3 (see https://psalm.dev/182)
/** @var array */
private array $copy = [];

public function __construct(\Traversable $iterator, int $count) {
parent::__construct();

if (!$iterator instanceof \Iterator) {
$iterator = new \IteratorIterator($iterator);
}
$iterator = new \NoRewindIterator($iterator);

while ($iterator->valid() && count($this->copy) < $count) {
$this->copy[] = $iterator->current();
$iterator->next();
}

$this->append($this->getFirstItems());
$this->append($iterator);
}

public function getFirstItems(): \Iterator {
return new \ArrayIterator($this->copy);
}
}
89 changes: 89 additions & 0 deletions apps/dav/lib/Paginate/PaginateCache.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
<?php

declare(strict_types=1);

/**
* SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/

namespace OCA\DAV\Paginate;

use OCP\AppFramework\Utility\ITimeFactory;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\IDBConnection;
use OCP\Security\ISecureRandom;

class PaginateCache {
public const TTL = 3600;

public function __construct(
private IDBConnection $database,
private ISecureRandom $random,
private ITimeFactory $timeFactory,
) {
}

public function store(string $uri, \Iterator $items): array {
$token = $this->random->generate(32);
$now = $this->timeFactory->getTime();

$query = $this->database->getQueryBuilder();
$query->insert('dav_page_cache')
->values([
'url_hash' => $query->createNamedParameter(md5($uri), IQueryBuilder::PARAM_STR),
'token' => $query->createNamedParameter($token, IQueryBuilder::PARAM_STR),
'insert_time' => $query->createNamedParameter($now, IQueryBuilder::PARAM_INT),
'result_index' => $query->createParameter('index'),
'result_value' => $query->createParameter('value'),
]);

$count = 0;
foreach ($items as $item) {
$value = json_encode($item);
$query->setParameter('index', $count, IQueryBuilder::PARAM_INT);
$query->setParameter('value', $value);
$query->executeStatement();
$count++;
}

return [$token, $count];
}

/**
* @param string $url
* @param string $token
* @param int $offset
* @param int $count
* @return array|\Traversable
*/
public function get(string $url, string $token, int $offset, int $count) {
$query = $this->database->getQueryBuilder();
$query->select(['result_value'])
->from('dav_page_cache')
->where($query->expr()->eq('token', $query->createNamedParameter($token)))
->andWhere($query->expr()->eq('url_hash', $query->createNamedParameter(md5($url))))
->andWhere($query->expr()->gte('result_index', $query->createNamedParameter($offset, IQueryBuilder::PARAM_INT)))
->andWhere($query->expr()->lt('result_index', $query->createNamedParameter($offset + $count, IQueryBuilder::PARAM_INT)));

$result = $query->executeQuery();
return array_map(function (string $entry) {
return json_decode($entry, true);
}, $result->fetchAll(\PDO::FETCH_COLUMN));
}

public function cleanup(): void {
$now = $this->timeFactory->getTime();

$query = $this->database->getQueryBuilder();
$query->delete('dav_page_cache')
->where($query->expr()->lt('insert_time', $query->createNamedParameter($now - self::TTL)));
$query->executeStatement();
}

public function clear(): void {
$query = $this->database->getQueryBuilder();
$query->delete('dav_page_cache');
$query->executeStatement();
}
}
92 changes: 92 additions & 0 deletions apps/dav/lib/Paginate/PaginatePlugin.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
<?php

declare(strict_types=1);

/**
* SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/

namespace OCA\DAV\Paginate;

use Sabre\DAV\Server;
use Sabre\DAV\ServerPlugin;
use Sabre\HTTP\RequestInterface;
use Sabre\HTTP\ResponseInterface;

class PaginatePlugin extends ServerPlugin {
public const PAGINATE_HEADER = 'x-nc-paginate';
public const PAGINATE_TOTAL_HEADER = 'x-nc-paginate-total';
public const PAGINATE_TOKEN_HEADER = 'x-nc-paginate-token';
public const PAGINATE_OFFSET_HEADER = 'x-nc-paginate-offset';
public const PAGINATE_COUNT_HEADER = 'x-nc-paginate-count';

/** @var Server */
private $server;

public function __construct(
private PaginateCache $cache,
private int $pageSize = 100,
) {
}

public function initialize(Server $server): void {
$this->server = $server;
$server->on('beforeMultiStatus', [$this, 'onMultiStatus']);
$server->on('method:SEARCH', [$this, 'onMethod'], 1);
$server->on('method:PROPFIND', [$this, 'onMethod'], 1);
$server->on('method:REPORT', [$this, 'onMethod'], 1);
}

public function getFeatures(): array {
return ['nc-paginate'];
}

public function onMultiStatus(&$fileProperties): void {
$request = $this->server->httpRequest;
if (is_array($fileProperties)) {
$fileProperties = new \ArrayIterator($fileProperties);
}
if (
$request->hasHeader(self::PAGINATE_HEADER) &&
!$request->hasHeader(self::PAGINATE_TOKEN_HEADER)
) {
$url = $request->getUrl();

$pageSize = (int)$request->getHeader(self::PAGINATE_COUNT_HEADER) ?: $this->pageSize;
$copyIterator = new LimitedCopyIterator($fileProperties, $pageSize);
[$token, $count] = $this->cache->store($url, $copyIterator);

$fileProperties = $copyIterator->getFirstItems();
$this->server->httpResponse->addHeader(self::PAGINATE_HEADER, 'true');
$this->server->httpResponse->addHeader(self::PAGINATE_TOKEN_HEADER, $token);
$this->server->httpResponse->addHeader(self::PAGINATE_TOTAL_HEADER, $count);
}
}

public function onMethod(RequestInterface $request, ResponseInterface $response) {
if (
$request->hasHeader(self::PAGINATE_TOKEN_HEADER) &&
$request->hasHeader(self::PAGINATE_OFFSET_HEADER)
) {
$url = $this->server->httpRequest->getUrl();
$token = $request->getHeader(self::PAGINATE_TOKEN_HEADER);
$offset = (int)$request->getHeader(self::PAGINATE_OFFSET_HEADER);
$count = (int)$request->getHeader(self::PAGINATE_COUNT_HEADER) ?: $this->pageSize;

$items = $this->cache->get($url, $token, $offset, $count);

$response->setStatus(207);
$response->addHeader(self::PAGINATE_HEADER, 'true');
$response->setHeader('Content-Type', 'application/xml; charset=utf-8');
$response->setHeader('Vary', 'Brief,Prefer');

$prefer = $this->server->getHTTPPrefer();
$minimal = $prefer['return'] === 'minimal';

$data = $this->server->generateMultiStatus($items, $minimal);
$response->setBody($data);
return false;
}
}
}
Loading

0 comments on commit 6a915b2

Please sign in to comment.