@ -37,7 +37,7 @@ The Arsse does not currently support the entire protocol. Notably missing featur
- Processing of the `search` parameter of the `getHeadlines` operation differs in the following ways:
- Values other than `"true"` or `"false"` for the `unread`, `star`, and `pub` special keywords treat the entire token as a search term rather than as `"false"`
- Invalid dates are ignored rather than assumed to be `"1970-01-01"`
- Only a single negative date is allowed (this is a known bug rather than intentional)
- Specifying multiple non-negative dates usually returns no results as articles must match all specified dates simultaneously; The Arsse instead returns articles matching any of the specified dates
- Dates are always relative to UTC
- Full-text search is not yet employed with any database, including PostgreSQL
- Article hashes are normally SHA1; The Arsse uses SHA256 hashes
select id from arsse_folders where owner = ? and coalesce(parent,0) = ? union all select arsse_folders.id from arsse_folders join folders on arsse_folders.parent=folders.id
)
select
id,
name,
arsse_folders.parent as parent,
coalesce(children,0) as children,
coalesce(feeds,0) as feeds
FROM arsse_folders
left join (SELECT parent,count(id) as children from arsse_folders group by parent) as child_stats on child_stats.parent = arsse_folders.id
left join (SELECT folder,count(id) as feeds from arsse_subscriptions group by folder) as sub_stats on sub_stats.folder = arsse_folders.id"
from arsse_folders
left join (select parent,count(id) as children from arsse_folders group by parent) as child_stats on child_stats.parent = arsse_folders.id
left join (select folder,count(id) as feeds from arsse_subscriptions group by folder) as sub_stats on sub_stats.folder = arsse_folders.id",
$q->setCTE("folders", "SELECT id from arsse_folders where owner = ? and coalesce(parent,0) = ? union all select arsse_folders.id from arsse_folders join folders on arsse_folders.parent=folders.id", ["str", "strict int"], [$user, $parent]);
$q->setCTE("topmost(f_id,top)", "SELECT id,id from arsse_folders where owner = ? and parent is null union all select id,top from arsse_folders join topmost on parent=f_id", ["str"], [$user]);
if ($id) {
// if an ID is specified, add a suitable WHERE condition and bindings
// this condition facilitates the implementation of subscriptionPropertiesGet, which would otherwise have to duplicate the complex query; it takes precedence over a specified folder
$q->setWhere("s.id = ?", "int", $id);
} elseif ($folder && $recursive) {
// if a folder is specified and we're listing recursively, add a common table expression to list it and its children so that we select from the entire subtree
$q->setCTE("folders(folder)", "SELECT ? union all select id from arsse_folders join folders on parent = folder", "int", $folder);
// add a suitable WHERE condition
// if a folder is specified and we're listing recursively, add a suitable WHERE condition
$q->setWhere("folder in (select folder from folders)");
} elseif (!$recursive) {
// if we're not listing recursively, match against only the specified folder (even if it is null)
// prepare the output column list; the column definitions are also used for ordering
$colDefs = $this->articleColumns();
if (!$cols) {
// if no columns are specified return a count; don't borther with sorting
@ -1534,7 +1549,23 @@ class Database {
assert(strlen($outColumns) > 0, new \Exception("No input columns matched whitelist"));
// define the basic query, to which we add lots of stuff where necessary
$q = new Query(
"SELECT
"WITH RECURSIVE
folders(id,req) as (
select 0, 0 union all select f.id, f.id from arsse_folders as f where owner = ? union all select f1.id, req from arsse_folders as f1 join folders on coalesce(parent,0)=folders.id
),
folders_top(id,top) as (
select f.id, f.id from arsse_folders as f where owner = ? and parent is null union all select f.id, top from arsse_folders as f join folders_top as t on parent=t.id
),
folder_data(id,name,top,top_name) as (
select f1.id, f1.name, top, f2.name from arsse_folders as f1 join folders_top as f0 on f1.id = f0.id join arsse_folders as f2 on f2.id = top
),
labelled(article,label_id,label_name) as (
select m.article, l.id, l.name from arsse_label_members as m join arsse_labels as l on l.id = m.label where l.owner = ? and m.assigned = 1
),
tagged(subscription,tag_id,tag_name) as (
select m.subscription, t.id, t.name from arsse_tag_members as m join arsse_tags as t on t.id = m.tag where t.owner = ? and m.assigned = 1
)
select
$outColumns
from arsse_articles
join arsse_subscriptions on arsse_subscriptions.feed = arsse_articles.feed and arsse_subscriptions.owner = ?
@ -1543,195 +1574,181 @@ class Database {
left join arsse_marks on arsse_marks.subscription = arsse_subscriptions.id and arsse_marks.article = arsse_articles.id
left join arsse_enclosures on arsse_enclosures.article = arsse_articles.id
join (
SELECT article, max(id) as edition from arsse_editions group by article
select article, max(id) as edition from arsse_editions group by article
) as latest_editions on arsse_articles.id = latest_editions.article
left join (
SELECT arsse_label_members.article, max(arsse_label_members.modified) as modified, sum(arsse_label_members.assigned) as assigned from arsse_label_members join arsse_labels on arsse_labels.id = arsse_label_members.label where arsse_labels.owner = ? group by arsse_label_members.article
select arsse_label_members.article, max(arsse_label_members.modified) as modified, sum(arsse_label_members.assigned) as assigned from arsse_label_members join arsse_labels on arsse_labels.id = arsse_label_members.label where arsse_labels.owner = ? group by arsse_label_members.article
) as label_stats on label_stats.article = arsse_articles.id",
["str", "str"],
[$user, $user]
["str", "str", "str", "str", "str", "str"],
[$user, $user, $user, $user, $user, $user]
);
$q->setCTE("topmost(f_id,top)", "SELECT id,id from arsse_folders where owner = ? and parent is null union all select id,top from arsse_folders join topmost on parent=f_id", ["str"], [$user]);
$q->setCTE("folder_data(id,name,top,top_name)", "SELECT f1.id, f1.name, top, f2.name from arsse_folders as f1 join topmost on f1.id = f_id join arsse_folders as f2 on f2.id = top");
$q->setLimit($context->limit, $context->offset);
if ($context instanceof UnionContext) {
// if the context is a union context, we compute each context in turn
$q->setWhereRestrictive(false);
foreach ($context as $c) {
$q->setWhereGroup($this->articleFilter($c));
}
} else {
// if the context is not a union, first validate input to catch 404s and the like
// next compute the context, supplying the query to manipulate directly
$this->articleFilter($context, $q);
}
// return the query
return $q;
}
protected function articleFilter(Context $context, QueryFilter $q = null) {
$q = $q ?? new QueryFilter;
$colDefs = $this->articleColumns();
// handle the simple context options
$options = [
// each context array consists of a column identifier (see $colDefs above), a comparison operator, a data type, and an option to pair with for BETWEEN evaluation
// each context array consists of a column identifier (see $colDefs above), a comparison operator, and a data type; the "between" operator has special handling
'cte_body' => "SELECT m.article, l.id, l.name from arsse_label_members as m join arsse_labels as l on l.id = m.label where l.owner = ? and m.assigned = 1",
'cte_body' => "SELECT m.subscription, t.id, t.name from arsse_tag_members as m join arsse_tags as t on t.id = m.tag where t.owner = ? and m.assigned = 1",
// each context array consists of a common table expression to select from, the column to match in the main join, the column to match in the CTE, the column to select in the CTE, an operator, and a type for the match in the CTE
// add a common table expression to list the folders and their children so that we select from the entire subtree
$q->setCTE("folders_multi(folder)", "SELECT id as folder from (select id from (select 0 as id union all select id from arsse_folders where owner = ?) as f where id in ($inClause)) as folders_multi union select id from arsse_folders join folders_multi on coalesce(parent,0) = folder", ["str", $inTypes], [$user, $inValues]);
// limit subscriptions to the listed folders
$q->setWhere("coalesce(arsse_subscriptions.folder,0) in (select folder from folders_multi)");
}
if ($context->not->folder()) {
// add a common table expression to list the folder and its children so that we exclude from the entire subtree
$q->setCTE("folders_excluded(folder)", "SELECT ? union all select id from arsse_folders join folders_excluded on coalesce(parent,0) = folder", "int", $context->not->folder);
// excluded any subscriptions in the listed folders
$q->setWhereNot("coalesce(arsse_subscriptions.folder,0) in (select folder from folders_excluded)");
// add a common table expression to list the folders and their children so that we select from the entire subtree
$q->setCTE("folders_multi_excluded(folder)", "SELECT id as folder from (select id from (select 0 as id union all select id from arsse_folders where owner = ?) as f where id in ($inClause)) as folders_multi_excluded union select id from arsse_folders join folders_multi_excluded on coalesce(parent,0) = folder", ["str", $inTypes], [$user, $inValues]);
// limit subscriptions to the listed folders
$q->setWhereNot("coalesce(arsse_subscriptions.folder,0) in (select folder from folders_multi_excluded)");
}
// handle text-matching context options
$options = [
@ -1741,29 +1758,56 @@ class Database {
"annotationTerms" => ["note"],
];
foreach ($options as $m => $columns) {
if (!$context->$m()) {
continue;
} elseif (!$context->$m) {
throw new Db\ExceptionInput("tooShort", ['field' => $m, 'action' => $this->caller(), 'min' => 1]); // must have at least one array element
}
$columns = array_map(function($c) use ($colDefs) {
assert(isset($colDefs[$c]), new Exception("constantUnknown", $c));
* @param string $user The user who owns the articles to be modified
* @param array $data An associative array of properties to modify. Anything not specified will remain unchanged
* @param Context $context The query context to match articles against
* @param RootContext $context The query context to match articles against
* @param bool $updateTimestamp Whether to also update the timestamp. This should only be false if a mark is changed as a result of an automated action not taken by the user
*/
public function articleMark(string $user, array $data, Context $context = null, bool $updateTimestamp = true): int {
public function articleMark(string $user, array $data, RootContext $context = null, bool $updateTimestamp = true): int {
$data = [
'read' => $data['read'] ?? null,
'starred' => $data['starred'] ?? null,
@ -1865,10 +1909,23 @@ class Database {
// marking as read is ignored if the edition is not the latest, but the same is not true of the other two marks
$this->db->query("UPDATE arsse_marks set touched = 0 where touched <> 0");
$q->setBody("UPDATE arsse_marks set \"read\" = ?, touched = 1 where article in(select article from target_articles) and subscription in(select distinct subscription from target_articles)", "bool", $data['read']);
$q->setBody("UPDATE arsse_marks set touched = 1, $set where article in(select article from target_articles) and subscription in(select distinct subscription from target_articles)", $setTypes, $setValues);
$q->setBody("UPDATE arsse_marks set $set where article in(select article from target_articles) and subscription in(select distinct subscription from target_articles)", $setTypes, $setValues);
->modifiedRange($query['after'], $query['before']) // FIXME: This may not be the correct date field
->articleRange($query['after_entry_id'] ? $query['after_entry_id'] + 1 : null, $query['before_entry_id'] ? $query['before_entry_id'] - 1 : null) // FIXME: This might be edition
->searchTerms(strlen($query['search'] ?? "") ? preg_split("/\s+/", $query['search']) : null); // NOTE: Miniflux matches only whole words; we match simple substrings
if ($query['category_id']) {
if ($query['category_id'] === 1) {
@ -905,17 +905,20 @@ class V1 extends \JKingWeb\Arsse\REST\AbstractHandler {
$c->folder($query['category_id'] - 1);
}
}
// FIXME: specifying e.g. ?status=read&status=removed should yield all hidden articles and all read articles, but the best we can do is all read articles which are or are not hidden
$status = array_unique($query['status']);
sort($status);
if ($status === ["read", "removed"]) {
$c->unread(false);
$c1 = $c;
$c2 = clone $c;
$c = new UnionContext($c1->unread(false), $c2->hidden(true));
} elseif ($status === ["read", "unread"]) {
$c->hidden(false);
} elseif ($status === ["read"]) {
$c->hidden(false)->unread(false);
} elseif ($status === ["removed", "unread"]) {
$c->unread(true);
$c1 = $c;
$c2 = clone $c;
$c = new UnionContext($c1->unread(true), $c2->hidden(true));
} elseif ($status === ["removed"]) {
$c->hidden(true);
} elseif ($status === ["unread"]) {
@ -1025,7 +1028,7 @@ class V1 extends \JKingWeb\Arsse\REST\AbstractHandler {
$c->markedSince(Date::sub("PT24H", $this->now()))->unread(false); // FIXME: this selects any recently touched (read, starred, annotated) article which is read, not necessarily a recently read one
$c->markedRange(Date::sub("PT24H", $this->now()), null)->unread(false); // FIXME: this selects any recently touched (read, starred, annotated) article which is read, not necessarily a recently read one
break;
default:
// any actual feed
@ -1520,7 +1520,8 @@ class API extends \JKingWeb\Arsse\REST\AbstractHandler {
'Not after edition 999' => [(new Context)->subscription(5)->latestEdition(999), [19]],
'Not after edition 19' => [(new Context)->subscription(5)->latestEdition(19), [19]],
'Not before edition 999' => [(new Context)->subscription(5)->oldestEdition(999), [20]],
'Not before edition 1001' => [(new Context)->subscription(5)->oldestEdition(1001), [20]],
'Not after article 3' => [(new Context)->latestArticle(3), [1,2,3]],
'Not before article 19' => [(new Context)->oldestArticle(19), [19,20]],
'Modified by author since 2005' => [(new Context)->modifiedSince("2005-01-01T00:00:00Z"), [2,4,6,8,20]],
'Modified by author since 2010' => [(new Context)->modifiedSince("2010-01-01T00:00:00Z"), [2,4,6,8,20]],
'Not modified by author since 2005' => [(new Context)->notModifiedSince("2005-01-01T00:00:00Z"), [1,3,5,7,19]],
'Not modified by author since 2000' => [(new Context)->notModifiedSince("2000-01-01T00:00:00Z"), [1,3,5,7,19]],
'Marked or labelled since 2014' => [(new Context)->markedSince("2014-01-01T00:00:00Z"), [8,19]],
'Marked or labelled since 2010' => [(new Context)->markedSince("2010-01-01T00:00:00Z"), [2,4,6,8,19,20]],
'Not marked or labelled since 2014' => [(new Context)->notMarkedSince("2014-01-01T00:00:00Z"), [1,2,3,4,5,6,7,20]],
'Not marked or labelled since 2005' => [(new Context)->notMarkedSince("2005-01-01T00:00:00Z"), [1,3,5,7]],
'Marked or labelled between 2000 and 2015' => [(new Context)->markedSince("2000-01-01T00:00:00Z")->notMarkedSince("2015-12-31T23:59:59Z"), [1,2,3,4,5,6,7,8,20]],
'Marked or labelled in 2010' => [(new Context)->markedSince("2010-01-01T00:00:00Z")->notMarkedSince("2010-12-31T23:59:59Z"), [2,4,6,20]],
'With label ID 1' => [(new Context)->label(1), [1,19]],
'With label ID 2' => [(new Context)->label(2), [1,5,20]],
'With label ID 1 or 2' => [(new Context)->labels([1,2]), [1,5,19,20]],
@ -505,7 +507,9 @@ trait SeriesArticle {
'Folder tree 1 excluding subscription 4' => [(new Context)->not->subscription(4)->folder(1), [5,6]],
'Folder tree 1 excluding articles 7 and 8' => [(new Context)->folder(1)->not->articles([7,8]), [5,6]],
'Folder tree 1 excluding no articles' => [(new Context)->folder(1)->not->articles([]), [5,6,7,8]],
'Marked or labelled between 2000 and 2015 excluding in 2010' => [(new Context)->markedSince("2000-01-01T00:00:00Z")->notMarkedSince("2015-12-31T23:59:59")->not->markedSince("2010-01-01T00:00:00Z")->not->notMarkedSince("2010-12-31T23:59:59Z"), [1,3,5,7,8]],
'Folder tree 1 excluding no labels' => [(new Context)->folder(1)->not->labels([]), [5,6,7,8]],
'Folder tree 1 excluding no tags' => [(new Context)->folder(1)->not->tags([]), [5,6,7,8]],
'Marked or labelled between 2000 and 2015 excluding in 2010' => [(new Context)->markedRange("2000-01-01T00:00:00Z", "2015-12-31T23:59:59")->not->markedRange("2010-01-01T00:00:00Z", "2010-12-31T23:59:59Z"), [1,3,5,7,8]],
'Search with exclusion' => [(new Context)->searchTerms(["Article"])->not->searchTerms(["one", "two"]), [3]],
'Marked or labelled in 2010 or 2015' => [(new Context)->markedRanges([["2010-01-01T00:00:00Z", "2010-12-31T23:59:59Z"], ["2015-01-01T00:00:00Z", "2015-12-31T23:59:59Z"]]), [2,4,6,8,20]],
'Not marked or labelled in 2010 or 2015' => [(new Context)->not->markedRanges([["2010-01-01T00:00:00Z", "2010-12-31T23:59:59Z"], ["2015-01-01T00:00:00Z", "2015-12-31T23:59:59Z"]]), [1,3,5,7,19]],
'Marked or labelled prior to 2010 or since 2015' => [(new Context)->markedRanges([[null, "2009-12-31T23:59:59Z"], ["2015-01-01T00:00:00Z", null]]), [1,3,5,7,8,19]],
'Not marked or labelled prior to 2010 or since 2015' => [(new Context)->not->markedRanges([[null, "2009-12-31T23:59:59Z"], ["2015-01-01T00:00:00Z", null]]), [2,4,6,20]],
'Modified in 2010 or 2015' => [(new Context)->modifiedRanges([["2010-01-01T00:00:00Z", "2010-12-31T23:59:59Z"], ["2015-01-01T00:00:00Z", "2015-12-31T23:59:59Z"]]), [2,4,6,8,20]],
'Not modified in 2010 or 2015' => [(new Context)->not->modifiedRanges([["2010-01-01T00:00:00Z", "2010-12-31T23:59:59Z"], ["2015-01-01T00:00:00Z", "2015-12-31T23:59:59Z"]]), [1,3,5,7,19]],
'Modified prior to 2010 or since 2015' => [(new Context)->modifiedRanges([[null, "2009-12-31T23:59:59Z"], ["2015-01-01T00:00:00Z", null]]), [1,3,5,7,19]],
'Not modified prior to 2010 or since 2015' => [(new Context)->not->modifiedRanges([[null, "2009-12-31T23:59:59Z"], ["2015-01-01T00:00:00Z", null]]), [2,4,6,8,20]],
'Either read or hidden' => [(new UnionContext((new Context)->unread(false), (new Context)->hidden(true))), [1, 6, 19]],
];
}
@ -929,7 +944,7 @@ trait SeriesArticle {
}
public function testMarkByOldestEdition(): void {
Arsse::$db->articleMark($this->user, ['starred' => true], (new Context)->oldestEdition(19));
Arsse::$db->articleMark($this->user, ['starred' => true], (new Context)->editionRange(19, null));
class TestQuery extends \JKingWeb\Arsse\Test\AbstractTest {
public function testBasicQuery(): void {
$q = new Query("select * from table where a = ?", "int", 3);
@ -77,38 +81,35 @@ class TestQuery extends \JKingWeb\Arsse\Test\AbstractTest {
$this->assertSame([], $q->getValues());
}
public function testQueryWithCommonTableExpression(): void {
$q = (new Query("select * from table where a in (select * from cte where a = ?)", "int", 1))->setCTE("cte", "select * from other_table where a = ? and b = ?", ["str", "str"], [2, 3]);
$this->assertSame("WITH RECURSIVE cte as (select * from other_table where a = ? and b = ?) select * from table where a in (select * from cte where a = ?)", $q->getQuery());
$q = (new Query("select * from table where a in (select * from cte1 join cte2 using (a) where a = ?)", "int", 1))->setCTE("cte1", "select * from other_table where a = ? and b = ?", ["str", "str"], [2, 3])->setCTE("cte2", "select * from other_table where c between ? and ?", ["datetime", "datetime"], [4, 5]);
$this->assertSame("WITH RECURSIVE cte1 as (select * from other_table where a = ? and b = ?), cte2 as (select * from other_table where c between ? and ?) select * from table where a in (select * from cte1 join cte2 using (a) where a = ?)", $q->getQuery());
public function testQueryWithPushedCommonTableExpression(): void {
$q = (new Query("select * from table1"))->setWhere("a between ? and ?", ["datetime", "datetime"], [1, 2])
->setCTE("cte1", "select * from table2 where a = ? and b = ?", ["str", "str"], [3, 4])
->pushCTE("cte2")
->setBody("select * from table3 join cte1 using (a) join cte2 using (a) where a = ?", "int", 5);
$this->assertSame("WITH RECURSIVE cte1 as (select * from table2 where a = ? and b = ?), cte2 as (select * from table1 WHERE a between ? and ?) select * from table3 join cte1 using (a) join cte2 using (a) where a = ?", $q->getQuery());
$q = (new query("select *, ? as const from table", "datetime", 1))
$q = (new query("SELECT *, ? as const from table", "datetime", 1))
->setWhereNot("b = ?", "bool", 2)
->setGroup("col1", "col2")
->setWhere("a = ?", "str", 3)
->setLimit(4, 5)
->setOrder("col3")
->setCTE("cte", "select ? as const", "int", 6);
$this->assertSame("WITH RECURSIVE cte as (select ? as const) select *, ? as const from table WHERE a = ? AND NOT (b = ?) GROUP BY col1, col2 ORDER BY col3 LIMIT 4 OFFSET 5", $q->getQuery());
[['feed_id' => -3, 'mode' => "1day"], (clone $c)->modifiedSince(Date::sub("PT24H", self::NOW))->notModifiedSince(Date::sub("PT24H", self::NOW))], // this is a nonsense query, but it's what TT-RSS appearsto do
[['feed_id' => -3, 'mode' => "1day"], (clone $c)->modifiedRange(Date::sub("PT24H", self::NOW), Date::sub("PT24H", self::NOW))], // this is a nonsense query, but it's what TT-RSS appearsto do
@ -101,25 +101,31 @@ class TestSearch extends \JKingWeb\Arsse\Test\AbstractTest {
'Doubled boolean' => ['unread:true unread:true', (new Context)->unread(true)],
'Bare blank date' => ['@', new Context],
'Quoted blank date' => ['"@"', new Context],
'Bare ISO date' => ['@2019-03-01', (new Context)->modifiedSince("2019-03-01T00:00:00Z")->notModifiedSince("2019-03-01T23:59:59Z")],
'Quoted ISO date' => ['"@March 1st, 2019"', (new Context)->modifiedSince("2019-03-01T00:00:00Z")->notModifiedSince("2019-03-01T23:59:59Z")],
'Bare negative ISO date' => ['-@2019-03-01', (new Context)->not->modifiedSince("2019-03-01T00:00:00Z")->not->notModifiedSince("2019-03-01T23:59:59Z")],
'Quoted negative English date' => ['"-@March 1st, 2019"', (new Context)->not->modifiedSince("2019-03-01T00:00:00Z")->not->notModifiedSince("2019-03-01T23:59:59Z")],
'Bare ISO date' => ['@2019-03-01', (new Context)->modifiedRanges([["2019-03-01T00:00:00Z", "2019-03-01T23:59:59Z"]])],
'Quoted ISO date' => ['"@March 1st, 2019"', (new Context)->modifiedRanges([["2019-03-01T00:00:00Z", "2019-03-01T23:59:59Z"]])],
'Bare negative ISO date' => ['-@2019-03-01', (new Context)->not->modifiedRanges([["2019-03-01T00:00:00Z", "2019-03-01T23:59:59Z"]])],
'Quoted negative English date' => ['"-@March 1st, 2019"', (new Context)->not->modifiedRanges([["2019-03-01T00:00:00Z", "2019-03-01T23:59:59Z"]])],
'Invalid date' => ['@Bugaboo', new Context],
'Escaped quoted date 1' => ['"@""Yesterday" and today', (new Context)->searchTerms(["and", "today"])],
'Escaped quoted date 2' => ['"@\\"Yesterday" and today', (new Context)->searchTerms(["and", "today"])],
'Escaped quoted date 3' => ['"@Yesterday\\', new Context],
'Escaped quoted date 4' => ['"@Yesterday\\and today', new Context],
'Escaped quoted date 5' => ['"@Yesterday"and today', (new Context)->searchTerms(["today"])],