This involved changes to the driver interface as well as the database
schemata. The most significantly altered queries were for article
selection and marking, which relied upon unusual features of SQLite.
Overall query efficiency should not be adversely affected (it may have
even imprved) in the common case, while very rare cases (not presently
triggered by any REST handlers) require more queries.
One notable benefit of these changes is that functions which query
articles can now have complete control over which columns are returned.
This has not, however, been implemented yet: symbolic column groups are
still used for now.
Note that PostgreSQL still fails many tests, but the test suite runs to
completion. Note also that one line of the Database class is not
covered; later changes will eventually make it easier to cover the line
in question.
$columns = "count(distinct arsse_articles.id) as count";
} else {
$columns = [];
foreach ($cols as $col) {
$col = trim(strtolower($col));
if (!isset($colDefs[$col])) {
continue;
}
$columns[] = $colDefs[$col]." as ".$col;
}
}
$columns = implode(",", $columns);
}
// define the basic query, to which we add lots of stuff where necessary
$q = new Query(
$q = new Query(
"SELECT
"SELECT
$extraColumns
$columns
arsse_articles.id as id,
from arsse_articles
arsse_articles.feed as feed,
join arsse_subscriptions on arsse_subscriptions.feed = arsse_articles.feed and arsse_subscriptions.owner = ?
arsse_articles.modified as modified_date,
join arsse_feeds on arsse_subscriptions.feed = arsse_feeds.id
(
left join arsse_marks on arsse_marks.subscription = arsse_subscriptions.id and arsse_marks.article = arsse_articles.id
select
left join arsse_enclosures on arsse_enclosures.article = arsse_articles.id
arsse_articles.modified as term
left join arsse_label_members on arsse_label_members.subscription = arsse_subscriptions.id and arsse_label_members.article = arsse_articles.id and arsse_label_members.assigned = 1
union select
left join arsse_labels on arsse_labels.owner = arsse_subscriptions.owner and arsse_label_members.label = arsse_labels.id",
coalesce((select modified from arsse_marks where article = arsse_articles.id and subscription in (select sub from subscribed_feeds)),'0001-01-01 00:00:00') as term
["str"], [$user]
union select
coalesce((select modified from arsse_label_members where article = arsse_articles.id and subscription in (select sub from subscribed_feeds)),'0001-01-01 00:00:00') as term
order by term desc limit 1
) as marked_date,
NOT (select count(*) from arsse_marks where article = arsse_articles.id and read = 1 and subscription in (select sub from subscribed_feeds)) as unread,
(select count(*) from arsse_marks where article = arsse_articles.id and starred = 1 and subscription in (select sub from subscribed_feeds)) as starred,
(select max(id) from arsse_editions where article = arsse_articles.id) as edition,
subscribed_feeds.sub as subscription
FROM arsse_articles"
);
);
$q->setCTE("latest_editions(article,edition)", "SELECT article,max(id) from arsse_editions group by article", [], [], "join latest_editions on arsse_articles.id = latest_editions.article");
if ($cols) {
// if there are no output columns requested we're getting a count and should not group, but otherwise we should
$q->setWhere("arsse_articles.id in ($inParams)", $inTypes, $context->articles);
"requested_articles(id,edition)",
"SELECT id,(select max(id) from arsse_editions where article = arsse_articles.id) as edition from arsse_articles where arsse_articles.id in ($inParams)",
$inTypes,
$context->articles
);
$q->setWhere("arsse_articles.id in (select id from requested_articles)");
} else {
// if neither list is specified, mock an empty table
$q->setCTE("requested_articles(id,edition)", "SELECT 'empty','table' where 1 = 0");
}
}
// filter based on label by ID or name
// filter based on label by ID or name
if ($context->labelled()) {
if ($context->labelled()) {
// any label (true) or no label (false)
// any label (true) or no label (false)
$q->setWhere((!$context->labelled ? "not " : "")."exists(select article from arsse_label_members where assigned = 1 and article = arsse_articles.id and subscription in (select sub from subscribed_feeds))");
$q->setWhere((!$context->annotated ? "not " : "")."exists(select modified from arsse_marks where article = arsse_articles.id and note <> '' and subscription in (select sub from subscribed_feeds))");
protected function articleChunk(Context $context): array {
protected function contextChunk(Context $context): array {
$exception = "";
$exception = "";
if ($context->editions()) {
if ($context->editions()) {
// editions take precedence over articles
// editions take precedence over articles
@ -983,7 +1000,7 @@ class Database {
}
}
$context = $context ?? new Context;
$context = $context ?? new Context;
// if the context has more articles or editions than we can process in one query, perform a series of queries and return an aggregate result
// if the context has more articles or editions than we can process in one query, perform a series of queries and return an aggregate result
if ($contexts = $this->articleChunk($context)) {
if ($contexts = $this->contextChunk($context)) {
$out = [];
$out = [];
$tr = $this->begin();
$tr = $this->begin();
foreach ($contexts as $context) {
foreach ($contexts as $context) {
@ -997,32 +1014,39 @@ class Database {
// NOTE: the cases all cascade into each other: a given verbosity level is always a superset of the previous one
// NOTE: the cases all cascade into each other: a given verbosity level is always a superset of the previous one
case self::LIST_FULL: // everything
case self::LIST_FULL: // everything
$columns = array_merge($columns, [
$columns = array_merge($columns, [
"(select note from arsse_marks where article = arsse_articles.id and subscription in (select sub from subscribed_feeds)) as note",
"note",
]);
]);
// no break
// no break
case self::LIST_TYPICAL: // conservative, plus content
case self::LIST_TYPICAL: // conservative, plus content
$columns = array_merge($columns, [
$columns = array_merge($columns, [
"content",
"content",
"arsse_enclosures.url as media_url", // enclosures are potentially large due to data: URLs
"media_url", // enclosures are potentially large due to data: URLs
"arsse_enclosures.type as media_type", // FIXME: enclosures should eventually have their own fetch method
"media_type", // FIXME: enclosures should eventually have their own fetch method
]);
]);
// no break
// no break
case self::LIST_CONSERVATIVE: // base metadata, plus anything that is not likely to be large text
case self::LIST_CONSERVATIVE: // base metadata, plus anything that is not likely to be large text
$columns = array_merge($columns, [
$columns = array_merge($columns, [
"arsse_articles.url as url",
"url",
"arsse_articles.title as title",
"title",
"(select coalesce(arsse_subscriptions.title,arsse_feeds.title) from arsse_feeds join arsse_subscriptions on arsse_subscriptions.feed = arsse_feeds.id where arsse_feeds.id = arsse_articles.feed) as subscription_title",
"subscription_title",
"author",
"author",
"guid",
"guid",
"published as published_date",
"published_date",
"edited as edited_date",
"edited_date",
"url_title_hash||':'||url_content_hash||':'||title_content_hash as fingerprint",
"fingerprint",
]);
]);
// no break
// no break
case self::LIST_MINIMAL: // base metadata (always included: required for context matching)
case self::LIST_MINIMAL: // base metadata (always included: required for context matching)
// the two queries we want to execute to make the requested changes
$queries = [
"UPDATE arsse_marks
set
read = case when (select honour_read from target_articles where target_articles.id = article) = 1 then (select read from target_values) else read end,
starred = coalesce((select starred from target_values),starred),
note = coalesce((select note from target_values),note),
modified = CURRENT_TIMESTAMP
WHERE
subscription in (select sub from subscribed_feeds)
and article in (select id from target_articles where to_insert = 0 and (honour_read = 1 or honour_star = 1 or (select note from target_values) is not null))",
"INSERT INTO arsse_marks(subscription,article,read,starred,note)
select
(select id from arsse_subscriptions join userdata on userid = owner where arsse_subscriptions.feed = target_articles.feed),
id,
coalesce((select read from target_values) * honour_read,0),
coalesce((select starred from target_values),0),
coalesce((select note from target_values),'')
from target_articles where to_insert = 1 and (honour_read = 1 or honour_star = 1 or coalesce((select note from target_values),'') <> '')"
];
$out = 0;
// wrap this UPDATE and INSERT together into a transaction
$tr = $this->begin();
$tr = $this->begin();
// if an edition context is specified, make sure it's valid
$out = 0;
if ($data['read'] || $data['starred'] || strlen($data['note'] ?? "")) {
// first prepare a query to insert any missing marks rows for the articles we want to mark
// but only insert new mark records if we're setting at least one "positive" mark
$q->setBody("UPDATE arsse_marks set read = ?, touched = 1 where article in(select article from target_articles) and subscription in(select distinct subscription from target_articles)", "bool", $data['read']);
$q->setBody("UPDATE arsse_marks set touched = 1, $set where article in(select article from target_articles) and subscription in(select distinct subscription from target_articles)", $setTypes, $setValues);
// first build the query which will select the target articles; we will later turn this into a CTE for the actual query that manipulates the articles
$q = $this->articleQuery($user, $context, [
"(not exists(select article from arsse_marks where article = arsse_articles.id and subscription in (select sub from subscribed_feeds))) as to_insert",
"((select read from target_values) is not null and (select read from target_values) <> (coalesce((select read from arsse_marks where article = arsse_articles.id and subscription in (select sub from subscribed_feeds)),0)) and (not exists(select * from requested_articles) or (select max(id) from arsse_editions where article = arsse_articles.id) in (select edition from requested_articles))) as honour_read",
"((select starred from target_values) is not null and (select starred from target_values) <> (coalesce((select starred from arsse_marks where article = arsse_articles.id and subscription in (select sub from subscribed_feeds)),0))) as honour_star",
$q->setBody("UPDATE arsse_marks set $set, modified = CURRENT_TIMESTAMP where article in(select article from target_articles) and subscription in(select distinct subscription from target_articles)", $setTypes, $setValues);
public function editionLatest(string $user, Context $context = null): int {
public function editionLatest(string $user, Context $context = null): int {
@ -1273,19 +1308,35 @@ class Database {
throw new User\ExceptionAuthz("notAuthorized", ["action" => __FUNCTION__, "user" => $user]);
throw new User\ExceptionAuthz("notAuthorized", ["action" => __FUNCTION__, "user" => $user]);
}
}
$context = $context ?? new Context;
$context = $context ?? new Context;
$q = new Query("SELECT max(arsse_editions.id) from arsse_editions left join arsse_articles on article = arsse_articles.id left join arsse_feeds on arsse_articles.feed = arsse_feeds.id");
$q = new Query("SELECT max(arsse_editions.id) from arsse_editions left join arsse_articles on article = arsse_articles.id join arsse_subscriptions on arsse_articles.feed = arsse_subscriptions.feed and arsse_subscriptions.owner = ?", "str", $user);
if ($context->subscription()) {
if ($context->subscription()) {
// if a subscription is specified, make sure it exists
// if a subscription is specified, make sure it exists
public function labelAdd(string $user, array $data): int {
public function labelAdd(string $user, array $data): int {
// if the user isn't authorized to perform this action then throw an exception.
// if the user isn't authorized to perform this action then throw an exception.
if (!Arsse::$user->authorize($user, __FUNCTION__)) {
if (!Arsse::$user->authorize($user, __FUNCTION__)) {
@ -1304,14 +1355,16 @@ class Database {
throw new User\ExceptionAuthz("notAuthorized", ["action" => __FUNCTION__, "user" => $user]);
throw new User\ExceptionAuthz("notAuthorized", ["action" => __FUNCTION__, "user" => $user]);
}
}
return $this->db->prepare(
return $this->db->prepare(
"SELECT
"SELECT * FROM (
SELECT
id,name,
id,name,
(select count(*) from arsse_label_members where label = id and assigned = 1) as articles,
(select count(*) from arsse_label_members where label = id and assigned = 1) as articles,
(select count(*) from arsse_label_members
(select count(*) from arsse_label_members
join arsse_marks on arsse_label_members.article = arsse_marks.article and arsse_label_members.subscription = arsse_marks.subscription
join arsse_marks on arsse_label_members.article = arsse_marks.article and arsse_label_members.subscription = arsse_marks.subscription
where label = id and assigned = 1 and read = 1
where label = id and assigned = 1 and read = 1
) as read
) as read
FROM arsse_labels where owner = ? and articles >= ? order by name
FROM arsse_labels where owner = ?) as label_data
where articles >= ? order by name
",
",
"str",
"str",
"int"
"int"
@ -1418,14 +1471,14 @@ class Database {
$q->setWhere("exists(select article from arsse_label_members where label = ? and article = arsse_articles.id)", "int", $id);
$q->setWhere("exists(select article from arsse_label_members where label = ? and article = arsse_articles.id)", "int", $id);
$q->pushCTE("target_articles");
$q->pushCTE("target_articles");
$q->setBody(
$q->setBody(
"UPDATE arsse_label_members set assigned = ?, modified = CURRENT_TIMESTAMP where label = ? and assigned = not ? and article in (select id from target_articles)",
"UPDATE arsse_label_members set assigned = ?, modified = CURRENT_TIMESTAMP where label = ? and assigned <> ? and article in (select id from target_articles)",
-- allow marks to initially have a null date due to changes in how marks are first created
-- and also add a "touched" column to aid in tracking changes during the course of some transactions
altertablearsse_marksrenametoarsse_marks_old;
createtablearsse_marks(
-- users' actions on newsfeed entries
articleintegernotnullreferencesarsse_articles(id)ondeletecascade,-- article associated with the marks
subscriptionintegernotnullreferencesarsse_subscriptions(id)ondeletecascadeonupdatecascade,-- subscription associated with the marks; the subscription in turn belongs to a user
readbooleannotnulldefault0,-- whether the article has been read
starredbooleannotnulldefault0,-- whether the article is starred
modifiedtext,-- time at which an article was last modified by a given user
notetextnotnulldefault'',-- Tiny Tiny RSS freeform user note
touchedbooleannotnulldefault0,-- used to indicate a record has been modified during the course of some transactions
primarykey(article,subscription)-- no more than one mark-set per article per user