Browse Source

Fix feed fetching

There are almost certainly other bugs; proper tests forthcoming
microsub
J. King 7 years ago
parent
commit
9c7f4710aa
  1. 18
      lib/Database.php
  2. 2
      lib/Db/SQLite3/Result.php
  3. 17
      lib/Feed.php
  4. 3
      tests/test.php

18
lib/Database.php

@ -398,7 +398,7 @@ class Database {
if(!$this->userExists($user)) throw new User\Exception("doesNotExist", ["user" => $user, "action" => __FUNCTION__]);
// check to see if the feed exists
$feedID = $this->db->prepare("SELECT id from arsse_feeds where url is ? and username is ? and password is ?", "str", "str", "str")->run($url, $fetchUser, $fetchPassword)->getValue();
if(is_nill($feedID)) {
if(is_null($feedID)) {
// if the feed doesn't exist add it to the database; we do this unconditionally so as to lock SQLite databases for as little time as possible
$feedID = $this->db->prepare('INSERT INTO arsse_feeds(url,username,password) values(?,?,?)', 'str', 'str', 'str')->run($url, $fetchUser, $fetchPassword)->lastId();
try {
@ -419,6 +419,18 @@ class Database {
return (bool) $this->db->prepare("DELETE from arsse_subscriptions where owner is ? and id is ?", "str", "int")->run($user, $id)->changes();
}
public function subscriptionList(string $user, int $folder = null): Db\Result {
if(!Data::$user->authorize($user, __FUNCTION__)) throw new User\ExceptionAuthz("notAuthorized", ["action" => __FUNCTION__, "user" => $user]);
if(!$this->userExists($user)) throw new User\Exception("doesNotExist", ["user" => $user, "action" => __FUNCTION__]);
// check to make sure the folder exists, if one is specified
if(!is_null($folder)) {
if(!$this->db->prepare("SELECT count(*) from arsse_folders where owner is ? and id is ?", "str", "int")->run($user, $folder)->getValue()) {
throw new Db\ExceptionInput("idMissing", ["action" => __FUNCTION__, "field" => "folder", 'id' => $folder]);
}
}
return $this->db->prepare("SELECT arsse_subscriptions.id, arsse_feeds.url, arsse_feeds.title from arsse_subscriptions join arsse_feeds on feed = arsse_feeds.id where arsse_subscriptions.owner is ?", "str")->run($user);
}
public function feedUpdate(int $feedID, bool $throwError = false): bool {
$this->db->begin();
try {
@ -452,7 +464,7 @@ class Database {
//prepare the necessary statements to perform the update
if(sizeof($feed->newItems) || sizeof($feed->changedItems)) {
$qInsertCategory = $this->db->prepare('INSERT INTO arsse_categories(article,name) values(?,?)', 'int', 'str');
$qInsertEdition = $this->db->prepare('INSERT INTO arse_editions(article) values(?)', 'int');
$qInsertEdition = $this->db->prepare('INSERT INTO arsse_editions(article) values(?)', 'int');
}
if(sizeof($feed->newItems)) {
$qInsertArticle = $this->db->prepare(
@ -547,7 +559,7 @@ class Database {
// perform the query
return $articles = $this->db->prepare(
'SELECT id, DATEFORMAT("unix", edited) AS edited_date, guid, url_title_hash, url_content_hash, title_content_hash FROM arsse_articles '.
'WHERE feed is ? and (guid in($cId) or url_title_hash in($cHashUT) or url_content_hash in($cHashUC) or title_content_hash in($cHashTC)',
'WHERE feed is ? and (guid in($cId) or url_title_hash in($cHashUT) or url_content_hash in($cHashUC) or title_content_hash in($cHashTC))',
'int', $tId, $tHashUT, $tHashUC, $tHashTC
)->run($feedID, $ids, $hashesUT, $hashesUC, $hashesTC);
}

2
lib/Db/SQLite3/Result.php

@ -52,7 +52,7 @@ class Result implements \JKingWeb\Arsse\Db\Result {
}
public function __destruct() {
$this->set->finalize();
try{$this->set->finalize();} catch(\Throwable $e) {}
unset($this->set);
}

17
lib/Feed.php

@ -24,7 +24,7 @@ class Feed {
// format the HTTP Last-Modified date returned
$lastMod = $this->resource->getLastModified();
if(strlen($lastMod)) {
$this->$lastModified = \DateTime::createFromFormat("!D, d M Y H:i:s e", $lastMod);
$this->lastModified = \DateTime::createFromFormat("!D, d M Y H:i:s e", $lastMod);
}
$this->modified = $this->resource->isModified();
//parse the feed, if it has been modified
@ -162,6 +162,7 @@ class Feed {
$new = $tentative = $edited = [];
// iterate through the articles and for each determine whether it is existing, edited, or entirely new
foreach($items as $index => $i) {
$found = false;
foreach($articles as $a) {
if(
// the item matches if the GUID matches...
@ -174,28 +175,29 @@ class Feed {
if($i->updatedDate && $i->updatedDate->getTimestamp() !== $match['edited_date']) {
// if the item has an edit timestamp and it doesn't match that of the article in the database, the the article has been edited
// we store the item index and database record ID as a key/value pair
$found = true;
$edited[$index] = $a['id'];
break;
} else if($i->urlTitleHash !== $a['url_title_hash'] || $i->urlContentHash !== $a['url_content_hash'] || $i->titleContentHash !== $a['title_content_hash']) {
// if any of the hashes do not match, then the article has been edited
$found = true;
$edited[$index] = $a['id'];
break;
} else {
// otherwise the item is unchanged and we can ignore it
$found = true;
break;
}
} else {
// if we don't have a match, add the item to the tentatively new list
$tentative[] = $index;
}
}
if(!$found) $tentative[] = $index;
}
if(sizeof($tentative)) {
// if we need to, perform a second pass on the database looking specifically for IDs and hashes of the new items
$ids = $hashesUT = $hashesUC = $hashesTC = [];
foreach($tentative as $index) {
$i = $items[$index];
if($i->id) $ids[] = $id->id;
if($i->id) $ids[] = $i->id;
$hashesUT[] = $i->urlTitleHash;
$hashesUC[] = $i->urlContentHash;
$hashesTC[] = $i->titleContentHash;
@ -203,6 +205,7 @@ class Feed {
$articles = Data::$db->articleMatchIds($feedID, $ids, $hashesUT, $hashesUC, $hashesTC);
foreach($tentative as $index) {
$i = $items[$index];
$found = false;
foreach($articles as $a) {
if(
// the item matches if the GUID matches...
@ -215,14 +218,17 @@ class Feed {
if($i->updatedDate && $i->updatedDate->getTimestamp() !== $match['edited_date']) {
// if the item has an edit timestamp and it doesn't match that of the article in the database, the the article has been edited
// we store the item index and database record ID as a key/value pair
$found = true;
$edited[$index] = $a['id'];
break;
} else if($i->urlTitleHash !== $a['url_title_hash'] || $i->urlContentHash !== $a['url_content_hash'] || $i->titleContentHash !== $a['title_content_hash']) {
// if any of the hashes do not match, then the article has been edited
$found = true;
$edited[$index] = $a['id'];
break;
} else {
// otherwise the item is unchanged and we can ignore it
$found = true;
break;
}
} else {
@ -230,6 +236,7 @@ class Feed {
$new[] = $index;
}
}
if(!$found) $new[] = $index;
}
}
// FIXME: fetch full content when appropriate

3
tests/test.php

@ -20,4 +20,5 @@ Data::$user->authorizationEnabled(false);
Data::$user->rightsSet($user, User\Driver::RIGHTS_GLOBAL_ADMIN);
Data::$user->authorizationEnabled(true);
Data::$db->folderAdd($user, ['name' => 'ook']);
Data::$db->subscriptionAdd($user, "http://www.pcgamer.com/rss/");
Data::$db->subscriptionAdd($user, "https://jkingweb.ca/test.atom");
var_export(Data::$db->subscriptionList($user)->getAll());
Loading…
Cancel
Save