From 0a0aabe4edad8ad1423f98438ff5b72739921ba3 Mon Sep 17 00:00:00 2001 From: "J. King" Date: Sat, 30 Sep 2017 12:52:05 -0400 Subject: [PATCH] Merge master --- .gitattributes | 31 ++++------- .gitignore | 53 +++++++++---------- CHANGELOG | 12 +++++ bootstrap.php | 2 +- build.xml | 1 + dist/nginx-fcgi.conf | 3 +- lib/CLI.php | 2 +- lib/Database.php | 8 +-- lib/Feed.php | 13 ++--- lib/REST.php | 8 ++- lib/REST/NextCloudNews/Versions.php | 2 +- lib/REST/Request.php | 5 ++ lib/REST/Response.php | 26 +++++---- tests/Feed/TestFeed.php | 9 ++++ .../NextCloudNews/TestNCNVersionDiscovery.php | 2 +- tests/docroot/Feed/Discovery/Feed.php | 12 +++++ tests/docroot/Feed/Discovery/Invalid.php | 8 +++ tests/docroot/Feed/Discovery/Valid.php | 9 ++++ tests/lib/Database/SeriesSubscription.php | 4 +- 19 files changed, 135 insertions(+), 75 deletions(-) create mode 100644 CHANGELOG create mode 100644 tests/docroot/Feed/Discovery/Feed.php create mode 100644 tests/docroot/Feed/Discovery/Invalid.php create mode 100644 tests/docroot/Feed/Discovery/Valid.php diff --git a/.gitattributes b/.gitattributes index 2431c40..7df9c77 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,22 +1,13 @@ -# Auto detect text files and perform LF normalization -* text=auto +* text=auto encoding=utf-8 -# Custom for Visual Studio -*.cs diff=csharp -*.sln merge=union -*.csproj merge=union -*.vbproj merge=union -*.fsproj merge=union -*.dbproj merge=union +*.html diff=html +*.php diff=php +*.bat eol=crlf +.gitignore -eol -# Standard to msysgit -*.doc diff=astextplain -*.DOC diff=astextplain -*.docx diff=astextplain -*.DOCX diff=astextplain -*.dot diff=astextplain -*.DOT diff=astextplain -*.pdf diff=astextplain -*.PDF diff=astextplain -*.rtf diff=astextplain -*.RTF diff=astextplain + +tests/ export-ignore +.* export-ignore +build.xml export-ignore +composer.* export-ignore +phpdoc.* export-ignore diff --git a/.gitignore b/.gitignore index 9ac5f60..005d958 100644 --- a/.gitignore +++ b/.gitignore @@ -1,48 +1,45 @@ -#dependencies -vendor/ -#temp files +# Temporary files and dependencies + +vendor/ documentation/ -tests/coverage +tests/coverage/ +build/ arsse.db* config.php .php_cs.cache -build -# Windows image file caches + + + +# Windows files + Thumbs.db ehthumbs.db - -# Folder config file Desktop.ini - -# Recycle Bin used on file shares $RECYCLE.BIN/ -# Windows Installer files -*.cab -*.msi -*.msm -*.msp - -# ========================= -# Operating System Files -# ========================= -# OSX -# ========================= +# macOS files .DS_Store .AppleDouble .LSOverride - -# Icon must ends with two \r. Icon - - -# Thumbnails ._* - -# Files that might appear on external disk .Spotlight-V100 .Trashes + +# archives + +*.zip +*.7z +*.tar.gz +*.tgz +*.deb +*.rpm +*.dmg +*.cab +*.msi +*.msm +*.msp diff --git a/CHANGELOG b/CHANGELOG new file mode 100644 index 0000000..451ce8a --- /dev/null +++ b/CHANGELOG @@ -0,0 +1,12 @@ +Version 0.1.1 (2017-09-30) +========================== + +Bug fixes: +- Perform feed discovery like NextCloud News does +- Respond correctly to HEAD requests +- Various minor fixes + +Version 0.1.0 (2017-08-29) +========================== + +Initial release \ No newline at end of file diff --git a/bootstrap.php b/bootstrap.php index 8ba3a73..04b4f0b 100644 --- a/bootstrap.php +++ b/bootstrap.php @@ -4,7 +4,7 @@ namespace JKingWeb\Arsse; const BASE = __DIR__.DIRECTORY_SEPARATOR; const NS_BASE = __NAMESPACE__."\\"; -const VERSION = "0.1.0"; +const VERSION = "0.1.1"; require_once BASE."vendor".DIRECTORY_SEPARATOR."autoload.php"; ignore_user_abort(true); \ No newline at end of file diff --git a/build.xml b/build.xml index 5524305..25753f7 100644 --- a/build.xml +++ b/build.xml @@ -11,6 +11,7 @@ + diff --git a/dist/nginx-fcgi.conf b/dist/nginx-fcgi.conf index fc76d31..fb37825 100644 --- a/dist/nginx-fcgi.conf +++ b/dist/nginx-fcgi.conf @@ -9,4 +9,5 @@ fastcgi_param REQUEST_METHOD $request_method; fastcgi_param CONTENT_TYPE $content_type; fastcgi_param CONTENT_LENGTH $content_length; fastcgi_param REQUEST_URI $request_uri; -fastcgi_param HTTPS $https if_not_empty; \ No newline at end of file +fastcgi_param HTTPS $https if_not_empty; +fastcgi_param REMOTE_USER $remote_user; \ No newline at end of file diff --git a/lib/CLI.php b/lib/CLI.php index 56dfbad..971197f 100644 --- a/lib/CLI.php +++ b/lib/CLI.php @@ -84,7 +84,7 @@ USAGE_TEXT; public function userAdd(string $user, string $password = null): int { $passwd = Arsse::$user->add($user, $password); if (is_null($password)) { - echo $passwd; + echo $passwd.\PHP_EOL; } return 0; } diff --git a/lib/Database.php b/lib/Database.php index bf6ce12..5a90b19 100644 --- a/lib/Database.php +++ b/lib/Database.php @@ -459,7 +459,7 @@ class Database { } } - public function subscriptionAdd(string $user, string $url, string $fetchUser = "", string $fetchPassword = ""): int { + public function subscriptionAdd(string $user, string $url, string $fetchUser = "", string $fetchPassword = "", bool $discover = true): int { if (!Arsse::$user->authorize($user, __FUNCTION__)) { throw new User\ExceptionAuthz("notAuthorized", ["action" => __FUNCTION__, "user" => $user]); } @@ -470,7 +470,7 @@ class Database { $feedID = $this->db->prepare('INSERT INTO arsse_feeds(url,username,password) values(?,?,?)', 'str', 'str', 'str')->run($url, $fetchUser, $fetchPassword)->lastId(); try { // perform an initial update on the newly added feed - $this->feedUpdate($feedID, true); + $this->feedUpdate($feedID, true, $discover); } catch (\Throwable $e) { // if the update fails, delete the feed we just added $this->db->prepare('DELETE from arsse_feeds where id is ?', 'int')->run($feedID); @@ -601,7 +601,7 @@ class Database { return array_column($feeds, 'id'); } - public function feedUpdate($feedID, bool $throwError = false): bool { + public function feedUpdate($feedID, bool $throwError = false, bool $discover = false): bool { $tr = $this->db->begin(); // check to make sure the feed exists if (!ValueInfo::id($feedID)) { @@ -617,7 +617,7 @@ class Database { // here. When an exception is thrown it should update the database with the // error instead of failing; if other exceptions are thrown, we should simply roll back try { - $feed = new Feed((int) $feedID, $f['url'], (string) Date::transform($f['modified'], "http", "sql"), $f['etag'], $f['username'], $f['password'], $scrape); + $feed = new Feed((int) $feedID, $f['url'], (string) Date::transform($f['modified'], "http", "sql"), $f['etag'], $f['username'], $f['password'], $scrape, $discover); if (!$feed->modified) { // if the feed hasn't changed, just compute the next fetch time and record it $this->db->prepare("UPDATE arsse_feeds SET updated = CURRENT_TIMESTAMP, next_fetch = ? WHERE id is ?", 'datetime', 'int')->run($feed->nextFetch, $feedID); diff --git a/lib/Feed.php b/lib/Feed.php index a4dc476..ee6114c 100644 --- a/lib/Feed.php +++ b/lib/Feed.php @@ -21,7 +21,7 @@ class Feed { public $newItems = []; public $changedItems = []; - public function __construct(int $feedID = null, string $url, string $lastModified = '', string $etag = '', string $username = '', string $password = '', bool $scrape = false) { + public function __construct(int $feedID = null, string $url, string $lastModified = '', string $etag = '', string $username = '', string $password = '', bool $scrape = false, bool $discover = false) { // set the configuration $userAgent = Arsse::$conf->fetchUserAgentString ?? sprintf('Arsse/%s (%s %s; %s; https://code.jkingweb.ca/jking/arsse) PicoFeed (https://github.com/fguillot/picoFeed)', VERSION, // Arsse version @@ -36,7 +36,7 @@ class Feed { $this->config->setClientUserAgent($userAgent); $this->config->setGrabberUserAgent($userAgent); // fetch the feed - $this->download($url, $lastModified, $etag, $username, $password); + $this->download($url, $lastModified, $etag, $username, $password, $discover); // format the HTTP Last-Modified date returned $lastMod = $this->resource->getLastModified(); if (strlen($lastMod)) { @@ -65,10 +65,11 @@ class Feed { $this->nextFetch = $this->computeNextFetch(); } - protected function download(string $url, string $lastModified = '', string $etag = '', string $username = '', string $password = ''): bool { + protected function download(string $url, string $lastModified, string $etag, string $username, string $password, bool $discover): bool { + $action = $discover ? "discover" : "download"; try { $this->reader = new Reader($this->config); - $this->resource = $this->reader->download($url, $lastModified, $etag, $username, $password); + $this->resource = $this->reader->$action($url, $lastModified, $etag, $username, $password); } catch (PicoFeedException $e) { throw new Feed\Exception($url, $e); } @@ -361,13 +362,13 @@ class Feed { protected function computeLastModified() { if (!$this->modified) { - return $this->lastModified; + return $this->lastModified; // @codeCoverageIgnore } $dates = $this->gatherDates(); if (sizeof($dates)) { return Date::normalize($dates[0]); } else { - return null; + return null; // @codeCoverageIgnore } } diff --git a/lib/REST.php b/lib/REST.php index 28b9985..15bfac7 100644 --- a/lib/REST.php +++ b/lib/REST.php @@ -43,7 +43,13 @@ class REST { $req->refreshURL(); $class = $this->apis[$api]['class']; $drv = new $class(); - return $drv->dispatch($req); + if ($req->head) { + $res = $drv->dispatch($req); + $res->head = true; + return $res; + } else { + return $drv->dispatch($req); + } } public function apiMatch(string $url, array $map): string { diff --git a/lib/REST/NextCloudNews/Versions.php b/lib/REST/NextCloudNews/Versions.php index df260cc..9d5029d 100644 --- a/lib/REST/NextCloudNews/Versions.php +++ b/lib/REST/NextCloudNews/Versions.php @@ -11,7 +11,7 @@ class Versions implements \JKingWeb\Arsse\REST\Handler { public function dispatch(\JKingWeb\Arsse\REST\Request $req): Response { // if a method other than GET was used, this is an error if ($req->method != "GET") { - return new Response(405); + return new Response(405, "", "", ["Allow: GET"]); } if (preg_match("<^/?$>", $req->path)) { // if the request path is an empty string or just a slash, return the supported versions diff --git a/lib/REST/Request.php b/lib/REST/Request.php index bdd3a39..c21ca48 100644 --- a/lib/REST/Request.php +++ b/lib/REST/Request.php @@ -4,6 +4,7 @@ namespace JKingWeb\Arsse\REST; class Request { public $method = "GET"; + public $head = false; public $url = ""; public $path =""; public $paths = []; @@ -26,6 +27,10 @@ class Request { $this->url = $url; $this->body = $body; $this->type = $contentType; + if ($this->method=="HEAD") { + $this->head = true; + $this->method = "GET"; + } $this->refreshURL(); } diff --git a/lib/REST/Response.php b/lib/REST/Response.php index fc18723..5323695 100644 --- a/lib/REST/Response.php +++ b/lib/REST/Response.php @@ -9,6 +9,7 @@ class Response { const T_XML = "application/xml"; const T_TEXT = "text/plain"; + public $head = false; public $code; public $payload; public $type; @@ -24,15 +25,11 @@ class Response { public function output() { if (!headers_sent()) { - try { - $statusText = Arsse::$lang->msg("HTTP.Status.".$this->code); - } catch (\JKingWeb\Arsse\Lang\Exception $e) { - $statusText = ""; + foreach ($this->fields as $field) { + header($field); } - header("Status: ".$this->code." ".$statusText); $body = ""; if (!is_null($this->payload)) { - header("Content-Type: ".$this->type); switch ($this->type) { case self::T_JSON: $body = (string) json_encode($this->payload, \JSON_PRETTY_PRINT); @@ -42,10 +39,21 @@ class Response { break; } } - foreach ($this->fields as $field) { - header($field); + if (strlen($body)) { + header("Content-Type: ".$this->type); + header("Content-Length: ".strlen($body)); + } elseif ($this->code==200) { + $this->code = 204; + } + try { + $statusText = Arsse::$lang->msg("HTTP.Status.".$this->code); + } catch (\JKingWeb\Arsse\Lang\Exception $e) { + $statusText = ""; + } + header("Status: ".$this->code." ".$statusText); + if (!$this->head) { + echo $body; } - echo $body; } else { throw new REST\Exception("headersSent"); } diff --git a/tests/Feed/TestFeed.php b/tests/Feed/TestFeed.php index 74c2426..8957ba8 100644 --- a/tests/Feed/TestFeed.php +++ b/tests/Feed/TestFeed.php @@ -133,6 +133,15 @@ class TestFeed extends Test\AbstractTest { $this->assertSame($categories, $f->data->items[5]->categories); } + public function testDiscoverAFeedSuccessfully() { + $this->assertInstanceOf(Feed::class, new Feed(null, $this->base."Discovery/Valid", "", "", "", "", false, true)); + } + + public function testDiscoverAFeedUnsuccessfully() { + $this->assertException("subscriptionNotFound", "Feed"); + new Feed(null, $this->base."Discovery/Invalid", "", "", "", "", false, true); + } + public function testParseEntityExpansionAttack() { $this->assertException("xmlEntity", "Feed"); new Feed(null, $this->base."Parsing/XEEAttack"); diff --git a/tests/REST/NextCloudNews/TestNCNVersionDiscovery.php b/tests/REST/NextCloudNews/TestNCNVersionDiscovery.php index ad3f15a..ddef865 100644 --- a/tests/REST/NextCloudNews/TestNCNVersionDiscovery.php +++ b/tests/REST/NextCloudNews/TestNCNVersionDiscovery.php @@ -26,7 +26,7 @@ class TestNCNVersionDiscovery extends Test\AbstractTest { } public function testUseIncorrectMethod() { - $exp = new Response(405); + $exp = new Response(405, "", "", ["Allow: GET"]); $h = new REST\NextCloudNews\Versions(); $req = new Request("POST", "/"); $res = $h->dispatch($req); diff --git a/tests/docroot/Feed/Discovery/Feed.php b/tests/docroot/Feed/Discovery/Feed.php new file mode 100644 index 0000000..a13398a --- /dev/null +++ b/tests/docroot/Feed/Discovery/Feed.php @@ -0,0 +1,12 @@ + "application/rss+xml", + 'content' => << + + Test feed + http://example.com/ + Example newsfeed title + + +MESSAGE_BODY +]; diff --git a/tests/docroot/Feed/Discovery/Invalid.php b/tests/docroot/Feed/Discovery/Invalid.php new file mode 100644 index 0000000..9a2f49f --- /dev/null +++ b/tests/docroot/Feed/Discovery/Invalid.php @@ -0,0 +1,8 @@ + "text/html", + 'content' => << +Example article + +MESSAGE_BODY +]; diff --git a/tests/docroot/Feed/Discovery/Valid.php b/tests/docroot/Feed/Discovery/Valid.php new file mode 100644 index 0000000..9f34f71 --- /dev/null +++ b/tests/docroot/Feed/Discovery/Valid.php @@ -0,0 +1,9 @@ + "text/html", + 'content' => << +Example article + + +MESSAGE_BODY +]; diff --git a/tests/lib/Database/SeriesSubscription.php b/tests/lib/Database/SeriesSubscription.php index b68ecb4..fc528c3 100644 --- a/tests/lib/Database/SeriesSubscription.php +++ b/tests/lib/Database/SeriesSubscription.php @@ -135,7 +135,7 @@ trait SeriesSubscription { Phake::when(Arsse::$db)->feedUpdate->thenReturn(true); $this->assertSame($subID, Arsse::$db->subscriptionAdd($this->user, $url)); Phake::verify(Arsse::$user)->authorize($this->user, "subscriptionAdd"); - Phake::verify(Arsse::$db)->feedUpdate($feedID, true); + Phake::verify(Arsse::$db)->feedUpdate($feedID, true, true); $state = $this->primeExpectations($this->data, [ 'arsse_feeds' => ['id','url','username','password'], 'arsse_subscriptions' => ['id','owner','feed'], @@ -153,7 +153,7 @@ trait SeriesSubscription { Arsse::$db->subscriptionAdd($this->user, $url); } catch (FeedException $e) { Phake::verify(Arsse::$user)->authorize($this->user, "subscriptionAdd"); - Phake::verify(Arsse::$db)->feedUpdate($feedID, true); + Phake::verify(Arsse::$db)->feedUpdate($feedID, true, true); $state = $this->primeExpectations($this->data, [ 'arsse_feeds' => ['id','url','username','password'], 'arsse_subscriptions' => ['id','owner','feed'],