From 5335d331f79d57bf831971b89ddae92b082c35ac Mon Sep 17 00:00:00 2001 From: "J. King" Date: Mon, 21 Jan 2019 09:55:25 -0500 Subject: [PATCH 01/11] Fix configuration exporting --- lib/Conf.php | 42 +++++++++++++++++++++++++---------- tests/cases/Conf/TestConf.php | 2 ++ 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/lib/Conf.php b/lib/Conf.php index e241d47..3f8f5f2 100644 --- a/lib/Conf.php +++ b/lib/Conf.php @@ -184,17 +184,23 @@ class Conf { /** Outputs configuration settings, either non-default ones or all, as an associative array * @param bool $full Whether to output all configuration options rather than only changed ones */ public function export(bool $full = false): array { - $ref = new self; - $out = []; $conf = new \ReflectionObject($this); + $ref = (new \ReflectionClass($this))->getDefaultProperties(); + $out = []; foreach ($conf->getProperties(\ReflectionProperty::IS_PUBLIC) as $prop) { $name = $prop->name; - // add the property to the output if the value is of a supported type and either: - // 1. full output has been requested - // 2. the property is not defined in the class - // 3. it differs from the default - if ((is_scalar($this->$name) || is_null($this->$name)) && ($full || !$prop->isDefault() || $this->$name !== $ref->$name)) { - $out[$name] = $this->$name; + $value = $prop->getValue($this); + if ($prop->isDefault()) { + $default = $ref[$name]; + // if the property is a known property (rather than one added by a hypothetical plug-in) + // we convert intervals to strings and then export anything which doesn't match the default value + $value = $this->propertyExport($name, $value); + if ((is_scalar($value) || is_null($value)) && ($full || $value !== $ref[$name])) { + $out[$name] = $value; + } + } elseif (is_scalar($value) || is_null($value)) { + // otherwise export the property only if it is scalar + $out[$name] = $value; } } return $out; @@ -213,13 +219,11 @@ class Conf { // retrieve the property's docblock, if it exists try { $doc = (new \ReflectionProperty(self::class, $prop))->getDocComment(); - } catch (\ReflectionException $e) { - } - if ($doc) { // parse the docblock to extract the property description - if (preg_match("<@var\s+\S+\s+(.+?)(?:\s*\*/)?$>m", $doc, $match)) { + if (preg_match("<@var\s+\S+\s+(.+?)(?:\s*\*/)?\s*$>m", $doc, $match)) { $comment = $match[1]; } + } catch (\ReflectionException $e) { } // append the docblock description if there is one, or an empty comment otherwise $out .= " // ".$comment.PHP_EOL; @@ -310,4 +314,18 @@ class Conf { throw new Conf\Exception("typeMismatch", ['param' => $key, 'type' => self::TYPE_NAMES[$type], 'file' => $file, 'nullable' => $nullable]); } } + + protected function propertyExport(string $key, $value) { + $value = ($value instanceof \DateInterval) ? Value::normalize($value, Value::T_STRING) : $value; + switch ($key) { + case "dbDriver": + return array_flip(Database::DRIVER_NAMES)[$value] ?? $value; + case "userDriver": + return array_flip(User::DRIVER_NAMES)[$value] ?? $value; + case "serviceDriver": + return array_flip(Service::DRIVER_NAMES)[$value] ?? $value; + default: + return $value; + } + } } diff --git a/tests/cases/Conf/TestConf.php b/tests/cases/Conf/TestConf.php index f4a4430..4016443 100644 --- a/tests/cases/Conf/TestConf.php +++ b/tests/cases/Conf/TestConf.php @@ -122,10 +122,12 @@ class TestConf extends \JKingWeb\Arsse\Test\AbstractTest { $conf->lang = ["en", "fr"]; // should not be exported: not scalar $conf->dbSQLite3File = "test.db"; // should be exported: value changed $conf->userDriver = null; // should be exported: changed value, even when null + $conf->serviceFrequency = new \DateInterval("PT1H"); // should be exported (as string): value changed $conf->someCustomProperty = "Look at me!"; // should be exported: unknown property $exp = [ 'dbSQLite3File' => "test.db", 'userDriver' => null, + 'serviceFrequency' => "PT1H", 'someCustomProperty' => "Look at me!", ]; $this->assertSame($exp, $conf->export()); From 37025bb49f67b0345cc308fbcc73d2d40ec5fc05 Mon Sep 17 00:00:00 2001 From: "J. King" Date: Mon, 21 Jan 2019 10:23:25 -0500 Subject: [PATCH 02/11] Documentation update --- CHANGELOG | 4 +++- README.md | 3 ++- UPGRADING | 32 ++++++++++++++++++++------------ 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index b3873f5..0c71507 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,15 +1,17 @@ -Version 0.6.0 (????-??-??) +Version 0.6.0 (2019-01-21) ========================== New features: - Support for PostgreSQL databases - Support for MySQL databases +- Validation of configuration parameters Bug fixes: - Use a general-purpose Unicode collation with SQLite databases Changes: - Improve performance of common database queries by 80-90% +- Make configuration defaults consistent with their defined types Version 0.5.1 (2018-11-10) ========================== diff --git a/README.md b/README.md index d7dcb39..2cec044 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ At present the software should be considered in an "alpha" state: though its cor - Providing more sync protocols (Google Reader, Fever, others) - Better packaging and configuration samples +- A user manual ## Requirements @@ -75,7 +76,7 @@ Please refer to `CONTRIBUTING.md` for guidelines on contributing code to The Ars Functionally there is no reason to prefer either SQLite or PostgreSQL over the other. SQLite is significantly simpler to set up in most cases, requiring only read and write access to a containing directory in order to function; PostgreSQL may perform better than SQLite when serving hundreds of users or more, though this has not been tested. -MySQL, on the other hand, is *not recommended* due to its relatively constrained index prefix limits which may cause some newsfeeds which would otherwise work to be rejected. If using MySQL, special care should also be taken when performing schema upgrades, as errors during the process can leave the database in a half-upgraded state which The Arsse cannot itself recover from. +MySQL, on the other hand, is **not recommended** due to its relatively constrained index prefix limits which may cause some newsfeeds which would otherwise work to be rejected. If using MySQL, special care should also be taken when performing schema upgrades, as errors during the process can leave the database in a half-upgraded state which The Arsse cannot itself recover from. Note that MariaDB is not compatible with The Arsse: its support for common table expressions is, as of this writing, not sufficient for our needs. diff --git a/UPGRADING b/UPGRADING index 3abe6ee..a837396 100644 --- a/UPGRADING +++ b/UPGRADING @@ -1,41 +1,49 @@ General upgrade notes ===================== -When upgrading between any two versions of The Arsse, the following are usually prudent: +When upgrading between any two versions of The Arsse, the following are +usually prudent: - Back up your database - Check for any changes to sample Web server configuration - Check for any changes to sample systemd unit or other init files -- If installing from source, update dependencies with `composer install -o --no-dev` +- If installing from source, update dependencies with: + `composer install -o --no-dev` Upgrading from 0.5.1 to 0.6.0 ============================= -- The database schema has changed from rev3 to rev4; if upgrading the database manually, apply the 3.sql file +- The database schema has changed from rev3 to rev4; if upgrading the database + manually, apply the 3.sql file +- Configuration is now validated for type and semantics: some previously + working configurations may no longer be accepted Upgrading from 0.2.1 to 0.3.0 ============================= - The following Composer dependencies have been added: - - zendframework/zend-diactoros - - psr/http-message + - zendframework/zend-diactoros + - psr/http-message Upgrading from 0.2.0 to 0.2.1 ============================= -- The database schema has changed from rev2 to rev3; if upgrading the database manually, apply the 2.sql file +- The database schema has changed from rev2 to rev3; if upgrading the database + manually, apply the 2.sql file Upgrading from 0.1.x to 0.2.0 ============================= -- The database schema has changed from rev1 to rev2; if upgrading the database manually, apply the 1.sql file -- Web server configuration has changed to accommodate Tiny Tiny RSS; the following URL paths are affected: - - /tt-rss/api/ - - /tt-rss/feed-icons/ - - /tt-rss/images/ +- The database schema has changed from rev1 to rev2; if upgrading the database + manually, apply the 1.sql file +- Web server configuration has changed to accommodate Tiny Tiny RSS; the + following URL paths are affected: + - /tt-rss/api/ + - /tt-rss/feed-icons/ + - /tt-rss/images/ - The following Composer dependencies have been added: - - jkingweb/druuid + - jkingweb/druuid From 05aadfe7c778c82018c47c4556c45e694f544345 Mon Sep 17 00:00:00 2001 From: "J. King" Date: Mon, 21 Jan 2019 10:40:39 -0500 Subject: [PATCH 03/11] Use correct SQLite chema change procedure; version bump --- CHANGELOG | 1 + lib/Arsse.php | 2 +- lib/Db/SQLite3/Driver.php | 2 -- sql/SQLite3/1.sql | 14 +++++----- sql/SQLite3/2.sql | 56 +++++++++++++++++++-------------------- sql/SQLite3/3.sql | 8 +++--- 6 files changed, 41 insertions(+), 42 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 0c71507..ee6bce2 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -8,6 +8,7 @@ New features: Bug fixes: - Use a general-purpose Unicode collation with SQLite databases +- Use the correct SQLite schema change procedure for 3.25 and later Changes: - Improve performance of common database queries by 80-90% diff --git a/lib/Arsse.php b/lib/Arsse.php index 39b8336..58b843d 100644 --- a/lib/Arsse.php +++ b/lib/Arsse.php @@ -7,7 +7,7 @@ declare(strict_types=1); namespace JKingWeb\Arsse; class Arsse { - const VERSION = "0.5.1"; + const VERSION = "0.6.0"; /** @var Lang */ public static $lang; diff --git a/lib/Db/SQLite3/Driver.php b/lib/Db/SQLite3/Driver.php index 612072b..2e741b5 100644 --- a/lib/Db/SQLite3/Driver.php +++ b/lib/Db/SQLite3/Driver.php @@ -123,14 +123,12 @@ class Driver extends \JKingWeb\Arsse\Db\AbstractDriver { public function schemaUpdate(int $to, string $basePath = null): bool { // turn off foreign keys $this->exec("PRAGMA foreign_keys = no"); - $this->exec("PRAGMA legacy_alter_table = yes"); // run the generic updater try { parent::schemaUpdate($to, $basePath); } finally { // turn foreign keys back on $this->exec("PRAGMA foreign_keys = yes"); - $this->exec("PRAGMA legacy_alter_table = no"); } return true; } diff --git a/sql/SQLite3/1.sql b/sql/SQLite3/1.sql index b96bd79..dc7862d 100644 --- a/sql/SQLite3/1.sql +++ b/sql/SQLite3/1.sql @@ -2,7 +2,7 @@ -- Copyright 2017 J. King, Dustin Wilson et al. -- See LICENSE and AUTHORS files for details -create table arsse_sessions ( +create table arsse_sessions( -- sessions for Tiny Tiny RSS (and possibly others) id text primary key, -- UUID of session created text not null default CURRENT_TIMESTAMP, -- Session start timestamp @@ -10,7 +10,7 @@ create table arsse_sessions ( user text not null references arsse_users(id) on delete cascade on update cascade -- user associated with the session ) without rowid; -create table arsse_labels ( +create table arsse_labels( -- user-defined article labels for Tiny Tiny RSS id integer primary key, -- numeric ID owner text not null references arsse_users(id) on delete cascade on update cascade, -- owning user @@ -19,7 +19,7 @@ create table arsse_labels ( unique(owner,name) ); -create table arsse_label_members ( +create table arsse_label_members( -- uabels assignments for articles label integer not null references arsse_labels(id) on delete cascade, -- label ID associated to an article; label IDs belong to a user article integer not null references arsse_articles(id) on delete cascade, -- article associated to a label @@ -32,8 +32,7 @@ create table arsse_label_members ( -- alter marks table to add Tiny Tiny RSS' notes -- SQLite has limited ALTER TABLE support, so the table must be re-created -- and its data re-entered; other database systems have a much simpler prodecure -alter table arsse_marks rename to arsse_marks_old; -create table arsse_marks( +create table arsse_marks_new( -- users' actions on newsfeed entries article integer not null references arsse_articles(id) on delete cascade, -- article associated with the marks subscription integer not null references arsse_subscriptions(id) on delete cascade on update cascade, -- subscription associated with the marks; the subscription in turn belongs to a user @@ -43,8 +42,9 @@ create table arsse_marks( note text not null default '', -- Tiny Tiny RSS freeform user note primary key(article,subscription) -- no more than one mark-set per article per user ); -insert into arsse_marks(article,subscription,read,starred,modified) select article,subscription,read,starred,modified from arsse_marks_old; -drop table arsse_marks_old; +insert into arsse_marks_new(article,subscription,read,starred,modified) select article,subscription,read,starred,modified from arsse_marks; +drop table arsse_marks; +alter table arsse_marks_new rename to arsse_marks; -- set version marker pragma user_version = 2; diff --git a/sql/SQLite3/2.sql b/sql/SQLite3/2.sql index 7340290..b378467 100644 --- a/sql/SQLite3/2.sql +++ b/sql/SQLite3/2.sql @@ -5,8 +5,7 @@ -- Correct collation sequences in order for various things to sort case-insensitively -- SQLite has limited ALTER TABLE support, so the tables must be re-created -- and their data re-entered; other database systems have a much simpler prodecure -alter table arsse_users rename to arsse_users_old; -create table arsse_users( +create table arsse_users_new( -- users id text primary key not null collate nocase, -- user id password text, -- password, salted and hashed; if using external authentication this would be blank @@ -16,11 +15,11 @@ create table arsse_users( admin boolean default 0, -- whether the user is a member of the special "admin" group rights integer not null default 0 -- temporary admin-rights marker FIXME: remove reliance on this ); -insert into arsse_users(id,password,name,avatar_type,avatar_data,admin,rights) select id,password,name,avatar_type,avatar_data,admin,rights from arsse_users_old; -drop table arsse_users_old; +insert into arsse_users_new(id,password,name,avatar_type,avatar_data,admin,rights) select id,password,name,avatar_type,avatar_data,admin,rights from arsse_users; +drop table arsse_users; +alter table arsse_users_new rename to arsse_users; -alter table arsse_folders rename to arsse_folders_old; -create table arsse_folders( +create table arsse_folders_new( -- folders, used by NextCloud News and Tiny Tiny RSS -- feed subscriptions may belong to at most one folder; -- in Tiny Tiny RSS folders may nest @@ -31,11 +30,11 @@ create table arsse_folders( modified text not null default CURRENT_TIMESTAMP, -- time at which the folder itself (not its contents) was changed; not currently used unique(owner,name,parent) -- cannot have multiple folders with the same name under the same parent for the same owner ); -insert into arsse_folders select * from arsse_folders_old; -drop table arsse_folders_old; +insert into arsse_folders_new select * from arsse_folders; +drop table arsse_folders; +alter table arsse_folders_new rename to arsse_folders; -alter table arsse_feeds rename to arsse_feeds_old; -create table arsse_feeds( +create table arsse_feeds_new( -- newsfeeds, deduplicated -- users have subscriptions to these feeds in another table id integer primary key, -- sequence number @@ -56,11 +55,11 @@ create table arsse_feeds( scrape boolean not null default 0, -- whether to use picoFeed's content scraper with this feed unique(url,username,password) -- a URL with particular credentials should only appear once ); -insert into arsse_feeds select * from arsse_feeds_old; -drop table arsse_feeds_old; +insert into arsse_feeds_new select * from arsse_feeds; +drop table arsse_feeds; +alter table arsse_feeds_new rename to arsse_feeds; -alter table arsse_subscriptions rename to arsse_subscriptions_old; -create table arsse_subscriptions( +create table arsse_subscriptions_new( -- users' subscriptions to newsfeeds, with settings id integer primary key, -- sequence number owner text not null references arsse_users(id) on delete cascade on update cascade, -- owner of subscription @@ -73,11 +72,11 @@ create table arsse_subscriptions( folder integer references arsse_folders(id) on delete cascade, -- TT-RSS category (nestable); the first-level category (which acts as NextCloud folder) is joined in when needed unique(owner,feed) -- a given feed should only appear once for a given owner ); -insert into arsse_subscriptions select * from arsse_subscriptions_old; -drop table arsse_subscriptions_old; +insert into arsse_subscriptions_new select * from arsse_subscriptions; +drop table arsse_subscriptions; +alter table arsse_subscriptions_new rename to arsse_subscriptions; -alter table arsse_articles rename to arsse_articles_old; -create table arsse_articles( +create table arsse_articles_new( -- entries in newsfeeds id integer primary key, -- sequence number feed integer not null references arsse_feeds(id) on delete cascade, -- feed for the subscription @@ -93,22 +92,22 @@ create table arsse_articles( url_content_hash text not null, -- hash of URL + content, enclosure URL, & content type; used when checking for updates and for identification if there is no guid. title_content_hash text not null -- hash of title + content, enclosure URL, & content type; used when checking for updates and for identification if there is no guid. ); -insert into arsse_articles select * from arsse_articles_old; -drop table arsse_articles_old; +insert into arsse_articles_new select * from arsse_articles; +drop table arsse_articles; +alter table arsse_articles_new rename to arsse_articles; -alter table arsse_categories rename to arsse_categories_old; -create table arsse_categories( +create table arsse_categories_new( -- author categories associated with newsfeed entries -- these are not user-modifiable article integer not null references arsse_articles(id) on delete cascade, -- article associated with the category name text collate nocase -- freeform name of the category ); -insert into arsse_categories select * from arsse_categories_old; -drop table arsse_categories_old; +insert into arsse_categories_new select * from arsse_categories; +drop table arsse_categories; +alter table arsse_categories_new rename to arsse_categories; -alter table arsse_labels rename to arsse_labels_old; -create table arsse_labels ( +create table arsse_labels_new( -- user-defined article labels for Tiny Tiny RSS id integer primary key, -- numeric ID owner text not null references arsse_users(id) on delete cascade on update cascade, -- owning user @@ -116,8 +115,9 @@ create table arsse_labels ( modified text not null default CURRENT_TIMESTAMP, -- time at which the label was last modified unique(owner,name) ); -insert into arsse_labels select * from arsse_labels_old; -drop table arsse_labels_old; +insert into arsse_labels_new select * from arsse_labels; +drop table arsse_labels; +alter table arsse_labels_new rename to arsse_labels; -- set version marker pragma user_version = 3; diff --git a/sql/SQLite3/3.sql b/sql/SQLite3/3.sql index bac79a8..0d58324 100644 --- a/sql/SQLite3/3.sql +++ b/sql/SQLite3/3.sql @@ -4,8 +4,7 @@ -- allow marks to initially have a null date due to changes in how marks are first created -- and also add a "touched" column to aid in tracking changes during the course of some transactions -alter table arsse_marks rename to arsse_marks_old; -create table arsse_marks( +create table arsse_marks_new( -- users' actions on newsfeed entries article integer not null references arsse_articles(id) on delete cascade, -- article associated with the marks subscription integer not null references arsse_subscriptions(id) on delete cascade on update cascade, -- subscription associated with the marks; the subscription in turn belongs to a user @@ -16,8 +15,9 @@ create table arsse_marks( touched boolean not null default 0, -- used to indicate a record has been modified during the course of some transactions primary key(article,subscription) -- no more than one mark-set per article per user ); -insert into arsse_marks select article,subscription,read,starred,modified,note,0 from arsse_marks_old; -drop table arsse_marks_old; +insert into arsse_marks_new select article,subscription,read,starred,modified,note,0 from arsse_marks; +drop table arsse_marks; +alter table arsse_marks_new rename to arsse_marks; -- reindex anything which uses the nocase collation sequence; it has been replaced with a Unicode collation reindex nocase; From 37131d37759e983929b16fc7eb9ce7dd675f7b78 Mon Sep 17 00:00:00 2001 From: "J. King" Date: Wed, 23 Jan 2019 09:19:26 -0500 Subject: [PATCH 04/11] Remove non-functional curl service driver for now Its requiring extensive configuration to function makes me disinclined to revive it, though it may nevertheless happen. --- lib/Conf.php | 8 +--- lib/Service/Curl/Driver.php | 77 ------------------------------------- locale/en.php | 1 - 3 files changed, 1 insertion(+), 85 deletions(-) delete mode 100644 lib/Service/Curl/Driver.php diff --git a/lib/Conf.php b/lib/Conf.php index 3f8f5f2..ce092e9 100644 --- a/lib/Conf.php +++ b/lib/Conf.php @@ -75,19 +75,13 @@ class Conf { * @see https://en.wikipedia.org/wiki/ISO_8601#Durations */ public $userSessionLifetime = "P7D"; - /** @var string Feed update service driver to use, one of "serial", "subprocess", or "curl". A fully-qualified class name may also be used for custom drivers */ + /** @var string Feed update service driver to use, one of "serial" or "subprocess". A fully-qualified class name may also be used for custom drivers */ public $serviceDriver = "subprocess"; /** @var \DateInterval The interval between checks for new articles, as an ISO 8601 duration * @see https://en.wikipedia.org/wiki/ISO_8601#Durations */ public $serviceFrequency = "PT2M"; /** @var integer Number of concurrent feed updates to perform */ public $serviceQueueWidth = 5; - /** @var string The base server address (with scheme, host, port if necessary, and terminal slash) to connect to the server when performing feed updates using cURL */ - public $serviceCurlBase = "http://localhost/"; - /** @var string The user name to use when performing feed updates using cURL */ - public $serviceCurlUser = ""; - /** @var string The password to use when performing feed updates using cURL */ - public $serviceCurlPassword = ""; /** @var \DateInterval Number of seconds to wait for data when fetching feeds from foreign servers */ public $fetchTimeout = 10; diff --git a/lib/Service/Curl/Driver.php b/lib/Service/Curl/Driver.php deleted file mode 100644 index cfb73d4..0000000 --- a/lib/Service/Curl/Driver.php +++ /dev/null @@ -1,77 +0,0 @@ -msg("Driver.Service.Curl.Name"); - } - - public static function requirementsMet(): bool { - return extension_loaded("curl"); - } - - public function __construct() { - //default curl options for individual requests - $this->options = [ - \CURLOPT_URL => Arsse::$serviceCurlBase."index.php/apps/news/api/v1-2/feeds/update", - \CURLOPT_CUSTOMREQUEST => "GET", - \CURLOPT_FAILONERROR => false, - \CURLOPT_FOLLOWLOCATION => false, - \CURLOPT_FORBID_REUSE => false, - \CURLOPT_CONNECTTIMEOUT => 20, - \CURLOPT_DNS_CACHE_TIMEOUT => 360, // FIXME: this should probably be twice the update-check interval so that the DNS cache is always in memory - \CURLOPT_PROTOCOLS => \CURLPROTO_HTTP | \CURLPROTO_HTTPS, - \CURLOPT_DEFAULT_PROTOCOL => "https", - \CURLOPT_USERAGENT => Arsse::$conf->fetchUserAgentString, - \CURLMOPT_MAX_HOST_CONNECTIONS => Arsse::$conf->serviceQueueWidth, - \CURLOPT_HTTPHEADER => [ - 'Accept: application/json', - 'Content-Type: application/json', - ], - \CURLOPT_HEADER => false, - ]; - // start an async session - $this->queue = curl_multi_init(); - // enable pipelining - curl_multi_setopt($this->queue, \CURLMOPT_PIPELINING, 1); - } - - public function queue(int ...$feeds): int { - foreach ($feeds as $id) { - $h = curl_init(); - curl_setopt($h, \CURLOPT_POSTFIELDS, json_encode(['userId' => "", 'feedId' => $id])); - $this->handles[] = $h; - curl_multi_add_handle($this->queue, $h); - } - return sizeof($this->handles); - } - - public function exec(): int { - $active = 0; - do { - curl_multi_exec($this->queue, $active); - curl_multi_select($this->queue); - } while ($active > 0); - return Arsse::$conf->serviceQueueWidth - $active; - } - - public function clean(): bool { - foreach ($this->handles as $h) { - curl_multi_remove_handle($this->queue, $h); - curl_close($h); - } - $this->handles = []; - return true; - } -} diff --git a/locale/en.php b/locale/en.php index dcaa848..4c645e1 100644 --- a/locale/en.php +++ b/locale/en.php @@ -27,7 +27,6 @@ return [ 'Driver.Service.Serial.Name' => 'Serialized', 'Driver.Service.Subprocess.Name' => 'Concurrent subprocess', - 'Driver.Service.Curl.Name' => 'Concurrent HTTP (curl)', 'Driver.User.Internal.Name' => 'Internal', From a5049ac6466653b9cbd49c2dfefbf382014222c6 Mon Sep 17 00:00:00 2001 From: "J. King" Date: Wed, 23 Jan 2019 09:21:35 -0500 Subject: [PATCH 05/11] Remove reference to PicoFeed in the User-Agent string PicoFeed is dead, so there's no point. --- lib/Feed.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Feed.php b/lib/Feed.php index 5a548d8..aaa048b 100644 --- a/lib/Feed.php +++ b/lib/Feed.php @@ -78,7 +78,7 @@ class Feed { protected static function configure(): Config { $userAgent = Arsse::$conf->fetchUserAgentString ?? sprintf( - 'Arsse/%s (%s %s; %s; https://thearsse.com/) PicoFeed (https://github.com/miniflux/picoFeed)', + 'Arsse/%s (%s %s; %s; https://thearsse.com/)', Arsse::VERSION, // Arsse version php_uname('s'), // OS php_uname('r'), // OS version From 9120d3b3e3625ae786d92dae61fa87002f6cc0cb Mon Sep 17 00:00:00 2001 From: "J. King" Date: Wed, 23 Jan 2019 09:32:44 -0500 Subject: [PATCH 06/11] Correctly escape shell command in subprocesds service driver --- lib/Service/Subprocess/Driver.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/Service/Subprocess/Driver.php b/lib/Service/Subprocess/Driver.php index a657232..5e79ed0 100644 --- a/lib/Service/Subprocess/Driver.php +++ b/lib/Service/Subprocess/Driver.php @@ -31,8 +31,8 @@ class Driver implements \JKingWeb\Arsse\Service\Driver { $pp = []; while ($this->queue) { $id = (int) array_shift($this->queue); - $php = '"'.\PHP_BINARY.'"'; - $arsse = '"'.$_SERVER['argv'][0].'"'; + $php = escapeshellarg(\PHP_BINARY); + $arsse = escapeshellarg($_SERVER['argv'][0]); array_push($pp, popen("$php $arsse feed refresh $id", "r")); } while ($pp) { From 970731073d4175763529098fb165d8003cda01ec Mon Sep 17 00:00:00 2001 From: "J. King" Date: Wed, 23 Jan 2019 09:37:41 -0500 Subject: [PATCH 07/11] Fetch timeout should be a float, not an integer --- lib/Conf.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Conf.php b/lib/Conf.php index ce092e9..eee9129 100644 --- a/lib/Conf.php +++ b/lib/Conf.php @@ -84,7 +84,7 @@ class Conf { public $serviceQueueWidth = 5; /** @var \DateInterval Number of seconds to wait for data when fetching feeds from foreign servers */ - public $fetchTimeout = 10; + public $fetchTimeout = 10.0; /** @var integer Maximum size, in bytes, of data when fetching feeds from foreign servers */ public $fetchSizeLimit = 2 * 1024 * 1024; /** @var boolean Whether to allow the possibility of fetching full article contents using an item's URL. Whether fetching will actually happen is also governed by a per-feed setting */ From bc8d443d84589720b462188857f1abf70424807f Mon Sep 17 00:00:00 2001 From: "J. King" Date: Wed, 23 Jan 2019 12:36:43 -0500 Subject: [PATCH 08/11] Change PicoFeed dependency to a maintained variant --- composer.json | 2 +- composer.lock | 124 +++++++++++++++---------------- vendor-bin/csfixer/composer.lock | 111 ++++++++++++++------------- 3 files changed, 121 insertions(+), 116 deletions(-) diff --git a/composer.json b/composer.json index 5f943d9..0f5570c 100644 --- a/composer.json +++ b/composer.json @@ -22,7 +22,7 @@ "ext-intl": "*", "ext-json": "*", "ext-hash": "*", - "fguillot/picofeed": ">=0.1.31", + "p3k/picofeed": "0.1.*", "hosteurope/password-generator": "^1.0", "docopt/docopt": "^1.0", "jkingweb/druuid": "^3.0", diff --git a/composer.lock b/composer.lock index 30c0c79..b5e5d38 100644 --- a/composer.lock +++ b/composer.lock @@ -1,10 +1,10 @@ { "_readme": [ "This file locks the dependencies of your project to a known state", - "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", + "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "7d381fa958169b7079c1d3c5b911f3bd", + "content-hash": "d7a6a00be3d97c11d09ec4d4e56d36e0", "packages": [ { "name": "docopt/docopt", @@ -52,59 +52,6 @@ ], "time": "2015-10-30T03:21:23+00:00" }, - { - "name": "fguillot/picofeed", - "version": "v0.1.37", - "source": { - "type": "git", - "url": "https://github.com/miniflux/picoFeed.git", - "reference": "402b7f07629577e7929625e78bc88d3d5831a22d" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/miniflux/picoFeed/zipball/402b7f07629577e7929625e78bc88d3d5831a22d", - "reference": "402b7f07629577e7929625e78bc88d3d5831a22d", - "shasum": "" - }, - "require": { - "ext-dom": "*", - "ext-iconv": "*", - "ext-libxml": "*", - "ext-simplexml": "*", - "ext-xml": "*", - "php": ">=5.3.0", - "zendframework/zendxml": "^1.0" - }, - "require-dev": { - "phpdocumentor/reflection-docblock": "2.0.4", - "phpunit/phpunit": "4.8.26", - "symfony/yaml": "2.8.7" - }, - "suggest": { - "ext-curl": "PicoFeed will use cURL if present" - }, - "bin": [ - "picofeed" - ], - "type": "library", - "autoload": { - "psr-0": { - "PicoFeed": "lib/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Frédéric Guillot" - } - ], - "description": "Modern library to handle RSS/Atom feeds", - "homepage": "https://github.com/miniflux/picoFeed", - "time": "2017-11-02T03:20:36+00:00" - }, { "name": "hosteurope/password-generator", "version": "v1.0.1", @@ -190,6 +137,59 @@ ], "time": "2017-02-09T14:17:01+00:00" }, + { + "name": "p3k/picofeed", + "version": "v0.1.38", + "source": { + "type": "git", + "url": "https://github.com/aaronpk/picoFeed.git", + "reference": "989c0bcf2eac016a4104abce1aadff791fc287ab" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/aaronpk/picoFeed/zipball/989c0bcf2eac016a4104abce1aadff791fc287ab", + "reference": "989c0bcf2eac016a4104abce1aadff791fc287ab", + "shasum": "" + }, + "require": { + "ext-dom": "*", + "ext-iconv": "*", + "ext-libxml": "*", + "ext-simplexml": "*", + "ext-xml": "*", + "php": ">=5.3.0", + "zendframework/zendxml": "^1.0" + }, + "require-dev": { + "phpdocumentor/reflection-docblock": "2.0.4", + "phpunit/phpunit": "4.8.26", + "symfony/yaml": "2.8.7" + }, + "suggest": { + "ext-curl": "PicoFeed will use cURL if present" + }, + "bin": [ + "picofeed" + ], + "type": "library", + "autoload": { + "psr-0": { + "PicoFeed": "lib/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Frédéric Guillot" + } + ], + "description": "Modern library to handle RSS/Atom feeds", + "homepage": "https://github.com/miniflux/picoFeed", + "time": "2017-11-30T00:16:58+00:00" + }, { "name": "psr/http-message", "version": "1.0.1", @@ -306,16 +306,16 @@ }, { "name": "zendframework/zendxml", - "version": "1.1.0", + "version": "1.2.0", "source": { "type": "git", "url": "https://github.com/zendframework/ZendXml.git", - "reference": "267db6a2c431a08a8f8ff0f1f4c302a5ba6f5b99" + "reference": "eceab37a591c9e140772a1470338258857339e00" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/zendframework/ZendXml/zipball/267db6a2c431a08a8f8ff0f1f4c302a5ba6f5b99", - "reference": "267db6a2c431a08a8f8ff0f1f4c302a5ba6f5b99", + "url": "https://api.github.com/repos/zendframework/ZendXml/zipball/eceab37a591c9e140772a1470338258857339e00", + "reference": "eceab37a591c9e140772a1470338258857339e00", "shasum": "" }, "require": { @@ -328,8 +328,8 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "1.1.x-dev", - "dev-develop": "1.2.x-dev" + "dev-master": "1.2.x-dev", + "dev-develop": "1.3.x-dev" } }, "autoload": { @@ -348,7 +348,7 @@ "xml", "zf" ], - "time": "2018-04-30T15:11:04+00:00" + "time": "2019-01-22T19:42:14+00:00" } ], "packages-dev": [ diff --git a/vendor-bin/csfixer/composer.lock b/vendor-bin/csfixer/composer.lock index 9a89268..fe1d67e 100644 --- a/vendor-bin/csfixer/composer.lock +++ b/vendor-bin/csfixer/composer.lock @@ -1,7 +1,7 @@ { "_readme": [ "This file locks the dependencies of your project to a known state", - "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", + "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], "content-hash": "3fb53f1e3b4dd21c4d1e88c3ffd570ca", @@ -70,16 +70,16 @@ }, { "name": "composer/xdebug-handler", - "version": "1.3.0", + "version": "1.3.1", "source": { "type": "git", "url": "https://github.com/composer/xdebug-handler.git", - "reference": "b8e9745fb9b06ea6664d8872c4505fb16df4611c" + "reference": "dc523135366eb68f22268d069ea7749486458562" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/xdebug-handler/zipball/b8e9745fb9b06ea6664d8872c4505fb16df4611c", - "reference": "b8e9745fb9b06ea6664d8872c4505fb16df4611c", + "url": "https://api.github.com/repos/composer/xdebug-handler/zipball/dc523135366eb68f22268d069ea7749486458562", + "reference": "dc523135366eb68f22268d069ea7749486458562", "shasum": "" }, "require": { @@ -110,7 +110,7 @@ "Xdebug", "performance" ], - "time": "2018-08-31T19:07:57+00:00" + "time": "2018-11-29T10:59:02+00:00" }, { "name": "doctrine/annotations", @@ -236,16 +236,16 @@ }, { "name": "friendsofphp/php-cs-fixer", - "version": "v2.13.1", + "version": "v2.14.0", "source": { "type": "git", "url": "https://github.com/FriendsOfPHP/PHP-CS-Fixer.git", - "reference": "54814c62d5beef3ba55297b9b3186ed8b8a1b161" + "reference": "b788ea0af899cedc8114dca7db119c93b6685da2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/FriendsOfPHP/PHP-CS-Fixer/zipball/54814c62d5beef3ba55297b9b3186ed8b8a1b161", - "reference": "54814c62d5beef3ba55297b9b3186ed8b8a1b161", + "url": "https://api.github.com/repos/FriendsOfPHP/PHP-CS-Fixer/zipball/b788ea0af899cedc8114dca7db119c93b6685da2", + "reference": "b788ea0af899cedc8114dca7db119c93b6685da2", "shasum": "" }, "require": { @@ -254,7 +254,7 @@ "doctrine/annotations": "^1.2", "ext-json": "*", "ext-tokenizer": "*", - "php": "^5.6 || >=7.0 <7.3", + "php": "^5.6 || ^7.0", "php-cs-fixer/diff": "^1.3", "symfony/console": "^3.4.17 || ^4.1.6", "symfony/event-dispatcher": "^3.0 || ^4.0", @@ -272,7 +272,7 @@ "require-dev": { "johnkary/phpunit-speedtrap": "^1.1 || ^2.0 || ^3.0", "justinrainbow/json-schema": "^5.0", - "keradus/cli-executor": "^1.1", + "keradus/cli-executor": "^1.2", "mikey179/vfsstream": "^1.6", "php-coveralls/php-coveralls": "^2.1", "php-cs-fixer/accessible-object": "^1.0", @@ -292,6 +292,11 @@ "php-cs-fixer" ], "type": "application", + "extra": { + "branch-alias": { + "dev-master": "2.14-dev" + } + }, "autoload": { "psr-4": { "PhpCsFixer\\": "src/" @@ -323,7 +328,7 @@ } ], "description": "A tool to automatically fix PHP code style", - "time": "2018-10-21T00:32:10+00:00" + "time": "2019-01-04T18:29:47+00:00" }, { "name": "paragonie/random_compat", @@ -470,16 +475,16 @@ }, { "name": "symfony/console", - "version": "v3.4.18", + "version": "v3.4.21", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "1d228fb4602047d7b26a0554e0d3efd567da5803" + "reference": "a700b874d3692bc8342199adfb6d3b99f62cc61a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/1d228fb4602047d7b26a0554e0d3efd567da5803", - "reference": "1d228fb4602047d7b26a0554e0d3efd567da5803", + "url": "https://api.github.com/repos/symfony/console/zipball/a700b874d3692bc8342199adfb6d3b99f62cc61a", + "reference": "a700b874d3692bc8342199adfb6d3b99f62cc61a", "shasum": "" }, "require": { @@ -535,20 +540,20 @@ ], "description": "Symfony Console Component", "homepage": "https://symfony.com", - "time": "2018-10-30T16:50:50+00:00" + "time": "2019-01-04T04:42:43+00:00" }, { "name": "symfony/debug", - "version": "v3.4.18", + "version": "v3.4.21", "source": { "type": "git", "url": "https://github.com/symfony/debug.git", - "reference": "fe9793af008b651c5441bdeab21ede8172dab097" + "reference": "26d7f23b9bd0b93bee5583e4d6ca5cb1ab31b186" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/debug/zipball/fe9793af008b651c5441bdeab21ede8172dab097", - "reference": "fe9793af008b651c5441bdeab21ede8172dab097", + "url": "https://api.github.com/repos/symfony/debug/zipball/26d7f23b9bd0b93bee5583e4d6ca5cb1ab31b186", + "reference": "26d7f23b9bd0b93bee5583e4d6ca5cb1ab31b186", "shasum": "" }, "require": { @@ -591,20 +596,20 @@ ], "description": "Symfony Debug Component", "homepage": "https://symfony.com", - "time": "2018-10-31T09:06:03+00:00" + "time": "2019-01-01T13:45:19+00:00" }, { "name": "symfony/event-dispatcher", - "version": "v3.4.18", + "version": "v3.4.21", "source": { "type": "git", "url": "https://github.com/symfony/event-dispatcher.git", - "reference": "db9e829c8f34c3d35cf37fcd4cdb4293bc4a2f14" + "reference": "d1cdd46c53c264a2bd42505bd0e8ce21423bd0e2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/event-dispatcher/zipball/db9e829c8f34c3d35cf37fcd4cdb4293bc4a2f14", - "reference": "db9e829c8f34c3d35cf37fcd4cdb4293bc4a2f14", + "url": "https://api.github.com/repos/symfony/event-dispatcher/zipball/d1cdd46c53c264a2bd42505bd0e8ce21423bd0e2", + "reference": "d1cdd46c53c264a2bd42505bd0e8ce21423bd0e2", "shasum": "" }, "require": { @@ -654,20 +659,20 @@ ], "description": "Symfony EventDispatcher Component", "homepage": "https://symfony.com", - "time": "2018-10-30T16:50:50+00:00" + "time": "2019-01-01T18:08:36+00:00" }, { "name": "symfony/filesystem", - "version": "v3.4.18", + "version": "v3.4.21", "source": { "type": "git", "url": "https://github.com/symfony/filesystem.git", - "reference": "d69930fc337d767607267d57c20a7403d0a822a4" + "reference": "c24ce3d18ccc9bb9d7e1d6ce9330fcc6061cafde" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/filesystem/zipball/d69930fc337d767607267d57c20a7403d0a822a4", - "reference": "d69930fc337d767607267d57c20a7403d0a822a4", + "url": "https://api.github.com/repos/symfony/filesystem/zipball/c24ce3d18ccc9bb9d7e1d6ce9330fcc6061cafde", + "reference": "c24ce3d18ccc9bb9d7e1d6ce9330fcc6061cafde", "shasum": "" }, "require": { @@ -704,20 +709,20 @@ ], "description": "Symfony Filesystem Component", "homepage": "https://symfony.com", - "time": "2018-10-02T12:28:39+00:00" + "time": "2019-01-01T13:45:19+00:00" }, { "name": "symfony/finder", - "version": "v3.4.18", + "version": "v3.4.21", "source": { "type": "git", "url": "https://github.com/symfony/finder.git", - "reference": "54ba444dddc5bd5708a34bd095ea67c6eb54644d" + "reference": "3f2a2ab6315dd7682d4c16dcae1e7b95c8b8555e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/finder/zipball/54ba444dddc5bd5708a34bd095ea67c6eb54644d", - "reference": "54ba444dddc5bd5708a34bd095ea67c6eb54644d", + "url": "https://api.github.com/repos/symfony/finder/zipball/3f2a2ab6315dd7682d4c16dcae1e7b95c8b8555e", + "reference": "3f2a2ab6315dd7682d4c16dcae1e7b95c8b8555e", "shasum": "" }, "require": { @@ -753,20 +758,20 @@ ], "description": "Symfony Finder Component", "homepage": "https://symfony.com", - "time": "2018-10-03T08:46:40+00:00" + "time": "2019-01-01T13:45:19+00:00" }, { "name": "symfony/options-resolver", - "version": "v3.4.18", + "version": "v3.4.21", "source": { "type": "git", "url": "https://github.com/symfony/options-resolver.git", - "reference": "1cf7d8e704a9cc4164c92e430f2dfa3e6983661d" + "reference": "8a10e36ffd04c0c551051594952304d34ecece71" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/options-resolver/zipball/1cf7d8e704a9cc4164c92e430f2dfa3e6983661d", - "reference": "1cf7d8e704a9cc4164c92e430f2dfa3e6983661d", + "url": "https://api.github.com/repos/symfony/options-resolver/zipball/8a10e36ffd04c0c551051594952304d34ecece71", + "reference": "8a10e36ffd04c0c551051594952304d34ecece71", "shasum": "" }, "require": { @@ -807,7 +812,7 @@ "configuration", "options" ], - "time": "2018-09-17T17:29:18+00:00" + "time": "2019-01-01T13:45:19+00:00" }, { "name": "symfony/polyfill-ctype", @@ -1042,16 +1047,16 @@ }, { "name": "symfony/process", - "version": "v3.4.18", + "version": "v3.4.21", "source": { "type": "git", "url": "https://github.com/symfony/process.git", - "reference": "35c2914a9f50519bd207164c353ae4d59182c2cb" + "reference": "0d41dd7d95ed179aed6a13393b0f4f97bfa2d25c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/process/zipball/35c2914a9f50519bd207164c353ae4d59182c2cb", - "reference": "35c2914a9f50519bd207164c353ae4d59182c2cb", + "url": "https://api.github.com/repos/symfony/process/zipball/0d41dd7d95ed179aed6a13393b0f4f97bfa2d25c", + "reference": "0d41dd7d95ed179aed6a13393b0f4f97bfa2d25c", "shasum": "" }, "require": { @@ -1087,20 +1092,20 @@ ], "description": "Symfony Process Component", "homepage": "https://symfony.com", - "time": "2018-10-14T17:33:21+00:00" + "time": "2019-01-02T21:24:08+00:00" }, { "name": "symfony/stopwatch", - "version": "v3.4.18", + "version": "v3.4.21", "source": { "type": "git", "url": "https://github.com/symfony/stopwatch.git", - "reference": "05e52a39de52ba690aebaed462b2bc8a9649f0a4" + "reference": "af55d31cb58c5452d2c160655fa1968b872a8084" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/stopwatch/zipball/05e52a39de52ba690aebaed462b2bc8a9649f0a4", - "reference": "05e52a39de52ba690aebaed462b2bc8a9649f0a4", + "url": "https://api.github.com/repos/symfony/stopwatch/zipball/af55d31cb58c5452d2c160655fa1968b872a8084", + "reference": "af55d31cb58c5452d2c160655fa1968b872a8084", "shasum": "" }, "require": { @@ -1136,7 +1141,7 @@ ], "description": "Symfony Stopwatch Component", "homepage": "https://symfony.com", - "time": "2018-10-02T12:28:39+00:00" + "time": "2019-01-01T13:45:19+00:00" } ], "packages-dev": [], From 8ea1df920a73619617ba2ec8b19cc29dbccd3bad Mon Sep 17 00:00:00 2001 From: "J. King" Date: Wed, 23 Jan 2019 16:31:54 -0500 Subject: [PATCH 09/11] Unify SQL timeouts - Exec and lock timeouts now apply to MySQL - Lock timeout now applies to PostgreSQL - SQLite now uses a generic lock timeout setting which applies to all --- lib/AbstractException.php | 1 + lib/Conf.php | 38 +++++++++++++++++++++++------------ lib/Db/MySQL/Driver.php | 15 ++++++++++---- lib/Db/PostgreSQL/Driver.php | 6 ++++-- lib/Db/SQLite3/Driver.php | 3 ++- locale/en.php | 2 ++ tests/cases/Db/BaseDriver.php | 1 + 7 files changed, 46 insertions(+), 20 deletions(-) diff --git a/lib/AbstractException.php b/lib/AbstractException.php index cd5fcc8..0249678 100644 --- a/lib/AbstractException.php +++ b/lib/AbstractException.php @@ -65,6 +65,7 @@ abstract class AbstractException extends \Exception { "Conf/Exception.fileCorrupt" => 10306, "Conf/Exception.typeMismatch" => 10311, "Conf/Exception.semanticMismatch" => 10312, + "Conf/Exception.ambiguousDefault" => 10313, "User/Exception.functionNotImplemented" => 10401, "User/Exception.doesNotExist" => 10402, "User/Exception.alreadyExists" => 10403, diff --git a/lib/Conf.php b/lib/Conf.php index eee9129..bba5821 100644 --- a/lib/Conf.php +++ b/lib/Conf.php @@ -21,16 +21,16 @@ class Conf { public $dbDriver = "sqlite3"; /** @var boolean Whether to attempt to automatically update the database when upgrading to a new version with schema changes */ public $dbAutoUpdate = true; - /** @var \DateInterval Number of seconds to wait before returning a timeout error when connecting to a database (zero waits forever; not applicable to SQLite) */ + /** @var \DateInterval|null Number of seconds to wait before returning a timeout error when connecting to a database (null waits forever; not applicable to SQLite) */ public $dbTimeoutConnect = 5.0; - /** @var \DateInterval Number of seconds to wait before returning a timeout error when executing a database operation (zero waits forever; not applicable to SQLite) */ - public $dbTimeoutExec = 0.0; + /** @var \DateInterval|null Number of seconds to wait before returning a timeout error when executing a database operation (null waits forever; not applicable to SQLite) */ + public $dbTimeoutExec = null; + /** @var \DateInterval|null Number of seconds to wait before returning a timeout error when acquiring a database lock (null waits forever) */ + public $dbTimeoutLock = 60.0; /** @var string|null Full path and file name of SQLite database (if using SQLite) */ public $dbSQLite3File = null; /** @var string Encryption key to use for SQLite database (if using a version of SQLite with SEE) */ public $dbSQLite3Key = ""; - /** @var \DateInterval Number of seconds for SQLite to wait before returning a timeout error when trying to acquire a write lock on the database (zero does not wait) */ - public $dbSQLite3Timeout = 60.0; /** @var string Host name, address, or socket path of PostgreSQL database server (if using PostgreSQL) */ public $dbPostgreSQLHost = ""; /** @var string Log-in user name for PostgreSQL database server (if using PostgreSQL) */ @@ -109,6 +109,11 @@ class Conf { /** @var string Space-separated list of origins from which to deny cross-origin resource sharing */ public $httpOriginsDenied = ""; + ### OBSOLETE SETTINGS + + /** @var \DateInterval|null (OBSOLETE) Number of seconds for SQLite to wait before returning a timeout error when trying to acquire a write lock on the database (zero does not wait) */ + public $dbSQLite3Timeout = null; // previously 60.0 + const TYPE_NAMES = [ Value::T_BOOL => "boolean", Value::T_STRING => "string", @@ -116,6 +121,12 @@ class Conf { VALUE::T_INT => "integer", Value::T_INTERVAL => "interval", ]; + const EXPECTED_TYPES = [ + 'dbTimeoutExec' => "double", + 'dbTimeoutLock' => "double", + 'dbTimeoutConnect' => "double", + 'dbSQLite3Timeout' => "double", + ]; protected static $types = []; @@ -261,26 +272,28 @@ class Conf { } protected function propertyImport(string $key, $value, string $file = "") { + $typeName = static::$types[$key]['name'] ?? "mixed"; + $typeConst = static::$types[$key]['const'] ?? Value::T_MIXED; + $nullable = (int) (bool) (static::$types[$key]['const'] & Value::M_NULL); try { - $typeName = static::$types[$key]['name'] ?? "mixed"; - $typeConst = static::$types[$key]['const'] ?? Value::T_MIXED; if ($typeName === "\\DateInterval") { // date intervals have special handling: if the existing value (ultimately, the default value) // is an integer or float, the new value should be imported as numeric. If the new value is a string // it is first converted to an interval and then converted to the numeric type if necessary + $mode = $nullable ? Value::M_STRICT | Value::M_NULL : Value::M_STRICT; if (is_string($value)) { - $value = Value::normalize($value, Value::T_INTERVAL | Value::M_STRICT); + $value = Value::normalize($value, Value::T_INTERVAL | $mode); } - switch (gettype($this->$key)) { + switch (self::EXPECTED_TYPES[$key] ?? gettype($this->$key)) { case "integer": - return Value::normalize($value, Value::T_INT | Value::M_STRICT); + return Value::normalize($value, Value::T_INT | $mode); case "double": - return Value::normalize($value, Value::T_FLOAT | Value::M_STRICT); + return Value::normalize($value, Value::T_FLOAT | $mode); case "string": case "object": return $value; default: - throw new ExceptionType("strictFailure"); // @codeCoverageIgnore + throw new Conf\Exception("ambiguousDefault", ['param' => $key]); // @codeCoverageIgnore } } $value = Value::normalize($value, $typeConst); @@ -303,7 +316,6 @@ class Conf { } return $value; } catch (ExceptionType $e) { - $nullable = (int) (bool) (static::$types[$key] & Value::M_NULL); $type = static::$types[$key]['const'] & ~(Value::M_STRICT | Value::M_DROP | Value::M_NULL | Value::M_ARRAY); throw new Conf\Exception("typeMismatch", ['param' => $key, 'type' => self::TYPE_NAMES[$type], 'file' => $file, 'nullable' => $nullable]); } diff --git a/lib/Db/MySQL/Driver.php b/lib/Db/MySQL/Driver.php index e054800..8a4fe44 100644 --- a/lib/Db/MySQL/Driver.php +++ b/lib/Db/MySQL/Driver.php @@ -18,7 +18,7 @@ class Driver extends \JKingWeb\Arsse\Db\AbstractDriver { const SQL_MODE = "ANSI_QUOTES,HIGH_NOT_PRECEDENCE,NO_BACKSLASH_ESCAPES,NO_ENGINE_SUBSTITUTION,PIPES_AS_CONCAT,STRICT_ALL_TABLES"; const TRANSACTIONAL_LOCKS = false; - /** @var \mysql */ + /** @var \mysqli */ protected $db; protected $transStart = 0; protected $packetSize = 4194304; @@ -48,7 +48,8 @@ class Driver extends \JKingWeb\Arsse\Db\AbstractDriver { return [ "SET sql_mode = '".self::SQL_MODE."'", "SET time_zone = '+00:00'", - "SET lock_wait_timeout = 1", + "SET lock_wait_timeout = ".self::lockTimeout(), + "SET max_execution_time = ".ceil(Arsse::$conf->dbTimeoutExec * 1000), ]; } @@ -130,7 +131,7 @@ class Driver extends \JKingWeb\Arsse\Db\AbstractDriver { try { $this->exec("SET lock_wait_timeout = 1; LOCK TABLES $tables"); } finally { - $this->exec("SET lock_wait_timeout = 60"); + $this->exec("SET lock_wait_timeout = ".self::lockTimeout()); } } return true; @@ -141,6 +142,10 @@ class Driver extends \JKingWeb\Arsse\Db\AbstractDriver { return true; } + protected static function lockTimeout(): int { + return (int) max(min(ceil(Arsse::$conf->dbTimeoutLock ?? 31536000), 31536000), 1); + } + public function __destruct() { if (isset($this->db)) { $this->db->close(); @@ -157,7 +162,9 @@ class Driver extends \JKingWeb\Arsse\Db\AbstractDriver { } protected function makeConnection(string $db, string $user, string $password, string $host, int $port, string $socket) { - $this->db = @new \mysqli($host, $user, $password, $db, $port, $socket); + $this->db = mysqli_init(); + $this->db->options(\MYSQLI_OPT_CONNECT_TIMEOUT, ceil(Arsse::$conf->dbTimeoutConnect)); + @$this->db->real_connect($host, $user, $password, $db, $port, $socket); if ($this->db->connect_errno) { list($excClass, $excMsg, $excData) = $this->buildConnectionException($this->db->connect_errno, $this->db->connect_error); throw new $excClass($excMsg, $excData); diff --git a/lib/Db/PostgreSQL/Driver.php b/lib/Db/PostgreSQL/Driver.php index 8886546..513ce99 100644 --- a/lib/Db/PostgreSQL/Driver.php +++ b/lib/Db/PostgreSQL/Driver.php @@ -74,11 +74,13 @@ class Driver extends \JKingWeb\Arsse\Db\AbstractDriver { } public static function makeSetupQueries(string $schema = ""): array { - $timeout = ceil(Arsse::$conf->dbTimeoutExec * 1000); + $timeExec = is_null(Arsse::$conf->dbTimeoutExec) ? 0 : ceil(max(Arsse::$conf->dbTimeoutExec * 1000, 1)); + $timeLock = is_null(Arsse::$conf->dbTimeoutLock) ? 0 : ceil(max(Arsse::$conf->dbTimeoutLock * 1000, 1)); $out = [ "SET TIME ZONE UTC", "SET DateStyle = 'ISO, MDY'", - "SET statement_timeout = '$timeout'", + "SET statement_timeout = '$timeExec'", + "SET lock_timeout = '$timeLock'", ]; if (strlen($schema) > 0) { $schema = '"'.str_replace('"', '""', $schema).'"'; diff --git a/lib/Db/SQLite3/Driver.php b/lib/Db/SQLite3/Driver.php index 2e741b5..f7e47fb 100644 --- a/lib/Db/SQLite3/Driver.php +++ b/lib/Db/SQLite3/Driver.php @@ -55,7 +55,8 @@ class Driver extends \JKingWeb\Arsse\Db\AbstractDriver { throw new Exception("fileCorrupt", $dbFile); } // set the timeout - $timeout = (int) ceil(Arsse::$conf->dbSQLite3Timeout * 1000); + $timeout = Arsse::$conf->dbSQLite3Timeout ?? Arsse::$conf->dbTimeoutLock; // old SQLite-specific timeout takes precedence + $timeout = is_null($timeout) ? PHP_INT_MAX : (int) ceil($timeout * 1000); $this->setTimeout($timeout); // set other initial options $this->exec("PRAGMA foreign_keys = yes"); diff --git a/locale/en.php b/locale/en.php index 4c645e1..f576442 100644 --- a/locale/en.php +++ b/locale/en.php @@ -74,6 +74,8 @@ return [ other {, or null} }', 'Exception.JKingWeb/Arsse/Conf/Exception.semanticMismatch' => 'Configuration parameter "{param}" in file "{file}" is not a valid value. Consult the documentation for possible values', + // indicates programming error + 'Exception.JKingWeb/Arsse/Conf/Exception.ambiguousDefault' => 'Preferred type of configuration parameter "{param}" could not be inferred from its default value. The parameter must be added to the Conf::EXPECTED_TYPES array', 'Exception.JKingWeb/Arsse/Db/Exception.extMissing' => 'Required PHP extension for driver "{0}" not installed', 'Exception.JKingWeb/Arsse/Db/Exception.fileMissing' => 'Database file "{0}" does not exist', 'Exception.JKingWeb/Arsse/Db/Exception.fileUnreadable' => 'Insufficient permissions to open database file "{0}" for reading', diff --git a/tests/cases/Db/BaseDriver.php b/tests/cases/Db/BaseDriver.php index dcc8b34..682c688 100644 --- a/tests/cases/Db/BaseDriver.php +++ b/tests/cases/Db/BaseDriver.php @@ -19,6 +19,7 @@ abstract class BaseDriver extends \JKingWeb\Arsse\Test\AbstractTest { protected $setVersion; protected static $conf = [ 'dbTimeoutExec' => 0.5, + 'dbTimeoutLock' => 0.001, 'dbSQLite3Timeout' => 0, //'dbSQLite3File' => "(temporary file)", ]; From 91b6fdc696a845417c47bfb87085665f521ebab2 Mon Sep 17 00:00:00 2001 From: "J. King" Date: Wed, 23 Jan 2019 16:32:48 -0500 Subject: [PATCH 10/11] Update changelog; bump version --- CHANGELOG | 11 +++++++++++ lib/Arsse.php | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index ee6bce2..0254409 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,14 @@ +Version 0.6.1 (2019-01-23) + +Bug Fixes: +- Unify SQL timeout settings +- Correctly escape shell command in subprocess service driver +- Correctly allow null time intervals in configuration when appropriate + +Changes: +- Change PicoFeed dependency to maintained version (Thanks, Aaron Parecki!) +- Remove non-functional cURL service driver + Version 0.6.0 (2019-01-21) ========================== diff --git a/lib/Arsse.php b/lib/Arsse.php index 58b843d..7fbd1b2 100644 --- a/lib/Arsse.php +++ b/lib/Arsse.php @@ -7,7 +7,7 @@ declare(strict_types=1); namespace JKingWeb\Arsse; class Arsse { - const VERSION = "0.6.0"; + const VERSION = "0.6.1"; /** @var Lang */ public static $lang; From 500851f161a14238472a1ad2525230de838b210a Mon Sep 17 00:00:00 2001 From: "J. King" Date: Wed, 23 Jan 2019 16:34:54 -0500 Subject: [PATCH 11/11] Style fixes --- CHANGELOG | 1 + lib/CLI.php | 2 +- lib/Db/MySQL/Statement.php | 4 ++-- lib/Db/PostgreSQL/Statement.php | 2 +- lib/Misc/ValueInfo.php | 2 +- tests/cases/Misc/TestValueInfo.php | 4 ++-- 6 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 0254409..10832aa 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,4 +1,5 @@ Version 0.6.1 (2019-01-23) +========================== Bug Fixes: - Unify SQL timeout settings diff --git a/lib/CLI.php b/lib/CLI.php index 36129e0..8ff1e1d 100644 --- a/lib/CLI.php +++ b/lib/CLI.php @@ -88,7 +88,7 @@ USAGE_TEXT; /** @codeCoverageIgnore */ protected function logError(string $msg) { - fwrite(STDERR,$msg.\PHP_EOL); + fwrite(STDERR, $msg.\PHP_EOL); } /** @codeCoverageIgnore */ diff --git a/lib/Db/MySQL/Statement.php b/lib/Db/MySQL/Statement.php index 8af64f4..9612615 100644 --- a/lib/Db/MySQL/Statement.php +++ b/lib/Db/MySQL/Statement.php @@ -94,8 +94,8 @@ class Statement extends \JKingWeb\Arsse\Db\AbstractStatement { } protected function bindValue($value, string $type, int $position): bool { - // this is a bit of a hack: we collect values (and MySQL bind types) here so that we can take - // advantage of the work done by bindValues() even though MySQL requires everything to be bound + // this is a bit of a hack: we collect values (and MySQL bind types) here so that we can take + // advantage of the work done by bindValues() even though MySQL requires everything to be bound // all at once; we also segregate large values for later packetization if (($type === "binary" && !is_null($value)) || (is_string($value) && strlen($value) > $this->packetSize)) { $this->values[] = null; diff --git a/lib/Db/PostgreSQL/Statement.php b/lib/Db/PostgreSQL/Statement.php index 8d6668f..df74e3d 100644 --- a/lib/Db/PostgreSQL/Statement.php +++ b/lib/Db/PostgreSQL/Statement.php @@ -29,7 +29,7 @@ class Statement extends \JKingWeb\Arsse\Db\AbstractStatement { protected $bindings; public function __construct($db, string $query, array $bindings = []) { - $this->db = $db; + $this->db = $db; $this->query = $query; $this->retypeArray($bindings); } diff --git a/lib/Misc/ValueInfo.php b/lib/Misc/ValueInfo.php index 3ce76e7..752704d 100644 --- a/lib/Misc/ValueInfo.php +++ b/lib/Misc/ValueInfo.php @@ -352,7 +352,7 @@ class ValueInfo { // input is a number, assume this is a number of seconds // for legibility we convert large numbers to minutes, hours, and days as necessary // the DateInterval constructor only allows 12 digits for any given part of an interval, - // so we also convert days to 365-day years where we must, and cap the number of years + // so we also convert days to 365-day years where we must, and cap the number of years // at (1e11 - 1); this being a very large number, the loss of precision is probably not // significant in practical usage $sec = abs($value); diff --git a/tests/cases/Misc/TestValueInfo.php b/tests/cases/Misc/TestValueInfo.php index 75a48bc..b6a34a6 100644 --- a/tests/cases/Misc/TestValueInfo.php +++ b/tests/cases/Misc/TestValueInfo.php @@ -457,7 +457,7 @@ class TestValueInfo extends \JKingWeb\Arsse\Test\AbstractTest { */ /* Input value null bool int float string array interval */ [null, [null,true], [false,false], [0, false], [0.0, false], ["", false], [[], false], [null, false]], - ["", [null,true], [false,true], [0, false], [0.0, false], ["", true], [[""], false], [null, false]], + ["", [null,true], [false,true], [0, false], [0.0, false], ["", true], [[""], false], [null, false]], [1, [null,true], [true, true], [1, true], [1.0, true], ["1", true], [[1], false], [$this->i("PT1S"), false]], [PHP_INT_MAX, [null,true], [true, false], [PHP_INT_MAX, true], [(float) PHP_INT_MAX,true], [(string) PHP_INT_MAX, true], [[PHP_INT_MAX], false], [$this->i("P292471208677Y195DT15H30M7S"), false]], [1.0, [null,true], [true, true], [1, true], [1.0, true], ["1", true], [[1.0], false], [$this->i("PT1S"), false]], @@ -571,7 +571,7 @@ class TestValueInfo extends \JKingWeb\Arsse\Test\AbstractTest { "!M j, Y (D)", null, ]; - foreach([ + foreach ([ /* Input value microtime iso8601 iso8601m http sql date time unix float '!M j, Y (D)' *strtotime* (null) */ [null, null, null, null, null, null, null, null, null, null, null, null, ], [INF, null, null, null, null, null, null, null, null, null, null, null, ],