diff --git a/CHANGELOG b/CHANGELOG index b3873f5..10832aa 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,15 +1,30 @@ -Version 0.6.0 (????-??-??) +Version 0.6.1 (2019-01-23) +========================== + +Bug Fixes: +- Unify SQL timeout settings +- Correctly escape shell command in subprocess service driver +- Correctly allow null time intervals in configuration when appropriate + +Changes: +- Change PicoFeed dependency to maintained version (Thanks, Aaron Parecki!) +- Remove non-functional cURL service driver + +Version 0.6.0 (2019-01-21) ========================== New features: - Support for PostgreSQL databases - Support for MySQL databases +- Validation of configuration parameters Bug fixes: - Use a general-purpose Unicode collation with SQLite databases +- Use the correct SQLite schema change procedure for 3.25 and later Changes: - Improve performance of common database queries by 80-90% +- Make configuration defaults consistent with their defined types Version 0.5.1 (2018-11-10) ========================== diff --git a/README.md b/README.md index d7dcb39..2cec044 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ At present the software should be considered in an "alpha" state: though its cor - Providing more sync protocols (Google Reader, Fever, others) - Better packaging and configuration samples +- A user manual ## Requirements @@ -75,7 +76,7 @@ Please refer to `CONTRIBUTING.md` for guidelines on contributing code to The Ars Functionally there is no reason to prefer either SQLite or PostgreSQL over the other. SQLite is significantly simpler to set up in most cases, requiring only read and write access to a containing directory in order to function; PostgreSQL may perform better than SQLite when serving hundreds of users or more, though this has not been tested. -MySQL, on the other hand, is *not recommended* due to its relatively constrained index prefix limits which may cause some newsfeeds which would otherwise work to be rejected. If using MySQL, special care should also be taken when performing schema upgrades, as errors during the process can leave the database in a half-upgraded state which The Arsse cannot itself recover from. +MySQL, on the other hand, is **not recommended** due to its relatively constrained index prefix limits which may cause some newsfeeds which would otherwise work to be rejected. If using MySQL, special care should also be taken when performing schema upgrades, as errors during the process can leave the database in a half-upgraded state which The Arsse cannot itself recover from. Note that MariaDB is not compatible with The Arsse: its support for common table expressions is, as of this writing, not sufficient for our needs. diff --git a/UPGRADING b/UPGRADING index 3abe6ee..a837396 100644 --- a/UPGRADING +++ b/UPGRADING @@ -1,41 +1,49 @@ General upgrade notes ===================== -When upgrading between any two versions of The Arsse, the following are usually prudent: +When upgrading between any two versions of The Arsse, the following are +usually prudent: - Back up your database - Check for any changes to sample Web server configuration - Check for any changes to sample systemd unit or other init files -- If installing from source, update dependencies with `composer install -o --no-dev` +- If installing from source, update dependencies with: + `composer install -o --no-dev` Upgrading from 0.5.1 to 0.6.0 ============================= -- The database schema has changed from rev3 to rev4; if upgrading the database manually, apply the 3.sql file +- The database schema has changed from rev3 to rev4; if upgrading the database + manually, apply the 3.sql file +- Configuration is now validated for type and semantics: some previously + working configurations may no longer be accepted Upgrading from 0.2.1 to 0.3.0 ============================= - The following Composer dependencies have been added: - - zendframework/zend-diactoros - - psr/http-message + - zendframework/zend-diactoros + - psr/http-message Upgrading from 0.2.0 to 0.2.1 ============================= -- The database schema has changed from rev2 to rev3; if upgrading the database manually, apply the 2.sql file +- The database schema has changed from rev2 to rev3; if upgrading the database + manually, apply the 2.sql file Upgrading from 0.1.x to 0.2.0 ============================= -- The database schema has changed from rev1 to rev2; if upgrading the database manually, apply the 1.sql file -- Web server configuration has changed to accommodate Tiny Tiny RSS; the following URL paths are affected: - - /tt-rss/api/ - - /tt-rss/feed-icons/ - - /tt-rss/images/ +- The database schema has changed from rev1 to rev2; if upgrading the database + manually, apply the 1.sql file +- Web server configuration has changed to accommodate Tiny Tiny RSS; the + following URL paths are affected: + - /tt-rss/api/ + - /tt-rss/feed-icons/ + - /tt-rss/images/ - The following Composer dependencies have been added: - - jkingweb/druuid + - jkingweb/druuid diff --git a/composer.json b/composer.json index 5f943d9..0f5570c 100644 --- a/composer.json +++ b/composer.json @@ -22,7 +22,7 @@ "ext-intl": "*", "ext-json": "*", "ext-hash": "*", - "fguillot/picofeed": ">=0.1.31", + "p3k/picofeed": "0.1.*", "hosteurope/password-generator": "^1.0", "docopt/docopt": "^1.0", "jkingweb/druuid": "^3.0", diff --git a/composer.lock b/composer.lock index baf62e3..b5e5d38 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "7d381fa958169b7079c1d3c5b911f3bd", + "content-hash": "d7a6a00be3d97c11d09ec4d4e56d36e0", "packages": [ { "name": "docopt/docopt", @@ -52,59 +52,6 @@ ], "time": "2015-10-30T03:21:23+00:00" }, - { - "name": "fguillot/picofeed", - "version": "v0.1.37", - "source": { - "type": "git", - "url": "https://github.com/miniflux/picoFeed.git", - "reference": "402b7f07629577e7929625e78bc88d3d5831a22d" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/miniflux/picoFeed/zipball/402b7f07629577e7929625e78bc88d3d5831a22d", - "reference": "402b7f07629577e7929625e78bc88d3d5831a22d", - "shasum": "" - }, - "require": { - "ext-dom": "*", - "ext-iconv": "*", - "ext-libxml": "*", - "ext-simplexml": "*", - "ext-xml": "*", - "php": ">=5.3.0", - "zendframework/zendxml": "^1.0" - }, - "require-dev": { - "phpdocumentor/reflection-docblock": "2.0.4", - "phpunit/phpunit": "4.8.26", - "symfony/yaml": "2.8.7" - }, - "suggest": { - "ext-curl": "PicoFeed will use cURL if present" - }, - "bin": [ - "picofeed" - ], - "type": "library", - "autoload": { - "psr-0": { - "PicoFeed": "lib/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Frédéric Guillot" - } - ], - "description": "Modern library to handle RSS/Atom feeds", - "homepage": "https://github.com/miniflux/picoFeed", - "time": "2017-11-02T03:20:36+00:00" - }, { "name": "hosteurope/password-generator", "version": "v1.0.1", @@ -190,6 +137,59 @@ ], "time": "2017-02-09T14:17:01+00:00" }, + { + "name": "p3k/picofeed", + "version": "v0.1.38", + "source": { + "type": "git", + "url": "https://github.com/aaronpk/picoFeed.git", + "reference": "989c0bcf2eac016a4104abce1aadff791fc287ab" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/aaronpk/picoFeed/zipball/989c0bcf2eac016a4104abce1aadff791fc287ab", + "reference": "989c0bcf2eac016a4104abce1aadff791fc287ab", + "shasum": "" + }, + "require": { + "ext-dom": "*", + "ext-iconv": "*", + "ext-libxml": "*", + "ext-simplexml": "*", + "ext-xml": "*", + "php": ">=5.3.0", + "zendframework/zendxml": "^1.0" + }, + "require-dev": { + "phpdocumentor/reflection-docblock": "2.0.4", + "phpunit/phpunit": "4.8.26", + "symfony/yaml": "2.8.7" + }, + "suggest": { + "ext-curl": "PicoFeed will use cURL if present" + }, + "bin": [ + "picofeed" + ], + "type": "library", + "autoload": { + "psr-0": { + "PicoFeed": "lib/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Frédéric Guillot" + } + ], + "description": "Modern library to handle RSS/Atom feeds", + "homepage": "https://github.com/miniflux/picoFeed", + "time": "2017-11-30T00:16:58+00:00" + }, { "name": "psr/http-message", "version": "1.0.1", @@ -306,16 +306,16 @@ }, { "name": "zendframework/zendxml", - "version": "1.1.0", + "version": "1.2.0", "source": { "type": "git", "url": "https://github.com/zendframework/ZendXml.git", - "reference": "267db6a2c431a08a8f8ff0f1f4c302a5ba6f5b99" + "reference": "eceab37a591c9e140772a1470338258857339e00" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/zendframework/ZendXml/zipball/267db6a2c431a08a8f8ff0f1f4c302a5ba6f5b99", - "reference": "267db6a2c431a08a8f8ff0f1f4c302a5ba6f5b99", + "url": "https://api.github.com/repos/zendframework/ZendXml/zipball/eceab37a591c9e140772a1470338258857339e00", + "reference": "eceab37a591c9e140772a1470338258857339e00", "shasum": "" }, "require": { @@ -328,8 +328,8 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "1.1.x-dev", - "dev-develop": "1.2.x-dev" + "dev-master": "1.2.x-dev", + "dev-develop": "1.3.x-dev" } }, "autoload": { @@ -348,7 +348,7 @@ "xml", "zf" ], - "time": "2018-04-30T15:11:04+00:00" + "time": "2019-01-22T19:42:14+00:00" } ], "packages-dev": [ diff --git a/lib/AbstractException.php b/lib/AbstractException.php index cd5fcc8..0249678 100644 --- a/lib/AbstractException.php +++ b/lib/AbstractException.php @@ -65,6 +65,7 @@ abstract class AbstractException extends \Exception { "Conf/Exception.fileCorrupt" => 10306, "Conf/Exception.typeMismatch" => 10311, "Conf/Exception.semanticMismatch" => 10312, + "Conf/Exception.ambiguousDefault" => 10313, "User/Exception.functionNotImplemented" => 10401, "User/Exception.doesNotExist" => 10402, "User/Exception.alreadyExists" => 10403, diff --git a/lib/Arsse.php b/lib/Arsse.php index 39b8336..7fbd1b2 100644 --- a/lib/Arsse.php +++ b/lib/Arsse.php @@ -7,7 +7,7 @@ declare(strict_types=1); namespace JKingWeb\Arsse; class Arsse { - const VERSION = "0.5.1"; + const VERSION = "0.6.1"; /** @var Lang */ public static $lang; diff --git a/lib/CLI.php b/lib/CLI.php index 36129e0..8ff1e1d 100644 --- a/lib/CLI.php +++ b/lib/CLI.php @@ -88,7 +88,7 @@ USAGE_TEXT; /** @codeCoverageIgnore */ protected function logError(string $msg) { - fwrite(STDERR,$msg.\PHP_EOL); + fwrite(STDERR, $msg.\PHP_EOL); } /** @codeCoverageIgnore */ diff --git a/lib/Conf.php b/lib/Conf.php index e241d47..bba5821 100644 --- a/lib/Conf.php +++ b/lib/Conf.php @@ -21,16 +21,16 @@ class Conf { public $dbDriver = "sqlite3"; /** @var boolean Whether to attempt to automatically update the database when upgrading to a new version with schema changes */ public $dbAutoUpdate = true; - /** @var \DateInterval Number of seconds to wait before returning a timeout error when connecting to a database (zero waits forever; not applicable to SQLite) */ + /** @var \DateInterval|null Number of seconds to wait before returning a timeout error when connecting to a database (null waits forever; not applicable to SQLite) */ public $dbTimeoutConnect = 5.0; - /** @var \DateInterval Number of seconds to wait before returning a timeout error when executing a database operation (zero waits forever; not applicable to SQLite) */ - public $dbTimeoutExec = 0.0; + /** @var \DateInterval|null Number of seconds to wait before returning a timeout error when executing a database operation (null waits forever; not applicable to SQLite) */ + public $dbTimeoutExec = null; + /** @var \DateInterval|null Number of seconds to wait before returning a timeout error when acquiring a database lock (null waits forever) */ + public $dbTimeoutLock = 60.0; /** @var string|null Full path and file name of SQLite database (if using SQLite) */ public $dbSQLite3File = null; /** @var string Encryption key to use for SQLite database (if using a version of SQLite with SEE) */ public $dbSQLite3Key = ""; - /** @var \DateInterval Number of seconds for SQLite to wait before returning a timeout error when trying to acquire a write lock on the database (zero does not wait) */ - public $dbSQLite3Timeout = 60.0; /** @var string Host name, address, or socket path of PostgreSQL database server (if using PostgreSQL) */ public $dbPostgreSQLHost = ""; /** @var string Log-in user name for PostgreSQL database server (if using PostgreSQL) */ @@ -75,22 +75,16 @@ class Conf { * @see https://en.wikipedia.org/wiki/ISO_8601#Durations */ public $userSessionLifetime = "P7D"; - /** @var string Feed update service driver to use, one of "serial", "subprocess", or "curl". A fully-qualified class name may also be used for custom drivers */ + /** @var string Feed update service driver to use, one of "serial" or "subprocess". A fully-qualified class name may also be used for custom drivers */ public $serviceDriver = "subprocess"; /** @var \DateInterval The interval between checks for new articles, as an ISO 8601 duration * @see https://en.wikipedia.org/wiki/ISO_8601#Durations */ public $serviceFrequency = "PT2M"; /** @var integer Number of concurrent feed updates to perform */ public $serviceQueueWidth = 5; - /** @var string The base server address (with scheme, host, port if necessary, and terminal slash) to connect to the server when performing feed updates using cURL */ - public $serviceCurlBase = "http://localhost/"; - /** @var string The user name to use when performing feed updates using cURL */ - public $serviceCurlUser = ""; - /** @var string The password to use when performing feed updates using cURL */ - public $serviceCurlPassword = ""; /** @var \DateInterval Number of seconds to wait for data when fetching feeds from foreign servers */ - public $fetchTimeout = 10; + public $fetchTimeout = 10.0; /** @var integer Maximum size, in bytes, of data when fetching feeds from foreign servers */ public $fetchSizeLimit = 2 * 1024 * 1024; /** @var boolean Whether to allow the possibility of fetching full article contents using an item's URL. Whether fetching will actually happen is also governed by a per-feed setting */ @@ -115,6 +109,11 @@ class Conf { /** @var string Space-separated list of origins from which to deny cross-origin resource sharing */ public $httpOriginsDenied = ""; + ### OBSOLETE SETTINGS + + /** @var \DateInterval|null (OBSOLETE) Number of seconds for SQLite to wait before returning a timeout error when trying to acquire a write lock on the database (zero does not wait) */ + public $dbSQLite3Timeout = null; // previously 60.0 + const TYPE_NAMES = [ Value::T_BOOL => "boolean", Value::T_STRING => "string", @@ -122,6 +121,12 @@ class Conf { VALUE::T_INT => "integer", Value::T_INTERVAL => "interval", ]; + const EXPECTED_TYPES = [ + 'dbTimeoutExec' => "double", + 'dbTimeoutLock' => "double", + 'dbTimeoutConnect' => "double", + 'dbSQLite3Timeout' => "double", + ]; protected static $types = []; @@ -184,17 +189,23 @@ class Conf { /** Outputs configuration settings, either non-default ones or all, as an associative array * @param bool $full Whether to output all configuration options rather than only changed ones */ public function export(bool $full = false): array { - $ref = new self; - $out = []; $conf = new \ReflectionObject($this); + $ref = (new \ReflectionClass($this))->getDefaultProperties(); + $out = []; foreach ($conf->getProperties(\ReflectionProperty::IS_PUBLIC) as $prop) { $name = $prop->name; - // add the property to the output if the value is of a supported type and either: - // 1. full output has been requested - // 2. the property is not defined in the class - // 3. it differs from the default - if ((is_scalar($this->$name) || is_null($this->$name)) && ($full || !$prop->isDefault() || $this->$name !== $ref->$name)) { - $out[$name] = $this->$name; + $value = $prop->getValue($this); + if ($prop->isDefault()) { + $default = $ref[$name]; + // if the property is a known property (rather than one added by a hypothetical plug-in) + // we convert intervals to strings and then export anything which doesn't match the default value + $value = $this->propertyExport($name, $value); + if ((is_scalar($value) || is_null($value)) && ($full || $value !== $ref[$name])) { + $out[$name] = $value; + } + } elseif (is_scalar($value) || is_null($value)) { + // otherwise export the property only if it is scalar + $out[$name] = $value; } } return $out; @@ -213,13 +224,11 @@ class Conf { // retrieve the property's docblock, if it exists try { $doc = (new \ReflectionProperty(self::class, $prop))->getDocComment(); - } catch (\ReflectionException $e) { - } - if ($doc) { // parse the docblock to extract the property description - if (preg_match("<@var\s+\S+\s+(.+?)(?:\s*\*/)?$>m", $doc, $match)) { + if (preg_match("<@var\s+\S+\s+(.+?)(?:\s*\*/)?\s*$>m", $doc, $match)) { $comment = $match[1]; } + } catch (\ReflectionException $e) { } // append the docblock description if there is one, or an empty comment otherwise $out .= " // ".$comment.PHP_EOL; @@ -263,26 +272,28 @@ class Conf { } protected function propertyImport(string $key, $value, string $file = "") { + $typeName = static::$types[$key]['name'] ?? "mixed"; + $typeConst = static::$types[$key]['const'] ?? Value::T_MIXED; + $nullable = (int) (bool) (static::$types[$key]['const'] & Value::M_NULL); try { - $typeName = static::$types[$key]['name'] ?? "mixed"; - $typeConst = static::$types[$key]['const'] ?? Value::T_MIXED; if ($typeName === "\\DateInterval") { // date intervals have special handling: if the existing value (ultimately, the default value) // is an integer or float, the new value should be imported as numeric. If the new value is a string // it is first converted to an interval and then converted to the numeric type if necessary + $mode = $nullable ? Value::M_STRICT | Value::M_NULL : Value::M_STRICT; if (is_string($value)) { - $value = Value::normalize($value, Value::T_INTERVAL | Value::M_STRICT); + $value = Value::normalize($value, Value::T_INTERVAL | $mode); } - switch (gettype($this->$key)) { + switch (self::EXPECTED_TYPES[$key] ?? gettype($this->$key)) { case "integer": - return Value::normalize($value, Value::T_INT | Value::M_STRICT); + return Value::normalize($value, Value::T_INT | $mode); case "double": - return Value::normalize($value, Value::T_FLOAT | Value::M_STRICT); + return Value::normalize($value, Value::T_FLOAT | $mode); case "string": case "object": return $value; default: - throw new ExceptionType("strictFailure"); // @codeCoverageIgnore + throw new Conf\Exception("ambiguousDefault", ['param' => $key]); // @codeCoverageIgnore } } $value = Value::normalize($value, $typeConst); @@ -305,9 +316,22 @@ class Conf { } return $value; } catch (ExceptionType $e) { - $nullable = (int) (bool) (static::$types[$key] & Value::M_NULL); $type = static::$types[$key]['const'] & ~(Value::M_STRICT | Value::M_DROP | Value::M_NULL | Value::M_ARRAY); throw new Conf\Exception("typeMismatch", ['param' => $key, 'type' => self::TYPE_NAMES[$type], 'file' => $file, 'nullable' => $nullable]); } } + + protected function propertyExport(string $key, $value) { + $value = ($value instanceof \DateInterval) ? Value::normalize($value, Value::T_STRING) : $value; + switch ($key) { + case "dbDriver": + return array_flip(Database::DRIVER_NAMES)[$value] ?? $value; + case "userDriver": + return array_flip(User::DRIVER_NAMES)[$value] ?? $value; + case "serviceDriver": + return array_flip(Service::DRIVER_NAMES)[$value] ?? $value; + default: + return $value; + } + } } diff --git a/lib/Db/MySQL/Driver.php b/lib/Db/MySQL/Driver.php index e054800..8a4fe44 100644 --- a/lib/Db/MySQL/Driver.php +++ b/lib/Db/MySQL/Driver.php @@ -18,7 +18,7 @@ class Driver extends \JKingWeb\Arsse\Db\AbstractDriver { const SQL_MODE = "ANSI_QUOTES,HIGH_NOT_PRECEDENCE,NO_BACKSLASH_ESCAPES,NO_ENGINE_SUBSTITUTION,PIPES_AS_CONCAT,STRICT_ALL_TABLES"; const TRANSACTIONAL_LOCKS = false; - /** @var \mysql */ + /** @var \mysqli */ protected $db; protected $transStart = 0; protected $packetSize = 4194304; @@ -48,7 +48,8 @@ class Driver extends \JKingWeb\Arsse\Db\AbstractDriver { return [ "SET sql_mode = '".self::SQL_MODE."'", "SET time_zone = '+00:00'", - "SET lock_wait_timeout = 1", + "SET lock_wait_timeout = ".self::lockTimeout(), + "SET max_execution_time = ".ceil(Arsse::$conf->dbTimeoutExec * 1000), ]; } @@ -130,7 +131,7 @@ class Driver extends \JKingWeb\Arsse\Db\AbstractDriver { try { $this->exec("SET lock_wait_timeout = 1; LOCK TABLES $tables"); } finally { - $this->exec("SET lock_wait_timeout = 60"); + $this->exec("SET lock_wait_timeout = ".self::lockTimeout()); } } return true; @@ -141,6 +142,10 @@ class Driver extends \JKingWeb\Arsse\Db\AbstractDriver { return true; } + protected static function lockTimeout(): int { + return (int) max(min(ceil(Arsse::$conf->dbTimeoutLock ?? 31536000), 31536000), 1); + } + public function __destruct() { if (isset($this->db)) { $this->db->close(); @@ -157,7 +162,9 @@ class Driver extends \JKingWeb\Arsse\Db\AbstractDriver { } protected function makeConnection(string $db, string $user, string $password, string $host, int $port, string $socket) { - $this->db = @new \mysqli($host, $user, $password, $db, $port, $socket); + $this->db = mysqli_init(); + $this->db->options(\MYSQLI_OPT_CONNECT_TIMEOUT, ceil(Arsse::$conf->dbTimeoutConnect)); + @$this->db->real_connect($host, $user, $password, $db, $port, $socket); if ($this->db->connect_errno) { list($excClass, $excMsg, $excData) = $this->buildConnectionException($this->db->connect_errno, $this->db->connect_error); throw new $excClass($excMsg, $excData); diff --git a/lib/Db/MySQL/Statement.php b/lib/Db/MySQL/Statement.php index 8af64f4..9612615 100644 --- a/lib/Db/MySQL/Statement.php +++ b/lib/Db/MySQL/Statement.php @@ -94,8 +94,8 @@ class Statement extends \JKingWeb\Arsse\Db\AbstractStatement { } protected function bindValue($value, string $type, int $position): bool { - // this is a bit of a hack: we collect values (and MySQL bind types) here so that we can take - // advantage of the work done by bindValues() even though MySQL requires everything to be bound + // this is a bit of a hack: we collect values (and MySQL bind types) here so that we can take + // advantage of the work done by bindValues() even though MySQL requires everything to be bound // all at once; we also segregate large values for later packetization if (($type === "binary" && !is_null($value)) || (is_string($value) && strlen($value) > $this->packetSize)) { $this->values[] = null; diff --git a/lib/Db/PostgreSQL/Driver.php b/lib/Db/PostgreSQL/Driver.php index 8886546..513ce99 100644 --- a/lib/Db/PostgreSQL/Driver.php +++ b/lib/Db/PostgreSQL/Driver.php @@ -74,11 +74,13 @@ class Driver extends \JKingWeb\Arsse\Db\AbstractDriver { } public static function makeSetupQueries(string $schema = ""): array { - $timeout = ceil(Arsse::$conf->dbTimeoutExec * 1000); + $timeExec = is_null(Arsse::$conf->dbTimeoutExec) ? 0 : ceil(max(Arsse::$conf->dbTimeoutExec * 1000, 1)); + $timeLock = is_null(Arsse::$conf->dbTimeoutLock) ? 0 : ceil(max(Arsse::$conf->dbTimeoutLock * 1000, 1)); $out = [ "SET TIME ZONE UTC", "SET DateStyle = 'ISO, MDY'", - "SET statement_timeout = '$timeout'", + "SET statement_timeout = '$timeExec'", + "SET lock_timeout = '$timeLock'", ]; if (strlen($schema) > 0) { $schema = '"'.str_replace('"', '""', $schema).'"'; diff --git a/lib/Db/PostgreSQL/Statement.php b/lib/Db/PostgreSQL/Statement.php index 8d6668f..df74e3d 100644 --- a/lib/Db/PostgreSQL/Statement.php +++ b/lib/Db/PostgreSQL/Statement.php @@ -29,7 +29,7 @@ class Statement extends \JKingWeb\Arsse\Db\AbstractStatement { protected $bindings; public function __construct($db, string $query, array $bindings = []) { - $this->db = $db; + $this->db = $db; $this->query = $query; $this->retypeArray($bindings); } diff --git a/lib/Db/SQLite3/Driver.php b/lib/Db/SQLite3/Driver.php index 612072b..f7e47fb 100644 --- a/lib/Db/SQLite3/Driver.php +++ b/lib/Db/SQLite3/Driver.php @@ -55,7 +55,8 @@ class Driver extends \JKingWeb\Arsse\Db\AbstractDriver { throw new Exception("fileCorrupt", $dbFile); } // set the timeout - $timeout = (int) ceil(Arsse::$conf->dbSQLite3Timeout * 1000); + $timeout = Arsse::$conf->dbSQLite3Timeout ?? Arsse::$conf->dbTimeoutLock; // old SQLite-specific timeout takes precedence + $timeout = is_null($timeout) ? PHP_INT_MAX : (int) ceil($timeout * 1000); $this->setTimeout($timeout); // set other initial options $this->exec("PRAGMA foreign_keys = yes"); @@ -123,14 +124,12 @@ class Driver extends \JKingWeb\Arsse\Db\AbstractDriver { public function schemaUpdate(int $to, string $basePath = null): bool { // turn off foreign keys $this->exec("PRAGMA foreign_keys = no"); - $this->exec("PRAGMA legacy_alter_table = yes"); // run the generic updater try { parent::schemaUpdate($to, $basePath); } finally { // turn foreign keys back on $this->exec("PRAGMA foreign_keys = yes"); - $this->exec("PRAGMA legacy_alter_table = no"); } return true; } diff --git a/lib/Feed.php b/lib/Feed.php index 5a548d8..aaa048b 100644 --- a/lib/Feed.php +++ b/lib/Feed.php @@ -78,7 +78,7 @@ class Feed { protected static function configure(): Config { $userAgent = Arsse::$conf->fetchUserAgentString ?? sprintf( - 'Arsse/%s (%s %s; %s; https://thearsse.com/) PicoFeed (https://github.com/miniflux/picoFeed)', + 'Arsse/%s (%s %s; %s; https://thearsse.com/)', Arsse::VERSION, // Arsse version php_uname('s'), // OS php_uname('r'), // OS version diff --git a/lib/Misc/ValueInfo.php b/lib/Misc/ValueInfo.php index 3ce76e7..752704d 100644 --- a/lib/Misc/ValueInfo.php +++ b/lib/Misc/ValueInfo.php @@ -352,7 +352,7 @@ class ValueInfo { // input is a number, assume this is a number of seconds // for legibility we convert large numbers to minutes, hours, and days as necessary // the DateInterval constructor only allows 12 digits for any given part of an interval, - // so we also convert days to 365-day years where we must, and cap the number of years + // so we also convert days to 365-day years where we must, and cap the number of years // at (1e11 - 1); this being a very large number, the loss of precision is probably not // significant in practical usage $sec = abs($value); diff --git a/lib/Service/Curl/Driver.php b/lib/Service/Curl/Driver.php deleted file mode 100644 index cfb73d4..0000000 --- a/lib/Service/Curl/Driver.php +++ /dev/null @@ -1,77 +0,0 @@ -msg("Driver.Service.Curl.Name"); - } - - public static function requirementsMet(): bool { - return extension_loaded("curl"); - } - - public function __construct() { - //default curl options for individual requests - $this->options = [ - \CURLOPT_URL => Arsse::$serviceCurlBase."index.php/apps/news/api/v1-2/feeds/update", - \CURLOPT_CUSTOMREQUEST => "GET", - \CURLOPT_FAILONERROR => false, - \CURLOPT_FOLLOWLOCATION => false, - \CURLOPT_FORBID_REUSE => false, - \CURLOPT_CONNECTTIMEOUT => 20, - \CURLOPT_DNS_CACHE_TIMEOUT => 360, // FIXME: this should probably be twice the update-check interval so that the DNS cache is always in memory - \CURLOPT_PROTOCOLS => \CURLPROTO_HTTP | \CURLPROTO_HTTPS, - \CURLOPT_DEFAULT_PROTOCOL => "https", - \CURLOPT_USERAGENT => Arsse::$conf->fetchUserAgentString, - \CURLMOPT_MAX_HOST_CONNECTIONS => Arsse::$conf->serviceQueueWidth, - \CURLOPT_HTTPHEADER => [ - 'Accept: application/json', - 'Content-Type: application/json', - ], - \CURLOPT_HEADER => false, - ]; - // start an async session - $this->queue = curl_multi_init(); - // enable pipelining - curl_multi_setopt($this->queue, \CURLMOPT_PIPELINING, 1); - } - - public function queue(int ...$feeds): int { - foreach ($feeds as $id) { - $h = curl_init(); - curl_setopt($h, \CURLOPT_POSTFIELDS, json_encode(['userId' => "", 'feedId' => $id])); - $this->handles[] = $h; - curl_multi_add_handle($this->queue, $h); - } - return sizeof($this->handles); - } - - public function exec(): int { - $active = 0; - do { - curl_multi_exec($this->queue, $active); - curl_multi_select($this->queue); - } while ($active > 0); - return Arsse::$conf->serviceQueueWidth - $active; - } - - public function clean(): bool { - foreach ($this->handles as $h) { - curl_multi_remove_handle($this->queue, $h); - curl_close($h); - } - $this->handles = []; - return true; - } -} diff --git a/lib/Service/Subprocess/Driver.php b/lib/Service/Subprocess/Driver.php index a657232..5e79ed0 100644 --- a/lib/Service/Subprocess/Driver.php +++ b/lib/Service/Subprocess/Driver.php @@ -31,8 +31,8 @@ class Driver implements \JKingWeb\Arsse\Service\Driver { $pp = []; while ($this->queue) { $id = (int) array_shift($this->queue); - $php = '"'.\PHP_BINARY.'"'; - $arsse = '"'.$_SERVER['argv'][0].'"'; + $php = escapeshellarg(\PHP_BINARY); + $arsse = escapeshellarg($_SERVER['argv'][0]); array_push($pp, popen("$php $arsse feed refresh $id", "r")); } while ($pp) { diff --git a/locale/en.php b/locale/en.php index dcaa848..f576442 100644 --- a/locale/en.php +++ b/locale/en.php @@ -27,7 +27,6 @@ return [ 'Driver.Service.Serial.Name' => 'Serialized', 'Driver.Service.Subprocess.Name' => 'Concurrent subprocess', - 'Driver.Service.Curl.Name' => 'Concurrent HTTP (curl)', 'Driver.User.Internal.Name' => 'Internal', @@ -75,6 +74,8 @@ return [ other {, or null} }', 'Exception.JKingWeb/Arsse/Conf/Exception.semanticMismatch' => 'Configuration parameter "{param}" in file "{file}" is not a valid value. Consult the documentation for possible values', + // indicates programming error + 'Exception.JKingWeb/Arsse/Conf/Exception.ambiguousDefault' => 'Preferred type of configuration parameter "{param}" could not be inferred from its default value. The parameter must be added to the Conf::EXPECTED_TYPES array', 'Exception.JKingWeb/Arsse/Db/Exception.extMissing' => 'Required PHP extension for driver "{0}" not installed', 'Exception.JKingWeb/Arsse/Db/Exception.fileMissing' => 'Database file "{0}" does not exist', 'Exception.JKingWeb/Arsse/Db/Exception.fileUnreadable' => 'Insufficient permissions to open database file "{0}" for reading', diff --git a/sql/SQLite3/1.sql b/sql/SQLite3/1.sql index b96bd79..dc7862d 100644 --- a/sql/SQLite3/1.sql +++ b/sql/SQLite3/1.sql @@ -2,7 +2,7 @@ -- Copyright 2017 J. King, Dustin Wilson et al. -- See LICENSE and AUTHORS files for details -create table arsse_sessions ( +create table arsse_sessions( -- sessions for Tiny Tiny RSS (and possibly others) id text primary key, -- UUID of session created text not null default CURRENT_TIMESTAMP, -- Session start timestamp @@ -10,7 +10,7 @@ create table arsse_sessions ( user text not null references arsse_users(id) on delete cascade on update cascade -- user associated with the session ) without rowid; -create table arsse_labels ( +create table arsse_labels( -- user-defined article labels for Tiny Tiny RSS id integer primary key, -- numeric ID owner text not null references arsse_users(id) on delete cascade on update cascade, -- owning user @@ -19,7 +19,7 @@ create table arsse_labels ( unique(owner,name) ); -create table arsse_label_members ( +create table arsse_label_members( -- uabels assignments for articles label integer not null references arsse_labels(id) on delete cascade, -- label ID associated to an article; label IDs belong to a user article integer not null references arsse_articles(id) on delete cascade, -- article associated to a label @@ -32,8 +32,7 @@ create table arsse_label_members ( -- alter marks table to add Tiny Tiny RSS' notes -- SQLite has limited ALTER TABLE support, so the table must be re-created -- and its data re-entered; other database systems have a much simpler prodecure -alter table arsse_marks rename to arsse_marks_old; -create table arsse_marks( +create table arsse_marks_new( -- users' actions on newsfeed entries article integer not null references arsse_articles(id) on delete cascade, -- article associated with the marks subscription integer not null references arsse_subscriptions(id) on delete cascade on update cascade, -- subscription associated with the marks; the subscription in turn belongs to a user @@ -43,8 +42,9 @@ create table arsse_marks( note text not null default '', -- Tiny Tiny RSS freeform user note primary key(article,subscription) -- no more than one mark-set per article per user ); -insert into arsse_marks(article,subscription,read,starred,modified) select article,subscription,read,starred,modified from arsse_marks_old; -drop table arsse_marks_old; +insert into arsse_marks_new(article,subscription,read,starred,modified) select article,subscription,read,starred,modified from arsse_marks; +drop table arsse_marks; +alter table arsse_marks_new rename to arsse_marks; -- set version marker pragma user_version = 2; diff --git a/sql/SQLite3/2.sql b/sql/SQLite3/2.sql index 7340290..b378467 100644 --- a/sql/SQLite3/2.sql +++ b/sql/SQLite3/2.sql @@ -5,8 +5,7 @@ -- Correct collation sequences in order for various things to sort case-insensitively -- SQLite has limited ALTER TABLE support, so the tables must be re-created -- and their data re-entered; other database systems have a much simpler prodecure -alter table arsse_users rename to arsse_users_old; -create table arsse_users( +create table arsse_users_new( -- users id text primary key not null collate nocase, -- user id password text, -- password, salted and hashed; if using external authentication this would be blank @@ -16,11 +15,11 @@ create table arsse_users( admin boolean default 0, -- whether the user is a member of the special "admin" group rights integer not null default 0 -- temporary admin-rights marker FIXME: remove reliance on this ); -insert into arsse_users(id,password,name,avatar_type,avatar_data,admin,rights) select id,password,name,avatar_type,avatar_data,admin,rights from arsse_users_old; -drop table arsse_users_old; +insert into arsse_users_new(id,password,name,avatar_type,avatar_data,admin,rights) select id,password,name,avatar_type,avatar_data,admin,rights from arsse_users; +drop table arsse_users; +alter table arsse_users_new rename to arsse_users; -alter table arsse_folders rename to arsse_folders_old; -create table arsse_folders( +create table arsse_folders_new( -- folders, used by NextCloud News and Tiny Tiny RSS -- feed subscriptions may belong to at most one folder; -- in Tiny Tiny RSS folders may nest @@ -31,11 +30,11 @@ create table arsse_folders( modified text not null default CURRENT_TIMESTAMP, -- time at which the folder itself (not its contents) was changed; not currently used unique(owner,name,parent) -- cannot have multiple folders with the same name under the same parent for the same owner ); -insert into arsse_folders select * from arsse_folders_old; -drop table arsse_folders_old; +insert into arsse_folders_new select * from arsse_folders; +drop table arsse_folders; +alter table arsse_folders_new rename to arsse_folders; -alter table arsse_feeds rename to arsse_feeds_old; -create table arsse_feeds( +create table arsse_feeds_new( -- newsfeeds, deduplicated -- users have subscriptions to these feeds in another table id integer primary key, -- sequence number @@ -56,11 +55,11 @@ create table arsse_feeds( scrape boolean not null default 0, -- whether to use picoFeed's content scraper with this feed unique(url,username,password) -- a URL with particular credentials should only appear once ); -insert into arsse_feeds select * from arsse_feeds_old; -drop table arsse_feeds_old; +insert into arsse_feeds_new select * from arsse_feeds; +drop table arsse_feeds; +alter table arsse_feeds_new rename to arsse_feeds; -alter table arsse_subscriptions rename to arsse_subscriptions_old; -create table arsse_subscriptions( +create table arsse_subscriptions_new( -- users' subscriptions to newsfeeds, with settings id integer primary key, -- sequence number owner text not null references arsse_users(id) on delete cascade on update cascade, -- owner of subscription @@ -73,11 +72,11 @@ create table arsse_subscriptions( folder integer references arsse_folders(id) on delete cascade, -- TT-RSS category (nestable); the first-level category (which acts as NextCloud folder) is joined in when needed unique(owner,feed) -- a given feed should only appear once for a given owner ); -insert into arsse_subscriptions select * from arsse_subscriptions_old; -drop table arsse_subscriptions_old; +insert into arsse_subscriptions_new select * from arsse_subscriptions; +drop table arsse_subscriptions; +alter table arsse_subscriptions_new rename to arsse_subscriptions; -alter table arsse_articles rename to arsse_articles_old; -create table arsse_articles( +create table arsse_articles_new( -- entries in newsfeeds id integer primary key, -- sequence number feed integer not null references arsse_feeds(id) on delete cascade, -- feed for the subscription @@ -93,22 +92,22 @@ create table arsse_articles( url_content_hash text not null, -- hash of URL + content, enclosure URL, & content type; used when checking for updates and for identification if there is no guid. title_content_hash text not null -- hash of title + content, enclosure URL, & content type; used when checking for updates and for identification if there is no guid. ); -insert into arsse_articles select * from arsse_articles_old; -drop table arsse_articles_old; +insert into arsse_articles_new select * from arsse_articles; +drop table arsse_articles; +alter table arsse_articles_new rename to arsse_articles; -alter table arsse_categories rename to arsse_categories_old; -create table arsse_categories( +create table arsse_categories_new( -- author categories associated with newsfeed entries -- these are not user-modifiable article integer not null references arsse_articles(id) on delete cascade, -- article associated with the category name text collate nocase -- freeform name of the category ); -insert into arsse_categories select * from arsse_categories_old; -drop table arsse_categories_old; +insert into arsse_categories_new select * from arsse_categories; +drop table arsse_categories; +alter table arsse_categories_new rename to arsse_categories; -alter table arsse_labels rename to arsse_labels_old; -create table arsse_labels ( +create table arsse_labels_new( -- user-defined article labels for Tiny Tiny RSS id integer primary key, -- numeric ID owner text not null references arsse_users(id) on delete cascade on update cascade, -- owning user @@ -116,8 +115,9 @@ create table arsse_labels ( modified text not null default CURRENT_TIMESTAMP, -- time at which the label was last modified unique(owner,name) ); -insert into arsse_labels select * from arsse_labels_old; -drop table arsse_labels_old; +insert into arsse_labels_new select * from arsse_labels; +drop table arsse_labels; +alter table arsse_labels_new rename to arsse_labels; -- set version marker pragma user_version = 3; diff --git a/sql/SQLite3/3.sql b/sql/SQLite3/3.sql index bac79a8..0d58324 100644 --- a/sql/SQLite3/3.sql +++ b/sql/SQLite3/3.sql @@ -4,8 +4,7 @@ -- allow marks to initially have a null date due to changes in how marks are first created -- and also add a "touched" column to aid in tracking changes during the course of some transactions -alter table arsse_marks rename to arsse_marks_old; -create table arsse_marks( +create table arsse_marks_new( -- users' actions on newsfeed entries article integer not null references arsse_articles(id) on delete cascade, -- article associated with the marks subscription integer not null references arsse_subscriptions(id) on delete cascade on update cascade, -- subscription associated with the marks; the subscription in turn belongs to a user @@ -16,8 +15,9 @@ create table arsse_marks( touched boolean not null default 0, -- used to indicate a record has been modified during the course of some transactions primary key(article,subscription) -- no more than one mark-set per article per user ); -insert into arsse_marks select article,subscription,read,starred,modified,note,0 from arsse_marks_old; -drop table arsse_marks_old; +insert into arsse_marks_new select article,subscription,read,starred,modified,note,0 from arsse_marks; +drop table arsse_marks; +alter table arsse_marks_new rename to arsse_marks; -- reindex anything which uses the nocase collation sequence; it has been replaced with a Unicode collation reindex nocase; diff --git a/tests/cases/Conf/TestConf.php b/tests/cases/Conf/TestConf.php index f4a4430..4016443 100644 --- a/tests/cases/Conf/TestConf.php +++ b/tests/cases/Conf/TestConf.php @@ -122,10 +122,12 @@ class TestConf extends \JKingWeb\Arsse\Test\AbstractTest { $conf->lang = ["en", "fr"]; // should not be exported: not scalar $conf->dbSQLite3File = "test.db"; // should be exported: value changed $conf->userDriver = null; // should be exported: changed value, even when null + $conf->serviceFrequency = new \DateInterval("PT1H"); // should be exported (as string): value changed $conf->someCustomProperty = "Look at me!"; // should be exported: unknown property $exp = [ 'dbSQLite3File' => "test.db", 'userDriver' => null, + 'serviceFrequency' => "PT1H", 'someCustomProperty' => "Look at me!", ]; $this->assertSame($exp, $conf->export()); diff --git a/tests/cases/Db/BaseDriver.php b/tests/cases/Db/BaseDriver.php index dcc8b34..682c688 100644 --- a/tests/cases/Db/BaseDriver.php +++ b/tests/cases/Db/BaseDriver.php @@ -19,6 +19,7 @@ abstract class BaseDriver extends \JKingWeb\Arsse\Test\AbstractTest { protected $setVersion; protected static $conf = [ 'dbTimeoutExec' => 0.5, + 'dbTimeoutLock' => 0.001, 'dbSQLite3Timeout' => 0, //'dbSQLite3File' => "(temporary file)", ]; diff --git a/tests/cases/Misc/TestValueInfo.php b/tests/cases/Misc/TestValueInfo.php index 75a48bc..b6a34a6 100644 --- a/tests/cases/Misc/TestValueInfo.php +++ b/tests/cases/Misc/TestValueInfo.php @@ -457,7 +457,7 @@ class TestValueInfo extends \JKingWeb\Arsse\Test\AbstractTest { */ /* Input value null bool int float string array interval */ [null, [null,true], [false,false], [0, false], [0.0, false], ["", false], [[], false], [null, false]], - ["", [null,true], [false,true], [0, false], [0.0, false], ["", true], [[""], false], [null, false]], + ["", [null,true], [false,true], [0, false], [0.0, false], ["", true], [[""], false], [null, false]], [1, [null,true], [true, true], [1, true], [1.0, true], ["1", true], [[1], false], [$this->i("PT1S"), false]], [PHP_INT_MAX, [null,true], [true, false], [PHP_INT_MAX, true], [(float) PHP_INT_MAX,true], [(string) PHP_INT_MAX, true], [[PHP_INT_MAX], false], [$this->i("P292471208677Y195DT15H30M7S"), false]], [1.0, [null,true], [true, true], [1, true], [1.0, true], ["1", true], [[1.0], false], [$this->i("PT1S"), false]], @@ -571,7 +571,7 @@ class TestValueInfo extends \JKingWeb\Arsse\Test\AbstractTest { "!M j, Y (D)", null, ]; - foreach([ + foreach ([ /* Input value microtime iso8601 iso8601m http sql date time unix float '!M j, Y (D)' *strtotime* (null) */ [null, null, null, null, null, null, null, null, null, null, null, null, ], [INF, null, null, null, null, null, null, null, null, null, null, null, ],