Browse Source

Better database update routine

microsub
J. King 8 years ago
parent
commit
1df238a25c
  1. 2
      bootstrap.php
  2. 20
      locale/en.php
  3. 49
      sql/SQLite3/0.feeds.sql
  4. 53
      sql/SQLite3/0.main.sql
  5. 137
      vendor/JKingWeb/NewsSync/Database.php
  6. 27
      vendor/JKingWeb/NewsSync/Db/Common.php
  7. 2
      vendor/JKingWeb/NewsSync/Db/CommonPDO.php
  8. 38
      vendor/JKingWeb/NewsSync/Db/CommonSQLite3.php
  9. 25
      vendor/JKingWeb/NewsSync/Db/Driver.php
  10. 16
      vendor/JKingWeb/NewsSync/Db/DriverSQLite3.php
  11. 8
      vendor/JKingWeb/NewsSync/Db/DriverSQLite3PDO.php
  12. 9
      vendor/JKingWeb/NewsSync/Db/Result.php
  13. 40
      vendor/JKingWeb/NewsSync/Db/ResultSQLite3.php
  14. 6
      vendor/JKingWeb/NewsSync/Db/Update/Exception.php
  15. 2
      vendor/JKingWeb/NewsSync/RuntimeData.php

2
bootstrap.php

@ -16,4 +16,6 @@ spl_autoload_register(function ($class) {
}
});
ignore_user_abort(true);
$data = new RuntimeData(new Conf());

20
locale/en.php

@ -20,14 +20,22 @@ return [
"Exception.JKingWeb/NewsSync/Db/Exception.fileUnusable" => "Insufficient permissions to open database file \"{0}\" for reading or writing",
"Exception.JKingWeb/NewsSync/Db/Exception.fileUncreatable" => "Insufficient permissions to create new database file \"{0}\"",
"Exception.JKingWeb/NewsSync/Db/Exception.fileCorrupt" => "Database file \"{0}\" is corrupt or not a valid database",
"Exception.JKingWeb/NewsSync/Db/ExceptionUpdate.manual" =>
"Exception.JKingWeb/NewsSync/Db/Update/Exception.manual" =>
"{from_version, select,
0 {{driver_name} database is configured for manual updates and is not initialized; please populate the database with the base schema}
other {{driver_name} database is configured for manual updates; please update from schema version {from_version} to version {to_version}}
other {{driver_name} database is configured for manual updates; please update from schema version {current} to version {target}}
}",
"Exception.JKingWeb/NewsSync/Db/ExceptionUpdate.failed" =>
"{reason select,
missing {Automatic updating of the {driver_name} database failed because instructions for updating from version {from_version} are not available}
"Exception.JKingWeb/NewsSync/Db/Update/Exception.manualOnly" =>
"{from_version, select,
0 {{driver_name} database must be updated manually and is not initialized; please populate the database with the base schema}
other {{driver_name} database must be updated manually; please update from schema version {current} to version {target}}
}",
"Exception.JKingWeb/NewsSync/Db/Update/Exception.missing" => "Automatic updating of the {driver_name} database failed due to instructions for updating from version {current} not being available",
"Exception.JKingWeb/NewsSync/Db/Update/Exception.unreadable" => "Automatic updating of the {driver_name} database failed due to insufficient permissions to read instructions for updating from version {current}",
"Exception.JKingWeb/NewsSync/Db/Update/Exception.unusable" => "Automatic updating of the {driver_name} database failed due to an error reading instructions for updating from version {current}",
"Exception.JKingWeb/NewsSync/Db/Update/Exception.tooNew" =>
"{difference, select,
0 {Automatic updating of the {driver_name} database failed because it is already up to date with the requested version, {target}}
other {Automatic updating of the {driver_name} database failed because its version, {current}, is newer than the requested version, {target}}
}",
"Exception.JKingWeb/NewsSync/Db/ExceptionUpdate.tooNew" => "Automatic updating of the {driver_name} database failed because its version, {current}, is newer than the requested version, {target}"
];

49
sql/SQLite3/0.feeds.sql

@ -0,0 +1,49 @@
-- newsfeeds, deduplicated
create table feeds.newssync_feeds(
id integer primary key not null, -- sequence number
url TEXT not null, -- URL of feed
title TEXT, -- default title of feed
favicon TEXT, -- URL of favicon
source TEXT, -- URL of site to which the feed belongs
updated datetime, -- time at which the feed was last fetched
modified datetime not null default CURRENT_TIMESTAMP, --
err_count integer not null default 0, -- count of successive times update resulted in error since last successful update
err_msg TEXT, -- last error message
username TEXT, -- HTTP authentication username
password TEXT, -- HTTP authentication password (this is stored in plain text)
unique(url,username,password) -- a URL with particular credentials should only appear once
);
-- entries in newsfeeds
create table feeds.newssync_articles(
id integer primary key not null, -- sequence number
feed integer not null references newssync_feeds(id) on delete cascade, -- feed for the subscription
url TEXT not null, -- URL of article
title TEXT, -- article title
author TEXT, -- author's name
published datetime, -- time of original publication
edited datetime, -- time of last edit
guid TEXT, -- GUID
content TEXT, -- content, as (X)HTML
modified datetime not null default CURRENT_TIMESTAMP, -- date when article properties were last modified
hash varchar(64) not null, -- ownCloud hash
fingerprint varchar(64) not null, -- ownCloud fingerprint
enclosures_hash varchar(64), -- hash of enclosures, if any; since enclosures are not uniquely identified, we need to know when they change
tags_hash varchar(64) -- hash of RSS/Atom categories included in article; since these categories are not uniquely identified, we need to know when they change
);
-- enclosures associated with articles
create table feeds.newssync_enclosures(
article integer not null references newssync_articles(id) on delete cascade,
url TEXT,
type varchar(255)
);
-- author labels ("categories" in RSS/Atom parlance) associated with newsfeed entries
create table feeds.newssync_tags(
article integer not null references newssync_articles(id) on delete cascade,
name TEXT
);
-- set version marker
pragma feeds.user_version = 1;

53
sql/SQLite3/0.sql → sql/SQLite3/0.main.sql

@ -1,58 +1,9 @@
-- newsfeeds, deduplicated
create table feeds.newssync_feeds(
id integer primary key not null, -- sequence number
url TEXT not null, -- URL of feed
title TEXT, -- default title of feed
favicon TEXT, -- URL of favicon
source TEXT, -- URL of site to which the feed belongs
updated datetime, -- time at which the feed was last fetched
modified datetime not null default CURRENT_TIMESTAMP, --
err_count integer not null default 0, -- count of successive times update resulted in error since last successful update
err_msg TEXT, -- last error message
username TEXT, -- HTTP authentication username
password TEXT, -- HTTP authentication password (this is stored in plain text)
unique(url,username,password) -- a URL with particular credentials should only appear once
);
-- entries in newsfeeds
create table feeds.newssync_articles(
id integer primary key not null, -- sequence number
feed integer not null references newssync_feeds(id) on delete cascade, -- feed for the subscription
url TEXT not null, -- URL of article
title TEXT, -- article title
author TEXT, -- author's name
published datetime, -- time of original publication
edited datetime, -- time of last edit
guid TEXT, -- GUID
content TEXT, -- content, as (X)HTML
modified datetime not null default CURRENT_TIMESTAMP, -- date when article properties were last modified
hash varchar(64) not null, -- ownCloud hash
fingerprint varchar(64) not null, -- ownCloud fingerprint
enclosures_hash varchar(64), -- hash of enclosures, if any; since enclosures are not uniquely identified, we need to know when they change
tags_hash varchar(64) -- hash of RSS/Atom categories included in article; since these categories are not uniquely identified, we need to know when they change
);
-- enclosures associated with articles
create table feeds.newssync_enclosures(
article integer not null references newssync_articles(id) on delete cascade,
url TEXT,
type varchar(255)
);
-- author labels ("categories" in RSS/Atom parlance) associated with newsfeed entries
create table feeds.newssync_tags(
article integer not null references newssync_articles(id) on delete cascade,
name TEXT
);
-- set version marker
pragma feeds.user_version = 1;
-- settings
create table main.newssync_settings(
key varchar(255) primary key not null, --
value varchar(255), --
type varchar(255) not null check(
type in('int', 'numeric','text','timestamp', 'date', 'time', 'bool')
type in('int','numeric','text','timestamp','date','time','bool','null','json')
) --
);

137
vendor/JKingWeb/NewsSync/Database.php

@ -4,23 +4,24 @@ namespace JKingWeb\NewsSync;
class Database {
const SCHEMA_VERSION = 1;
const FORMAT_TS = "Y-m-d h:i:s";
const FORMAT_DATE = "Y-m-d";
const FORMAT_TIME = "h:i:s";
protected $data;
protected $db;
protected function clean_name(string $name): string {
protected function cleanName(string $name): string {
return (string) preg_filter("[^0-9a-zA-Z_\.]", "", $name);
}
public function __construct(Conf $conf) {
$driver = $conf->dbClass;
$this->db = $driver::create($conf, INSTALL);
public function __construct(RuntimeData $data) {
$this->data = $data;
$driver = $data->conf->dbClass;
$this->db = $driver::create($data, INSTALL);
$ver = $this->db->schemaVersion();
if($ver < self::SCHEMA_VERSION) {
if($conf->dbSQLite3AutoUpd) {
$this->db->update(self::SCHEMA_VERSION);
} else {
throw new Db\Exception("updateManual", ['from_version' => $ver, 'to_version' => self::SCHEMA_VERSION, 'driver_name' => $this->db->driverName()]);
}
if(!INSTALL && $ver < self::SCHEMA_VERSION) {
$this->db->update(self::SCHEMA_VERSION);
}
}
@ -44,6 +45,122 @@ class Database {
return $this->db->schemaVersion();
}
public function getSetting(string $key) {
$row = $this->db->prepare("SELECT value, type from newssync_settings where key = ?", "str")->run($key)->get();
if(!$row) return null;
switch($row['type']) {
case "int": return (int) $row['value'];
case "numeric": return (float) $row['value'];
case "text": return $row['value'];
case "json": return json_decode($row['value']);
case "timestamp": return date_create_from_format("!".self::FORMAT_TS, $row['value'], new DateTimeZone("UTC"));
case "date": return date_create_from_format("!".self::FORMAT_DATE, $row['value'], new DateTimeZone("UTC"));
case "time": return date_create_from_format("!".self::FORMAT_TIME, $row['value'], new DateTimeZone("UTC"));
case "bool": return (bool) $row['value'];
case "null": return null;
default: return $row['value'];
}
}
public function setSetting(string $key, $in, string $type = null): bool {
if(!$type) {
switch(gettype($in)) {
case "boolean": $type = "bool"; break;
case "integer": $type = "int"; break;
case "double": $type = "numeric"; break;
case "string":
case "array": $type = "json"; break;
case "resource":
case "unknown type":
case "NULL": $type = "null"; break;
case "object":
if($in instanceof DateTimeInterface) {
$type = "timestamp";
} else {
$type = "text";
}
break;
default: $type = 'null'; break;
}
}
$type = strtolower($type);
switch($type) {
case "integer":
$type = "int";
case "int":
$value =& $in;
break;
case "float":
case "double":
case "real":
$type = "numeric";
case "numeric":
$value =& $in;
break;
case "str":
case "string":
$type = "text";
case "text":
$value =& $in;
break;
case "json":
if(is_array($in) || is_object($in)) {
$value = json_encode($in);
} else {
$value =& $in;
}
break;
case "datetime":
$type = "timestamp";
case "timestamp":
if($in instanceof DateTimeInterface) {
$value = gmdate(self::FORMAT_TS, $in->format("U"));
} else if(is_numeric($in)) {
$value = gmdate(self::FORMAT_TS, $in);
} else {
$value = gmdate(self::FORMAT_TS, gmstrftime($in));
}
break;
case "date":
if($in instanceof DateTimeInterface) {
$value = gmdate(self::FORMAT_DATE, $in->format("U"));
} else if(is_numeric($in)) {
$value = gmdate(self::FORMAT_DATE, $in);
} else {
$value = gmdate(self::FORMAT_DATE, gmstrftime($in));
}
break;
case "time":
if($in instanceof DateTimeInterface) {
$value = gmdate(self::FORMAT_TIME, $in->format("U"));
} else if(is_numeric($in)) {
$value = gmdate(self::FORMAT_TIME, $in);
} else {
$value = gmdate(self::FORMAT_TIME, gmstrftime($in));
}
break;
case "boolean":
case "bit":
$type = "bool";
case "bool":
$value = (int) $in;
break;
case "null":
$value = null;
break;
default:
$type = "text";
$value =& $in;
break;
}
$this->db->prepare("REPLACE INTO newssync_settings(key,value,type) values(?,?,?)", "str", (($type=="null") ? "null" : "str"), "str")->run($key, $value, "text");
}
public function clearSetting(string $key): bool {
$this->db->prepare("DELETE from newssync_settings where key = ?", "str")->run($key);
return true;
}
public function userAdd(string $username, string $password = null, bool $admin = false): string {
$this->db->prepare("INSERT INTO newssync_users(id,password,admin) values(?,?,?)", "str", "str", "bool")->run($username,$password,$admin);
return $username;

27
vendor/JKingWeb/NewsSync/Db/Common.php

@ -1,14 +1,10 @@
<?php
declare(strict_types=1);
namespace JKingWeb\NewsSync\Db;
use JKingWeb\DrUUID\UUID as UUID;
Trait Common {
protected $transDepth = 0;
public function fail(\Throwable $e, bool $bool = false) {
$this->rollback($all);
throw $e;
}
public function begin(): bool {
$this->exec("SAVEPOINT newssync_".($this->transDepth));
@ -32,6 +28,8 @@ Trait Common {
if($this->transDepth==0) return false;
if(!$all) {
$this->exec("ROLLBACK TRANSACTION TO SAVEPOINT newssync_".($this->transDepth - 1));
// rollback to savepoint does not collpase the savepoint
$this->commit();
$this->transDepth -= 1;
if($this->transDepth==0) $this->exec("ROLLBACK TRANSACTION");
} else {
@ -41,6 +39,25 @@ Trait Common {
return true;
}
public function lock(): bool {
if($this->schemaVersion() < 1) return true;
if($this->isLocked()) return false;
$uuid = UUID::mintStr();
if(!$this->data->db->setSetting("lock", $uuid)) return false;
sleep(1);
if($this->data->db->getSetting("lock") != $uuid) return false;
return true;
}
public function unlock(): bool {
return $this->data->db->clearSetting("lock");
}
public function isLocked(): bool {
if($this->schemaVersion() < 1) return false;
return ($this->query("SELECT count(*) from newssync_settings where key = 'lock'")->getSingle() > 0);
}
public function prepare(string $query, string ...$paramType): Statement {
return $this->prepareArray($query, $paramType);
}

2
vendor/JKingWeb/NewsSync/Db/CommonPDO.php

@ -3,7 +3,7 @@ declare(strict_types=1);
namespace JKingWeb\NewsSync\Db;
Trait CommonPDO {
public function unsafeQuery(string $query): Result {
public function query(string $query): Result {
return new ResultPDO($this->db->query($query));
}

38
vendor/JKingWeb/NewsSync/Db/CommonSQLite3.php

@ -9,22 +9,42 @@ Trait CommonSQLite3 {
}
public function schemaVersion(string $schema = "main"): int {
return $this->unsafeQuery("PRAGMA $schema.user_version")->getSingle();
return $this->query("PRAGMA $schema.user_version")->getSingle();
}
public function update($to) {
public function update(int $to): bool {
$ver = $this->schemaVersion();
if(!$this->data->conf->dbSQLite3AutoUpd) throw new Update\Exception("manual", ['version' => $ver, 'driver_name' => $this->driverName()]);
if($ver >= $to) throw new Update\Exception("tooNew", ['difference' => ($ver - $to), 'current' => $ver, 'target' => $to, 'driver_name' => $this->driverName()]);
$sep = \DIRECTORY_SEPARATOR;
$path = \JKingWeb\NewsSync\BASE."sql".$sep."SQLite3".$sep;
$schemas = ["feeds", "main"];
$this->lock();
$this->begin();
for($a = $this->schemaVersion(); $a < $to; $a++) {
$file = $path.$a.".sql";
if(!file_exists($file)) $this->fail(new Exception("updateMissing", ['version' => $a, 'driver_name' => $this->driverName()]));
if(!is_readable($file)) $this->fail(new Exception("updateUnreadable", ['version' => $a, 'driver_name' => $this->driverName()]));
$sql = @file_get_contents($file);
if($sql===false) $this->fail(new Exception("updateUnusable", ['version' => $a, 'driver_name' => $this->driverName()]));
$this->exec($sql);
for($a = $ver; $a < $to; $a++) {
$this->begin();
foreach($schemas as $schema) {
try {
$file = $path.$a.".".$schema.".sql";
if(!file_exists($file)) throw new Update\Exception("missing", ['file' => $file, 'driver_name' => $this->driverName()]);
if(!is_readable($file)) throw new Update\Exception("unreadable", ['file' => $file, 'driver_name' => $this->driverName()]);
$sql = @file_get_contents($file);
if($sql===false) throw new Update\Exception("unusable", ['file' => $file, 'driver_name' => $this->driverName()]);
$this->exec($sql);
} catch(\Throwable $e) {
// undo any partial changes from the failed update
$this->rollback();
// commit any successful updates if updating by more than one version
$this->commit(true);
// throw the error received
throw $e;
}
}
$this->commit();
}
$this->unlock();
$this->commit();
return true;
}
public function exec(string $query): bool {

25
vendor/JKingWeb/NewsSync/Db/Driver.php

@ -2,14 +2,29 @@
declare(strict_types=1);
namespace JKingWeb\NewsSync\Db;
interface Driver {
static function create(\JKingWeb\NewsSync\Conf $conf, bool $install = false): Driver;
interface Driver {
// returns an instance of a class implementing this interface. Implemented as a static method so that classes may return their PDO equivalents instead of themselves
static function create(\JKingWeb\NewsSync\RuntimeData $data, bool $install = false): Driver;
// returns a human-friendly name for the driver (for display in installer, for example)
static function driverName(): string;
// returns the version of the scheme of the opened database; if uninitialized should return 0
function schemaVersion(): int;
// begin a real or synthetic transactions, with real or synthetic nesting
function begin(): bool;
function commit(): bool;
function rollback(): bool;
// commit either the latest or all pending nested transactions; use of this method should assume a partial commit is a no-op
function commit(bool $all = false): bool;
// rollback either the latest or all pending nested transactions; use of this method should assume a partial rollback will not work
function rollback(bool $all = false): bool;
// attempt to advise other processes that they should not attempt to access the database; used during live upgrades
function lock(): bool;
function unlock(): bool;
function isLocked(): bool;
// attempt to perform an in-place upgrade of the database schema; this may be a no-op which always throws an exception
function update(int $to): bool;
// execute one or more unsanitized SQL queries and return an indication of success
function exec(string $query): bool;
function unsafeQuery(string $query): Result;
// perform a single unsanitized query and return a result set
function query(string $query): Result;
// ready a prepared statement for later execution
function prepare(string $query, string ...$paramType): Statement;
}

16
vendor/JKingWeb/NewsSync/Db/DriverSQLite3.php

@ -6,17 +6,19 @@ class DriverSQLite3 implements Driver {
use Common, CommonSQLite3;
protected $db;
protected $data;
private function __construct(\JKingWeb\NewsSync\Conf $conf, bool $install = false) {
private function __construct(\JKingWeb\NewsSync\RuntimeData $data, bool $install = false) {
$this->data = $data;
// normalize the path
$path = $conf->dbSQLite3Path;
$path = $data->conf->dbSQLite3Path;
$sep = \DIRECTORY_SEPARATOR;
if(substr($path,-(strlen($sep))) != $sep) $path .= $sep;
$mainfile = $path."newssync-main.db";
$feedfile = $path."newssync-feeds.db";
// if the files exists (or we're initializing the database), try to open it and set initial options
try {
$this->db = new \SQLite3($mainfile, ($install) ? \SQLITE3_OPEN_READWRITE | \SQLITE3_OPEN_CREATE : \SQLITE3_OPEN_READWRITE, $conf->dbSQLite3Key);
$this->db = new \SQLite3($mainfile, ($install) ? \SQLITE3_OPEN_READWRITE | \SQLITE3_OPEN_CREATE : \SQLITE3_OPEN_READWRITE, $data->conf->dbSQLite3Key);
$this->db->enableExceptions(true);
$attach = "'".$this->db->escapeString($feedfile)."'";
$this->exec("ATTACH DATABASE $attach AS feeds");
@ -44,18 +46,18 @@ class DriverSQLite3 implements Driver {
unset($this->db);
}
static public function create(\JKingWeb\NewsSync\Conf $conf, bool $install = false): Driver {
static public function create(\JKingWeb\NewsSync\RuntimeData $data, bool $install = false): Driver {
// check to make sure required extensions are loaded
if(class_exists("SQLite3")) {
return new self($conf, $install);
return new self($data, $install);
} else if(class_exists("PDO") && in_array("sqlite",\PDO::getAvailableDrivers())) {
return new DriverSQLite3PDO($conf, $install);
return new DriverSQLite3PDO($data, $install);
} else {
throw new Exception("extMissing", self::driverName());
}
}
public function unsafeQuery(string $query): Result {
public function query(string $query): Result {
return new ResultSQLite3($this->db->query($query));
}

8
vendor/JKingWeb/NewsSync/Db/DriverSQLite3PDO.php

@ -7,7 +7,7 @@ class DriverSQLite3 implements Driver {
protected $db;
private function __construct(\JKingWeb\NewsSync\Conf $conf, bool $install = false) {
private function __construct(\JKingWeb\NewsSync\RuntimeData $data, bool $install = false) {
// FIXME: stub
}
@ -15,12 +15,12 @@ class DriverSQLite3 implements Driver {
// FIXME: stub
}
static public function create(\JKingWeb\NewsSync\Conf $conf, bool $install = false): Driver {
static public function create(\JKingWeb\NewsSync\RuntimeData $data, bool $install = false): Driver {
// check to make sure required extensions are loaded
if(class_exists("PDO") && in_array("sqlite",\PDO::getAvailableDrivers())) {
return new self($conf, $install);
return new self($data, $install);
} else if(class_exists("SQLite3")) {
return new DriverSQLite3($conf, $install);
return new DriverSQLite3($data, $install);
} else {
throw new Exception("extMissing", self::driverName());
}

9
vendor/JKingWeb/NewsSync/Db/Result.php

@ -2,8 +2,13 @@
declare(strict_types=1);
namespace JKingWeb\NewsSync\Db;
interface Result {
function __invoke(); // alias of get()
interface Result extends \Iterator {
function current();
function key();
function next();
function rewind();
function valid();
function get();
function getSingle();
}

40
vendor/JKingWeb/NewsSync/Db/ResultSQLite3.php

@ -4,6 +4,8 @@ namespace JKingWeb\NewsSync\Db;
class ResultSQLite3 implements Result {
protected $set;
protected $pos = 0;
protected $cur = null;
public function __construct(\SQLite3Result $resultObj) {
$this->set = $resultObj;
@ -14,17 +16,41 @@ class ResultSQLite3 implements Result {
unset($this->set);
}
public function __invoke() {
return $this->get();
public function valid() {
$this->cur = $this->set->fetchArray(\SQLITE3_ASSOC);
return ($this->cur !== false);
}
public function get() {
return $this->set->fetchArray(\SQLITE3_ASSOC);
public function next() {
$this->cur = null;
$this->pos += 1;
}
public function current() {
return $this->cur;
}
public function key() {
return $this->pos;
}
public function rewind() {
$this->pos = 0;
$this->cur = null;
$this->set->reset();
}
public function getSingle() {
$res = $this->get();
if($res===false) return null;
return array_shift($res);
$this->next();
if($this->valid()) {
$keys = array_keys($this->cur);
return $this->cur[array_shift($keys)];
}
return null;
}
public function get() {
$this->next();
return ($this->valid() ? $this->cur : null);
}
}

6
vendor/JKingWeb/NewsSync/Db/Update/Exception.php

@ -0,0 +1,6 @@
<?php
declare(strict_types=1);
namespace JKingWeb\NewsSync\Db\Update;
class Exception extends \JKingWeb\NewsSync\Db\Exception {
}

2
vendor/JKingWeb/NewsSync/RuntimeData.php

@ -10,7 +10,7 @@ class RuntimeData {
public function __construct(Conf $conf) {
$this->conf = $conf;
Lang::set($conf->lang);
$this->db = new Database($this->conf);
$this->db = new Database($this);
//$this->auth = new Authenticator($this);
}
}
Loading…
Cancel
Save