Browse Source

Start of higher-level database interface

microsub
J. King 8 years ago
parent
commit
b2b71c4557
  1. 1
      .gitignore
  2. 11
      locale/en.php
  3. 111
      schema.sql
  4. 113
      sql/SQLite3/0.sql
  5. 45
      vendor/JKingWeb/NewsSync/Conf.php
  6. 38
      vendor/JKingWeb/NewsSync/Database.php
  7. 17
      vendor/JKingWeb/NewsSync/Db/Common.php
  8. 15
      vendor/JKingWeb/NewsSync/Db/CommonSQLite3.php
  9. 8
      vendor/JKingWeb/NewsSync/Db/DriverSQLite3.php
  10. 2
      vendor/JKingWeb/NewsSync/Db/ResultSQLite3.php
  11. 7
      vendor/JKingWeb/NewsSync/Db/Statement.php
  12. 11
      vendor/JKingWeb/NewsSync/Db/StatementSQLite3.php
  13. 5
      vendor/JKingWeb/NewsSync/Lang.php

1
.gitignore

@ -1,5 +1,6 @@
#dependencies
vendor/simplepie/*
vendor/JKingWeb/DrUUID/*
#temp files
cache/*

11
locale/en.php

@ -5,6 +5,7 @@ return [
"Exception.JKingWeb/NewsSync/Lang/Exception.fileUnreadable" => "Insufficient permissions to read language file \"{0}\"",
"Exception.JKingWeb/NewsSync/Lang/Exception.fileCorrupt" => "Language file \"{0}\" is corrupt or does not conform to expected format",
"Exception.JKingWeb/NewsSync/Lang/Exception.stringMissing" => "Message string \"{msgID}\" missing from all loaded language files ({fileList})",
"Exception.JKingWeb/NewsSync/Lang/Exception.stringInvalid" => "Message string \"{msgID}\" is not a valid ICU message string (language files loaded: {fileList})",
"Exception.JKingWeb/NewsSync/Conf/Exception.fileMissing" => "Configuration file \"{0}\" does not exist",
"Exception.JKingWeb/NewsSync/Conf/Exception.fileUnreadable" => "Insufficient permissions to read configuration file \"{0}\"",
@ -19,4 +20,14 @@ return [
"Exception.JKingWeb/NewsSync/Db/Exception.fileUnusable" => "Insufficient permissions to open database file \"{0}\" for reading or writing",
"Exception.JKingWeb/NewsSync/Db/Exception.fileUncreatable" => "Insufficient permissions to create new database file \"{0}\"",
"Exception.JKingWeb/NewsSync/Db/Exception.fileCorrupt" => "Database file \"{0}\" is corrupt or not a valid database",
"Exception.JKingWeb/NewsSync/Db/ExceptionUpdate.manual" =>
"{from_version, select,
0 {{driver_name} database is configured for manual updates and is not initialized; please populate the database with the base schema}
other {{driver_name} database is configured for manual updates; please update from schema version {from_version} to version {to_version}}
}",
"Exception.JKingWeb/NewsSync/Db/ExceptionUpdate.failed" =>
"{reason select,
missing {Automatic updating of the {driver_name} database failed because instructions for updating from version {from_version} are not available}
}",
"Exception.JKingWeb/NewsSync/Db/ExceptionUpdate.tooNew" => "Automatic updating of the {driver_name} database failed because its version, {current}, is newer than the requested version, {target}"
];

111
schema.sql

@ -1,111 +0,0 @@
begin;
create table main.newssync_settings(
key varchar(255) primary key not null, --
value varchar(255), --
type varchar(255) not null check(
type in('numeric','text','timestamp', 'date', 'time', 'bool')
) --
);
insert into main.newssync_settings values('schema_version',1,'int');
-- users
create table main.newssync_users(
id TEXT primary key not null, -- user id
password TEXT, -- password, salted and hashed; if using external authentication this would be blank
name TEXT, -- display name
avatar_type TEXT, -- avatar image's MIME content type
avatar_data BLOB, -- avatar image's binary data
admin boolean not null default 0 -- whether the user is an administrator
);
-- TT-RSS categories and ownCloud folders
create table main.newssync_categories(
id integer primary key not null, -- sequence number
owner TEXT not null references users(id) on delete cascade on update cascade, -- owner of category
parent integer, -- parent category id
folder integer not null, -- first-level category (ownCloud folder)
name TEXT not null, -- category name
modified datetime not null default CURRENT_TIMESTAMP, --
unique(owner,name,parent) -- cannot have multiple categories with the same name under the same parent for the same owner
);
-- newsfeeds, deduplicated
create table feeds.newssync_feeds(
id integer primary key not null, -- sequence number
url TEXT not null, -- URL of feed
title TEXT, -- default title of feed
favicon TEXT, -- URL of favicon
source TEXT, -- URL of site to which the feed belongs
updated datetime, -- time at which the feed was last fetched
modified datetime not null default CURRENT_TIMESTAMP, --
err_count integer not null default 0, -- count of successive times update resulted in error since last successful update
err_msg TEXT, -- last error message
username TEXT, -- HTTP authentication username
password TEXT, -- HTTP authentication password (this is stored in plain text)
unique(url,username,password) -- a URL with particular credentials should only appear once
);
-- users' subscriptions to newsfeeds, with settings
create table main.newssync_subscriptions(
id integer primary key not null, -- sequence number
owner TEXT not null references users(id) on delete cascade on update cascade, -- owner of subscription
feed integer not null references feeds(id) on delete cascade, -- feed for the subscription
added datetime not null default CURRENT_TIMESTAMP, -- time at which feed was added
modified datetime not null default CURRENT_TIMESTAMP, -- date at which subscription properties were last modified
title TEXT, -- user-supplied title
order_type int not null default 0, -- ownCloud sort order
pinned boolean not null default 0, -- whether feed is pinned (always sorts at top)
category integer not null references categories(id) on delete set null, -- TT-RSS category (nestable); the first-level category (which acts as ownCloud folder) is joined in when needed
unique(owner,feed) -- a given feed should only appear once for a given owner
);
-- entries in newsfeeds
create table feeds.newssync_articles(
id integer primary key not null, -- sequence number
feed integer not null references feeds(id) on delete cascade, -- feed for the subscription
url TEXT not null, -- URL of article
title TEXT, -- article title
author TEXT, -- author's name
published datetime, -- time of original publication
edited datetime, -- time of last edit
guid TEXT, -- GUID
content TEXT, -- content, as (X)HTML
modified datetime not null default CURRENT_TIMESTAMP, -- date when article properties were last modified
hash varchar(64) not null, -- ownCloud hash
fingerprint varchar(64) not null, -- ownCloud fingerprint
enclosures_hash varchar(64), -- hash of enclosures, if any; since enclosures are not uniquely identified, we need to know when they change
tags_hash varchar(64) -- hash of RSS/Atom categories included in article; since these categories are not uniquely identified, we need to know when they change
);
-- users' actions on newsfeed entries
create table main.newssync_subscription_articles(
id integer primary key not null,
article integer not null references articles(id) on delete cascade,
read boolean not null default 0,
starred boolean not null default 0,
modified datetime not null default CURRENT_TIMESTAMP
);
-- enclosures associated with articles
create table main.newssync_enclosures(
article integer not null references articles(id) on delete cascade,
url TEXT,
type varchar(255)
);
-- author labels ("categories" in RSS/Atom parlance) associated with newsfeed entries
create table main.newssync_tags(
article integer not null references articles(id) on delete cascade,
name TEXT
);
-- user labels associated with newsfeed entries
create table main.newssync_labels(
sub_article integer not null references subscription_articles(id) on delete cascade,
owner TEXT not null references users(id) on delete cascade on update cascade,
name TEXT
);
create index main.newssync_label_names on newssync_labels(name);
commit;

113
sql/SQLite3/0.sql

@ -0,0 +1,113 @@
-- newsfeeds, deduplicated
create table feeds.newssync_feeds(
id integer primary key not null, -- sequence number
url TEXT not null, -- URL of feed
title TEXT, -- default title of feed
favicon TEXT, -- URL of favicon
source TEXT, -- URL of site to which the feed belongs
updated datetime, -- time at which the feed was last fetched
modified datetime not null default CURRENT_TIMESTAMP, --
err_count integer not null default 0, -- count of successive times update resulted in error since last successful update
err_msg TEXT, -- last error message
username TEXT, -- HTTP authentication username
password TEXT, -- HTTP authentication password (this is stored in plain text)
unique(url,username,password) -- a URL with particular credentials should only appear once
);
-- entries in newsfeeds
create table feeds.newssync_articles(
id integer primary key not null, -- sequence number
feed integer not null references newssync_feeds(id) on delete cascade, -- feed for the subscription
url TEXT not null, -- URL of article
title TEXT, -- article title
author TEXT, -- author's name
published datetime, -- time of original publication
edited datetime, -- time of last edit
guid TEXT, -- GUID
content TEXT, -- content, as (X)HTML
modified datetime not null default CURRENT_TIMESTAMP, -- date when article properties were last modified
hash varchar(64) not null, -- ownCloud hash
fingerprint varchar(64) not null, -- ownCloud fingerprint
enclosures_hash varchar(64), -- hash of enclosures, if any; since enclosures are not uniquely identified, we need to know when they change
tags_hash varchar(64) -- hash of RSS/Atom categories included in article; since these categories are not uniquely identified, we need to know when they change
);
-- enclosures associated with articles
create table feeds.newssync_enclosures(
article integer not null references newssync_articles(id) on delete cascade,
url TEXT,
type varchar(255)
);
-- author labels ("categories" in RSS/Atom parlance) associated with newsfeed entries
create table feeds.newssync_tags(
article integer not null references newssync_articles(id) on delete cascade,
name TEXT
);
-- set version marker
pragma feeds.user_version = 1;
create table main.newssync_settings(
key varchar(255) primary key not null, --
value varchar(255), --
type varchar(255) not null check(
type in('int', 'numeric','text','timestamp', 'date', 'time', 'bool')
) --
);
-- users
create table main.newssync_users(
id TEXT primary key not null, -- user id
password TEXT, -- password, salted and hashed; if using external authentication this would be blank
name TEXT, -- display name
avatar_type TEXT, -- avatar image's MIME content type
avatar_data BLOB, -- avatar image's binary data
admin boolean not null default 0 -- whether the user is an administrator
);
-- TT-RSS categories and ownCloud folders
create table main.newssync_categories(
id integer primary key not null, -- sequence number
owner TEXT not null references newssync_users(id) on delete cascade on update cascade, -- owner of category
parent integer, -- parent category id
folder integer not null, -- first-level category (ownCloud folder)
name TEXT not null, -- category name
modified datetime not null default CURRENT_TIMESTAMP, --
unique(owner,name,parent) -- cannot have multiple categories with the same name under the same parent for the same owner
);
-- users' subscriptions to newsfeeds, with settings
create table main.newssync_subscriptions(
id integer primary key not null, -- sequence number
owner TEXT not null references newssync_users(id) on delete cascade on update cascade, -- owner of subscription
feed integer not null references newssync_feeds(id) on delete cascade, -- feed for the subscription
added datetime not null default CURRENT_TIMESTAMP, -- time at which feed was added
modified datetime not null default CURRENT_TIMESTAMP, -- date at which subscription properties were last modified
title TEXT, -- user-supplied title
order_type int not null default 0, -- ownCloud sort order
pinned boolean not null default 0, -- whether feed is pinned (always sorts at top)
category integer not null references newssync_categories(id) on delete set null, -- TT-RSS category (nestable); the first-level category (which acts as ownCloud folder) is joined in when needed
unique(owner,feed) -- a given feed should only appear once for a given owner
);
-- users' actions on newsfeed entries
create table main.newssync_subscription_articles(
id integer primary key not null,
article integer not null references newssync_articles(id) on delete cascade,
read boolean not null default 0,
starred boolean not null default 0,
modified datetime not null default CURRENT_TIMESTAMP
);
-- user labels associated with newsfeed entries
create table main.newssync_labels(
sub_article integer not null references newssync_subscription_articles(id) on delete cascade, --
owner TEXT not null references newssync_users(id) on delete cascade on update cascade,
name TEXT
);
create index main.newssync_label_names on newssync_labels(name);
-- set version marker
pragma main.user_version = 1;
insert into main.newssync_settings values('schema_version',1,'int');

45
vendor/JKingWeb/NewsSync/Conf.php

@ -3,28 +3,31 @@ declare(strict_types=1);
namespace JKingWeb\NewsSync;
class Conf {
public $lang = "en";
public $lang = "en";
public $dbClass = NS_BASE."Db\\DriverSQLite3";
public $dbSQLite3Path = BASE."db";
public $dbSQLite3Key = "";
public $dbPostgreSQLHost = "localhost";
public $dbPostgreSQLUser = "newssync";
public $dbPostgreSQLPass = "";
public $dbPostgreSQLPort = 5432;
public $dbPostgreSQLDb = "newssync";
public $dbPostgreSQLSchema = "";
public $dbMySQLHost = "localhost";
public $dbMySQLUser = "newssync";
public $dbMySQLPass = "";
public $dbMySQLPort = 3306;
public $dbMySQLDb = "newssync";
public $authClass = NS_BASE."Auth\\DriverInternal";
public $authPreferHTTP = false;
public $authProvision = false;
public $simplepieCache = BASE.".cache";
public $dbClass = NS_BASE."Db\\DriverSQLite3";
public $dbSQLite3Path = BASE."db";
public $dbSQLite3Key = "";
public $dbSQLite3AutoUpd = true;
public $dbPostgreSQLHost = "localhost";
public $dbPostgreSQLUser = "newssync";
public $dbPostgreSQLPass = "";
public $dbPostgreSQLPort = 5432;
public $dbPostgreSQLDb = "newssync";
public $dbPostgreSQLSchema = "";
public $dbPostgreSQLAutoUpd = false;
public $dbMySQLHost = "localhost";
public $dbMySQLUser = "newssync";
public $dbMySQLPass = "";
public $dbMySQLPort = 3306;
public $dbMySQLDb = "newssync";
public $dbMySQLAutoUpd = false;
public $authClass = NS_BASE."Auth\\DriverInternal";
public $authPreferHTTP = false;
public $authAutoAdd = false;
public $simplepieCache = BASE.".cache";
public function __construct(string $import_file = "") {

38
vendor/JKingWeb/NewsSync/Database.php

@ -3,11 +3,25 @@ declare(strict_types=1);
namespace JKingWeb\NewsSync;
class Database {
protected $drv;
const SCHEMA_VERSION = 1;
protected $db;
protected function clean_name(string $name): string {
return (string) preg_filter("[^0-9a-zA-Z_\.]", "", $name);
}
public function __construct(Conf $conf) {
$driver = $conf->dbClass;
$this->drv = $driver::create($conf);
$this->db = $driver::create($conf, INSTALL);
$ver = $this->db->schemaVersion();
if($ver < self::SCHEMA_VERSION) {
if($conf->dbSQLite3AutoUpd) {
$this->db->update(self::SCHEMA_VERSION);
} else {
throw new Db\Exception("updateManual", ['from_version' => $ver, 'to_version' => self::SCHEMA_VERSION, 'driver_name' => $this->db->driverName()]);
}
}
}
static public function listDrivers(): array {
@ -27,6 +41,24 @@ class Database {
}
public function schemaVersion(): int {
return $this->drv->schemaVersion();
return $this->db->schemaVersion();
}
public function userAdd(string $username, string $password = null, bool $admin = false): string {
$this->db->prepare("INSERT INTO newssync_users(id,password,admin) values(?,?,?)", "str", "str", "bool")->run($username,$password,$admin);
return $username;
}
public function subscriptionAdd(string $user, string $url, string $fetchUser = null, string $fetchPassword = null): int {
$this->db->begin();
$qFeed = $this->db->prepare("SELECT id from newssync_feeds where url = ? and username = ? and password = ?", "str", "str", "str");
if(is_null($id = $qFeed->run($url, $fetchUser, $fetchPassword)->getSingle())) {
$this->db->prepare("INSERT INTO newssync_feeds(url,username,password) values(?,?,?)", "str", "str", "str")->run($url, $fetchUser, $fetchPassword);
$id = $qFeed->run($url, $fetchUser, $fetchPassword)->getSingle();
}
$this->db->prepare("INSERT INTO newssync_subscriptions(owner,feed) values(?,?)", "str", "int")->run($user,$id);
$this->db->commit();
return 0;
}
}

17
vendor/JKingWeb/NewsSync/Db/Common.php

@ -3,14 +3,15 @@ declare(strict_types=1);
namespace JKingWeb\NewsSync\Db;
Trait Common {
protected $transDepth;
protected $transDepth = 0;
public function fail(\Throwable $e, bool $bool = false) {
$this->rollback($all);
throw $e;
}
public function begin(): bool {
if($this->transDepth==0) {
$this->exec("BEGIN TRANSACTION");
} else{
$this->exec("SAVEPOINT newssync_".$this->transDepth);
}
$this->exec("SAVEPOINT newssync_".($this->transDepth));
$this->transDepth += 1;
return true;
}
@ -18,7 +19,7 @@ Trait Common {
public function commit(bool $all = false): bool {
if($this->transDepth==0) return false;
if(!$all) {
$this->exec("RELEASE SAVEPOINT newssync_".$this->transDepth-1);
$this->exec("RELEASE SAVEPOINT newssync_".($this->transDepth - 1));
$this->transDepth -= 1;
} else {
$this->exec("COMMIT TRANSACTION");
@ -30,7 +31,7 @@ Trait Common {
public function rollback(bool $all = false): bool {
if($this->transDepth==0) return false;
if(!$all) {
$this->exec("ROLLBACK TRANSACTION TO SAVEPOINT newssync_".$this->transDepth-1);
$this->exec("ROLLBACK TRANSACTION TO SAVEPOINT newssync_".($this->transDepth - 1));
$this->transDepth -= 1;
if($this->transDepth==0) $this->exec("ROLLBACK TRANSACTION");
} else {

15
vendor/JKingWeb/NewsSync/Db/CommonSQLite3.php

@ -12,6 +12,21 @@ Trait CommonSQLite3 {
return $this->unsafeQuery("PRAGMA $schema.user_version")->getSingle();
}
public function update($to) {
$sep = \DIRECTORY_SEPARATOR;
$path = \JKingWeb\NewsSync\BASE."sql".$sep."SQLite3".$sep;
$this->begin();
for($a = $this->schemaVersion(); $a < $to; $a++) {
$file = $path.$a.".sql";
if(!file_exists($file)) $this->fail(new Exception("updateMissing", ['version' => $a, 'driver_name' => $this->driverName()]));
if(!is_readable($file)) $this->fail(new Exception("updateUnreadable", ['version' => $a, 'driver_name' => $this->driverName()]));
$sql = @file_get_contents($file);
if($sql===false) $this->fail(new Exception("updateUnusable", ['version' => $a, 'driver_name' => $this->driverName()]));
$this->exec($sql);
}
$this->commit();
}
public function exec(string $query): bool {
return (bool) $this->db->exec($query);
}

8
vendor/JKingWeb/NewsSync/Db/DriverSQLite3.php

@ -20,12 +20,12 @@ class DriverSQLite3 implements Driver {
$this->db->enableExceptions(true);
$attach = "'".$this->db->escapeString($feedfile)."'";
$this->exec("ATTACH DATABASE $attach AS feeds");
$this->exec("PRAGMA main.jounral_mode = wal");
$this->exec("PRAGMA feeds.jounral_mode = wal");
$this->exec("PRAGMA main.journal_mode = wal");
$this->exec("PRAGMA feeds.journal_mode = wal");
$this->exec("PRAGMA foreign_keys = yes");
} catch(\Throwable $e) {
// if opening the database doesn't work, check various pre-conditions to find out what the problem might be
foreach([$mainfile, $mainfile."-wal", $mainfile."-shm", $feedfile, $feedfile."-wal", $feedfile."-shm"] as $file) {
foreach([$mainfile, $feedfile] as $file) {
if(!file_exists($file)) {
if($install && !is_writable(dirname($file))) throw new Exception("fileUncreatable", dirname($file));
throw new Exception("fileMissing", $file);
@ -60,6 +60,6 @@ class DriverSQLite3 implements Driver {
}
public function prepareArray(string $query, array $paramTypes): Statement {
return new StatementSQLite3($query, $paramTypes);
return new StatementSQLite3($this->db->prepare($query), $paramTypes);
}
}

2
vendor/JKingWeb/NewsSync/Db/ResultSQLite3.php

@ -24,7 +24,7 @@ class ResultSQLite3 implements Result {
public function getSingle() {
$res = $this->get();
if($res===FALSE) return null;
if($res===false) return null;
return array_shift($res);
}
}

7
vendor/JKingWeb/NewsSync/Db/Statement.php

@ -3,7 +3,8 @@ declare(strict_types=1);
namespace JKingWeb\NewsSync\Db;
interface Statement {
function __invoke(...$bindings); // alias of run()
function run(...$bindings): Result;
function runArray(array $bindings): Result;
function __construct($st, array $bindings = null);
function __invoke(&...$values); // alias of run()
function run(&...$values): Result;
function runArray(array &$values): Result;
}

11
vendor/JKingWeb/NewsSync/Db/StatementSQLite3.php

@ -6,7 +6,7 @@ class StatementSQLite3 implements Statement {
protected $st;
protected $types;
public function __construct(\SQLite3Stmt $st, $bindings = null) {
public function __construct(\SQLite3Stmt $st, array $bindings = null) {
$this->st = $st;
$this->types = [];
foreach($bindings as $binding) {
@ -31,6 +31,11 @@ class StatementSQLite3 implements Statement {
case "text":
case "string":
case "str":
$this->types[] = \SQLITE3_TEXT; break;
case "bool":
case "boolean":
case "bit":
$this->types[] = \SQLITE3_INTEGER; break;
default:
$this->types[] = \SQLITE3_TEXT; break;
}
@ -59,8 +64,8 @@ class StatementSQLite3 implements Statement {
} else {
$type = (array_key_exists($a,$this->types)) ? $this->types[$a] : \SQLITE3_TEXT;
}
$st->bindParam($a+1, $values[$a], $type);
$this->st->bindParam($a+1, $values[$a], $type);
}
return new ResultSQLite3($st->execute());
return new ResultSQLite3($this->st->execute());
}
}

5
vendor/JKingWeb/NewsSync/Lang.php

@ -11,6 +11,7 @@ class Lang {
"Exception.JKingWeb/NewsSync/Lang/Exception.fileUnreadable" => "Insufficient permissions to read language file \"{0}\"",
"Exception.JKingWeb/NewsSync/Lang/Exception.fileCorrupt" => "Language file \"{0}\" is corrupt or does not conform to expected format",
"Exception.JKingWeb/NewsSync/Lang/Exception.stringMissing" => "Message string \"{msgID}\" missing from all loaded language files ({fileList})",
"Exception.JKingWeb/NewsSync/Lang/Exception.stringInvalid" => "Message string \"{msgID}\" is not a valid ICU message string (language files loaded: {fileList})",
];
static protected $requirementsMet = false;
@ -51,7 +52,9 @@ class Lang {
if(!array_key_exists($msgID, self::$strings)) throw new Lang\Exception("stringMissing", ['msgID' => $msgID, 'fileList' => implode(", ",self::$loaded)]);
// variables fed to MessageFormatter must be contained in array
if($vars !== null && !is_array($vars)) $vars = [$vars];
return \MessageFormatter::formatMessage(self::$locale, self::$strings[$msgID], $vars);
$msg = \MessageFormatter::formatMessage(self::$locale, self::$strings[$msgID], $vars);
if($msg===false) throw new Lang\Exception("stringInvalid", ['msgID' => $msgID, 'fileList' => implode(", ",self::$loaded)]);
return $msg;
}
static public function list(string $locale = ""): array {

Loading…
Cancel
Save