Tests for articles, with fixes

Related tables (categories, enclosures, label members) still need tests

locale/en.php

@@ -147,7 +147,7 @@ return [
             0 {Automatic updating of the {driver_name} database failed because it is already up to date with the requested version, {target}}
             other {Automatic updating of the {driver_name} database failed because its version, {current}, is newer than the requested version, {target}}
         }',
-    'Exception.JKingWeb/Arsse/Db/Exception.updateSchemaChange'             => 'Database schema version has changed since the application was started; application restart is required',
+    'Exception.JKingWeb/Arsse/Db/Exception.updateSchemaChange'             => 'Database schema version is newer than the application schema version',
     'Exception.JKingWeb/Arsse/Db/Exception.engineErrorGeneral'             => '{0}',
     // indicates programming error
     'Exception.JKingWeb/Arsse/Db/Exception.savepointStatusUnknown'         => 'Savepoint status code {0} not implemented',

sql/MySQL/7.sql

@@ -21,7 +21,8 @@ insert into arsse_articles_map(article, subscription)
     from arsse_articles as a join arsse_subscriptions as s using(feed)
     where feed in (
         select feed from (select feed, count(*) as count from arsse_subscriptions group by feed) as c where c.count > 1
-    );
+    )
+    order by a.id, s.id;
 insert into arsse_articles_map(article, subscription, id)
     select 
         a.id as article, 
@@ -32,24 +33,6 @@ insert into arsse_articles_map(article, subscription, id)
         select feed from (select feed, count(*) as count from arsse_subscriptions group by feed) as c where c.count = 1
     );
 
--- First create the subsidiary table to hold article contents
-create table arsse_article_contents(
--- contents of articles, which is typically large text
-    id bigint unsigned primary key,
-    content longtext,
-    foreign key(id) references arsse_articles(id) on delete cascade on update cascade
-) character set utf8mb4 collate utf8mb4_unicode_ci;
-insert into arsse_article_contents
-    select
-        m.id,
-        coalesce(a.content_scraped, a.content)
-    from arsse_articles_map as m
-    left join arsse_articles as a on a.id = m.article;
-
--- Drop the two content columns from the article table as they are no longer needed
-alter table arsse_articles drop column content;
-alter table arsse_articles drop column content_scraped;
-
 -- Add any new columns required for the articles table
 alter table arsse_articles add column subscription bigint unsigned;
 alter table arsse_articles add column "read" smallint not null default 0;
@@ -81,7 +64,7 @@ insert into arsse_articles(id,feed,subscription,"read",starred,hidden,published,
         coalesce(m.note,'')
     from arsse_articles_map as i
     left join arsse_articles as a on a.id = i.article
-    left join arsse_marks as m on a.id = m.article
+    left join arsse_marks as m on a.id = m.article and m.subscription = i.subscription
 on duplicate key update
     subscription = values(subscription),
     "read" = values("read"),
@@ -90,6 +73,24 @@ on duplicate key update
     marked = values(marked),
     note = values(note);
 
+-- Next create the subsidiary table to hold article contents
+create table arsse_article_contents(
+-- contents of articles, which is typically large text
+    id bigint unsigned primary key,
+    content longtext,
+    foreign key(id) references arsse_articles(id) on delete cascade on update cascade
+) character set utf8mb4 collate utf8mb4_unicode_ci;
+insert into arsse_article_contents
+    select
+        m.id,
+        case when s.scrape = 0 then a.content else coalesce(a.content_scraped, a.content) end
+    from arsse_articles_map as m
+    left join arsse_articles as a on a.id = m.article
+    left join arsse_subscriptions as s on s.id = m.subscription;
+
+-- Drop the two content columns from the article table as they are no longer needed
+alter table arsse_articles drop column content;
+alter table arsse_articles drop column content_scraped;
 
 -- Create one edition for each renumbered article
 insert into arsse_editions(article) select id from arsse_articles_map where id <> article;
@@ -124,6 +125,7 @@ insert into arsse_label_members
 
 -- Clean up the articles table: delete obsolete rows, add necessary constraints on new columns which could not be satisfied before inserting information, and drop the obsolete feed column
 delete from arsse_articles where id in (select article from arsse_articles_map where id <> article);
+delete from arsse_articles where subscription is null;
 alter table arsse_articles modify subscription bigint unsigned not null;
 alter table arsse_articles add foreign key(subscription) references arsse_subscriptions(id) on delete cascade on update cascade;
 alter table arsse_articles drop foreign key arsse_articles_ibfk_1;

sql/PostgreSQL/7.sql

@@ -18,7 +18,8 @@ insert into arsse_articles_map(article, subscription)
     from arsse_articles as a join arsse_subscriptions as s using(feed)
     where feed in (
         select feed from (select feed, count(*) as count from arsse_subscriptions group by feed) as c where c.count > 1
-    );
+    )
+    order by a.id, s.id;
 insert into arsse_articles_map(article, subscription, id)
     select 
         a.id as article, 
@@ -29,23 +30,6 @@ insert into arsse_articles_map(article, subscription, id)
         select feed from (select feed, count(*) as count from arsse_subscriptions group by feed) as c where c.count = 1
     );
 
--- First create the subsidiary table to hold article contents
-create table arsse_article_contents(
--- contents of articles, which is typically large text
-    id bigint primary key references arsse_articles(id) on delete cascade on update cascade,
-    content text collate "und-x-icu"
-);
-insert into arsse_article_contents
-    select
-        m.id,
-        coalesce(a.content_scraped, a.content)
-    from arsse_articles_map as m
-    left join arsse_articles as a on a.id = m.article;
-
--- Drop the two content columns from the article table as they are no longer needed
-alter table arsse_articles drop column content;
-alter table arsse_articles drop column content_scraped;
-
 -- Add any new columns required for the articles table
 alter table arsse_articles add column subscription bigint references arsse_subscriptions(id) on delete cascade on update cascade;
 alter table arsse_articles add column read smallint not null default 0;
@@ -77,7 +61,7 @@ with new_data as (
         coalesce(m.note,'') as note
     from arsse_articles_map as i
     left join arsse_articles as a on a.id = i.article
-    left join arsse_marks as m on a.id = m.article
+    left join arsse_marks as m on a.id = m.article and m.subscription = i.subscription
 )
 insert into arsse_articles(id,feed,subscription,read,starred,hidden,published,edited,modified,marked,url,title,author,guid,url_title_hash,url_content_hash,title_content_hash,note)
     select * from new_data
@@ -85,6 +69,24 @@ on conflict (id) do update set (subscription,read,starred,hidden,marked,note) =
     select subscription, read, starred, hidden, marked, note from new_data where id = excluded.id
 );
 
+-- Next create the subsidiary table to hold article contents
+create table arsse_article_contents(
+-- contents of articles, which is typically large text
+    id bigint primary key references arsse_articles(id) on delete cascade on update cascade,
+    content text collate "und-x-icu"
+);
+insert into arsse_article_contents
+    select
+        m.id,
+        case when s.scrape = 0 then a.content else coalesce(a.content_scraped, a.content) end
+    from arsse_articles_map as m
+    left join arsse_articles as a on a.id = m.article
+    left join arsse_subscriptions as s on s.id = m.subscription;
+
+-- Drop the two content columns from the article table as they are no longer needed
+alter table arsse_articles drop column content_scraped;
+alter table arsse_articles drop column content;
+
 -- Create one edition for each renumbered article
 insert into arsse_editions(article) select id from arsse_articles_map where id <> article;
 
@@ -117,6 +119,7 @@ insert into arsse_label_members
 
 -- Clean up the articles table: delete obsolete rows, add necessary constraints on new columns which could not be satisfied before inserting information, and drop the obsolete feed column
 delete from arsse_articles where id in (select article from arsse_articles_map where id <> article);
+delete from arsse_articles where subscription is null;
 alter table arsse_articles alter column subscription set not null;
 alter table arsse_articles drop column feed;
 

sql/SQLite3/7.sql

@@ -71,7 +71,7 @@ insert into arsse_articles_new
         coalesce(m.note,'')
     from arsse_articles_map as i
     left join arsse_articles as a on a.id = i.article
-    left join arsse_marks as m on a.id = m.article;
+    left join arsse_marks as m on a.id = m.article and m.subscription = i.subscription;
 
 -- Create a new table to hold article content
 create table arsse_article_contents(
@@ -82,9 +82,10 @@ create table arsse_article_contents(
 insert into arsse_article_contents
     select
         m.id,
-        coalesce(a.content_scraped, a.content)
+        case when s.scrape = 0 then a.content else coalesce(a.content_scraped, a.content) end
     from arsse_articles_map as m
-    left join arsse_articles as a on a.id = m.article;
+    left join arsse_articles as a on a.id = m.article
+    left join arsse_subscriptions as s on s.id = m.subscription;
 
 -- Create one edition for each renumbered article, and delete any editions for obsolete articles
 insert into arsse_editions(article) select id from arsse_articles_map where id <> article;

tests/cases/Db/BaseUpdate.php

@@ -226,6 +226,18 @@ QUERY_TEXT
                 (1, 'a', 1, '2002-02-02 00:02:03', '2002-02-02 00:05:03', 'User Title', 2, 1, null, 'keep', 'block', 0),
                 (4, 'a', 3, '2002-02-03 00:02:03', '2002-02-03 00:05:03', 'Rosy Title', 1, 0, 1337, 'meep', 'bloop', 0),
                 (6, 'c', 3, '2002-02-04 00:02:03', '2002-02-04 00:05:03', null,         2, 0, 4400, null,   null,    1);
+            insert into arsse_articles(id,feed,url,title,author,published,edited,modified,guid,url_title_hash,url_content_hash,title_content_hash,content_scraped,content) values
+                (1, 1, 'https://example.com/1', 'Article 1', 'John Doe', '2001-11-08 22:07:55', '2002-11-08 07:51:12', '2001-11-08 23:44:56', 'GUID1', 'UTHASH1', 'UCHASH1', 'TCHASH1', 'Scraped 1', 'Content 1'),
+                (2, 1, 'https://example.com/2', 'Article 2', 'Jane Doe', '2001-11-09 22:07:55', '2002-11-09 07:51:12', '2001-11-09 23:44:56', 'GUID2', 'UTHASH2', 'UCHASH2', 'TCHASH2', 'Scraped 2', 'Content 2'),
+                (3, 2, 'https://example.org/1', 'Article 3', 'John Doe', '2001-11-10 22:07:55', '2002-11-10 07:51:12', '2001-11-10 23:44:56', 'GUID3', 'UTHASH3', 'UCHASH3', 'TCHASH3', 'Scraped 3', 'Content 3'),
+                (4, 2, 'https://example.org/2', 'Article 4', 'Jane Doe', '2001-11-11 22:07:55', '2002-11-11 07:51:12', '2001-11-11 23:44:56', 'GUID4', 'UTHASH4', 'UCHASH4', 'TCHASH4', 'Scraped 4', 'Content 4'),
+                (5, 3, 'https://example.net/1', 'Article 5', 'Adam Doe', '2001-11-12 22:07:55', '2002-11-12 07:51:12', '2001-11-12 23:44:56', 'GUID5', 'UTHASH5', 'UCHASH5', 'TCHASH5', null,        'Content 5'),
+                (6, 3, 'https://example.net/2', 'Article 6', 'Evie Doe', '2001-11-13 22:07:55', '2002-11-13 07:51:12', '2001-11-13 23:44:56', 'GUID6', 'UTHASH6', 'UCHASH6', 'TCHASH6', 'Scraped 6', 'Content 6');
+            insert into arsse_marks(article,subscription,"read",starred,modified,note,hidden) values
+                (1, 1, 1, 1, '2002-11-08 00:37:22', 'Note 1', 0),
+                (5, 4, 1, 0, '2002-11-12 00:37:22', 'Note 5', 0),
+                (5, 6, 0, 1, '2002-12-12 00:37:22', '',       0),
+                (6, 6, 0, 0, '2002-12-13 00:37:22', 'Note 6', 1);
 QUERY_TEXT
         );
         $this->drv->schemaUpdate(8);
@@ -245,39 +257,30 @@ QUERY_TEXT
                     [4, "a", "https://example.net/rss", "Title 3", "Rosy Title", 1337, "2001-06-15 06:56:23", '"ack"', "2001-06-15 06:57:23", "2002-02-03 00:02:03", "https://example.net/", "2001-06-15 06:55:23", 44, "This error", 3,  12,   "2002-02-03 00:05:03", 1, 0, 0, "meep", "bloop"],
                     [6, "c", "https://example.net/rss", "Title 3", null,         4400, "2001-06-15 06:56:23", '"ack"', "2001-06-15 06:57:23", "2002-02-04 00:02:03", "https://example.net/", "2001-06-15 06:55:23", 44, "This error", 3,  12,   "2002-02-04 00:05:03", 2, 0, 1, null,   null],
                 ]
-            ]
+            ],
+            'arsse_articles' => [
+                'columns' => ["id", "subscription", "read", "starred", "hidden", "published", "edited", "modified", "marked", "url", "title", "author", "guid", "url_title_hash", "url_content_hash", "title_content_hash", "note"],
+                'rows'    => [
+                    [1,  1, 1, 1, 0, "2001-11-08 22:07:55", "2002-11-08 07:51:12", "2001-11-08 23:44:56", "2002-11-08 00:37:22", "https://example.com/1", "Article 1", "John Doe", "GUID1", "UTHASH1", "UCHASH1", "TCHASH1", "Note 1"],
+                    [2,  1, 0, 0, 0, "2001-11-09 22:07:55", "2002-11-09 07:51:12", "2001-11-09 23:44:56", null,                  "https://example.com/2", "Article 2", "Jane Doe", "GUID2", "UTHASH2", "UCHASH2", "TCHASH2", ""],
+                    [7,  4, 1, 0, 0, "2001-11-12 22:07:55", "2002-11-12 07:51:12", "2001-11-12 23:44:56", "2002-11-12 00:37:22", "https://example.net/1", "Article 5", "Adam Doe", "GUID5", "UTHASH5", "UCHASH5", "TCHASH5", "Note 5"],
+                    [8,  6, 0, 1, 0, "2001-11-12 22:07:55", "2002-11-12 07:51:12", "2001-11-12 23:44:56", "2002-12-12 00:37:22", "https://example.net/1", "Article 5", "Adam Doe", "GUID5", "UTHASH5", "UCHASH5", "TCHASH5", ""],
+                    [9,  4, 0, 0, 0, "2001-11-13 22:07:55", "2002-11-13 07:51:12", "2001-11-13 23:44:56", null,                  "https://example.net/2", "Article 6", "Evie Doe", "GUID6", "UTHASH6", "UCHASH6", "TCHASH6", ""],
+                    [10, 6, 0, 0, 1, "2001-11-13 22:07:55", "2002-11-13 07:51:12", "2001-11-13 23:44:56", "2002-12-13 00:37:22", "https://example.net/2", "Article 6", "Evie Doe", "GUID6", "UTHASH6", "UCHASH6", "TCHASH6", "Note 6"],
+                ]
+            ],
+            'arsse_article_contents' => [
+                'columns' => ["id", "content"],
+                'rows'    => [
+                    [1,  "Content 1"],
+                    [2,  "Content 2"],
+                    [7,  "Content 5"],
+                    [8,  "Content 5"],
+                    [9,  "Content 6"],
+                    [10, "Scraped 6"],
+                ]
+            ]    
         ];
         $this->compareExpectations($this->drv, $exp);
     }
-}
-
-/*
-
-CREATE TABLE arsse_articles(
--- metadata for entries in newsfeeds, including user state
-    id integer primary key,                                                                                 -- sequence number
-    subscription integer not null references arsse_subscriptions(id) on delete cascade on update cascade,   -- associated subscription
-    read int not null default 0,                                                                            -- whether the article has been read
-    starred int not null default 0,                                                                         -- whether the article is starred
-    hidden int not null default 0,                                                                          -- whether the article should be excluded from selection by default
-    published text,                                                                                         -- time of original publication
-    edited text,                                                                                            -- time of last edit by author
-    modified text not null default CURRENT_TIMESTAMP,                                                       -- time when article was last modified in database pursuant to an authorial edit
-    marked text,                                                                                            -- time at which an article was last modified by the user
-    url text,                                                                                               -- URL of article
-    title text collate nocase,                                                                              -- article title
-    author text collate nocase,                                                                             -- author's name
-    guid text,                                                                                              -- a nominally globally unique identifier for the article, from the feed
-    url_title_hash text not null,                                                                           -- hash of URL + title; used when checking for updates and for identification if there is no guid
-    url_content_hash text not null,                                                                         -- hash of URL + content + enclosure URL + enclosure content type; used when checking for updates and for identification if there is no guid
-    title_content_hash text not null,                                                                       -- hash of title + content + enclosure URL + enclosure content type; used when checking for updates and for identification if there is no guid
-    note text not null default ''                                                                           -- Tiny Tiny RSS freeform user note
-)
-
-CREATE TABLE arsse_article_contents(
--- contents of articles, which is typically large text
-    id integer primary key references arsse_articles(id) on delete cascade on update cascade,   -- reference to the article ID
-    content text                                                                                -- the contents
-)
-
-*/
+}