diff --git a/crowdsourcing/wpconvlib/src/conversation.ts b/crowdsourcing/wpconvlib/src/conversation.ts
index 1effdcdc..29199d89 100644
--- a/crowdsourcing/wpconvlib/src/conversation.ts
+++ b/crowdsourcing/wpconvlib/src/conversation.ts
@@ -15,7 +15,7 @@ limitations under the License.
*/
export interface Comment {
id: string;
- comment_type: 'MODIFICATION'|'ADDITION'|'CREATION'|'RESTORATION'|'DELETION';
+ type: 'MODIFICATION'|'ADDITION'|'CREATION'|'RESTORATION'|'DELETION';
content: string;
cleaned_content: string;
parent_id: string|null;
@@ -38,7 +38,32 @@ export interface Comment {
// the current snapshot.
latestVersion?: string | null; // id of the latest version of this comment if isPresent
// is false, otherwise id of self.
- dfs_index?: number // index according to Depth First Search of conv.
+ dfs_index?: number; // index according to Depth First Search of conv.
+ // Starting here are toxicity scores, since they don't exist in all datasets
+ // except in English, this fields are optional.
+ // TODO(yiqingh, ldixon) : decide on which scores to use, delete non-public
+ // ones from the spanner table.
+ RockV6_1_SEVERE_TOXICITY?: number | null,
+ RockV6_1_SEXUAL_ORIENTATION?: number | null,
+ RockV6_1_SEXUALLY_EXPLICIT?: number | null,
+ RockV6_1_TOXICITY?: number | null,
+ //TODO(yiqingh): change TOXICITY_IDENTITY_HATE to IDENTITY_ATTACK
+ RockV6_1_TOXICITY_IDENTITY_HATE?: number | null,
+ RockV6_1_TOXICITY_INSULT?: number | null,
+ RockV6_1_TOXICITY_OBSCENE?: number | null,
+ RockV6_1_TOXICITY_THREAT?: number | null,
+ Smirnoff_2_ATTACK_ON_AUTHOR?: number | null,
+ Smirnoff_2_ATTACK_ON_COMMENTER?: number | null,
+ Smirnoff_2_INCOHERENT?: number | null,
+ Smirnoff_2_INFLAMMATORY?: number | null,
+ Smirnoff_2_LIKELY_TO_REJECT?: number | null,
+ Smirnoff_2_OBSCENE?: number | null,
+ Smirnoff_2_OFF_TOPIC?: number | null,
+ Smirnoff_2_SPAM?: number | null,
+ Smirnoff_2_UNSUBSTANTIAL?: number | null,
+ // The following fields are used in displaying comments in viz app.
+ isCollapsed?: boolean,
+ rootComment?: Comment,
}
export interface Conversation { [id: string]: Comment }
@@ -169,15 +194,18 @@ export function htmlForComment(
// Walk down a comment and its children depth first.
export function walkDfsComments(
rootComment: Comment, f: (c: Comment) => void): void {
- const commentsHtml = [];
- let agenda: Comment[] = [];
- let nextComment: Comment|undefined = rootComment;
- while (nextComment) {
- if (nextComment.children) {
- agenda = agenda.concat(nextComment.children);
- }
+ const stack = [rootComment];
+ let nextComment: Comment|undefined = stack.pop();
+ while (nextComment){
f(nextComment);
- nextComment = agenda.pop();
+ if (nextComment.children){
+ if (nextComment.children.reverse()) {
+ for (const ch of nextComment.children) {
+ stack.push(ch)
+ }
+ }
+ }
+ nextComment = stack.pop();
}
}
@@ -207,7 +235,7 @@ export function makeParent(comment: Comment, parent: Comment) {
parent.children = [];
}
parent.children.push(comment);
- parent.children.sort(compareCommentOrderSmallestFirst);
+ parent.children.sort(compareCommentOrder);
comment.parent_id = parent.id;
comment.isRoot = false;
}
@@ -263,13 +291,13 @@ export function structureConversaton(conversation: Conversation): Comment|null {
// If the action is deletion, the content must have been deleted.
conversation[i].isPresent = true;
conversation[i].latestVersion = i;
- if (comment.comment_type === 'DELETION') {
+ if (comment.type === 'DELETION') {
conversation[i].isPresent = false;
conversation[i].latestVersion = null;
}
if (comment.parent_id !== null && comment.parent_id !== ''
&& conversation[comment.parent_id]) {
- if (comment.comment_type !== 'RESTORATION') {
+ if (comment.type !== 'RESTORATION') {
conversation[comment.parent_id].isPresent = false;
} else {
conversation[i].isPresent = false;
@@ -278,7 +306,7 @@ export function structureConversaton(conversation: Conversation): Comment|null {
}
// When a modification happens, the current comment will
// be replaced by the new version.
- if (comment.comment_type === 'MODIFICATION') {
+ if (comment.type === 'MODIFICATION') {
conversation[comment.parent_id].latestVersion = i;
}
}
@@ -286,7 +314,7 @@ export function structureConversaton(conversation: Conversation): Comment|null {
for (const i of ids) {
const comment = conversation[i];
- if (comment.comment_type === "RESTORATION" || comment.comment_type === "DELETION") {
+ if (comment.type === "RESTORATION" || comment.type === "DELETION") {
continue;
}
comment.isFinal = false;
diff --git a/crowdsourcing/wpconvlib/src/testdata/example_conversations.ts b/crowdsourcing/wpconvlib/src/testdata/example_conversations.ts
index d15c25f8..f6aa1e4e 100755
--- a/crowdsourcing/wpconvlib/src/testdata/example_conversations.ts
+++ b/crowdsourcing/wpconvlib/src/testdata/example_conversations.ts
@@ -7,7 +7,7 @@ export const example_conversation1: conversation.Conversation = {
"page_title": "User talk:Menchi/貯藏室二",
"user_text": "Vegie",
"content": ": Actually, based on my research, I did not find any unifid traslation. All the articles used the english term directly followed a parenthetical explaination. it's not convinent to use a chinese traslation, because it needs around 20 chinese characters to define the meaning of LGBT. I thought it is forbidden here using an english expression as title of a item. So, thanks your response. -- [[User:vegie|vegie]] 19:44 2004年5月31日\n",
- "comment_type": "ADDITION",
+ "type": "ADDITION",
"timestamp": "2004-05-31T10:44:26.000Z",
"parent_id": null,
"cleaned_content": " Actually, based on my research, I did not find any unifid traslation. All the articles used the english term directly followed a parenthetical explaination. it's not convinent to use a chinese traslation, because it needs around 20 chinese characters to define the meaning of LGBT. I thought it is forbidden here using an english expression as title of a item. So, thanks your response. 19:44 2004年5月31日",
@@ -19,7 +19,7 @@ export const example_conversation1: conversation.Conversation = {
"page_title": "User talk:Menchi/貯藏室二",
"user_text": "Sunzx",
"content": ":: 用英文题目应该没问题的. 不过最好在文章里面说明\"此条目名称尚未有统一的中文翻译\"之类的... --[[User:Sunzx|Sunzx]] ([[User_talk:Sunzx|Talk]])[[北京大学| ]] 10:46 2004年5月31日 (UTC)\n",
- "comment_type": "ADDITION",
+ "type": "ADDITION",
"timestamp": "2004-05-31T10:46:48.000Z",
"parent_id": null,
"cleaned_content": " 用英文题目应该没问题的. 不过最好在文章里面说明\"此条目名称尚未有统一的中文翻译\"之类的... 10:46 2004年5月31日 (UTC)",
@@ -32,7 +32,7 @@ export const example_conversation1: conversation.Conversation = {
"page_title": "User talk:Menchi/貯藏室二",
"user_text": "Vegie",
"content": "==Translation Problem==\n",
- "comment_type": "CREATION",
+ "type": "CREATION",
"timestamp": "2004-05-31T06:03:04.000Z",
"parent_id": null,
"cleaned_content": "Translation Problem",
@@ -45,7 +45,7 @@ export const example_conversation1: conversation.Conversation = {
"page_title": "User talk:Menchi/貯藏室二",
"user_text": "Vegie",
"content": "what if I could not find a proper expression for a specail english word, which is using by some chinese website, can i use the english word as title of item? for example, abbreviation of \"LGBT\". --[[User:Wing|Wing]] 15:02 2004年31日\n",
- "comment_type": "ADDITION",
+ "type": "ADDITION",
"timestamp": "2004-05-31T06:03:04.000Z",
"parent_id": null,
"cleaned_content": "what if I could not find a proper expression for a specail english word, which is using by some chinese website, can i use the english word as title of item? for example, abbreviation of \"LGBT\". 15:02 2004年31日",
@@ -58,7 +58,7 @@ export const example_conversation1: conversation.Conversation = {
"page_title": "User talk:Menchi/貯藏室二",
"user_text": "Vegie",
"content": "what if I could not find a proper expression for a specail english word, which is using by some chinese website, can i use the english word as title of item? for example, abbreviation of \"LGBT\". --[[User:Vegie|Vegie]] 15:02 2004年31日\n",
- "comment_type": "MODIFICATION",
+ "type": "MODIFICATION",
"timestamp": "2004-05-31T06:25:49.000Z",
"parent_id": "99858.17811.17782",
"cleaned_content": "what if I could not find a proper expression for a specail english word, which is using by some chinese website, can i use the english word as title of item? for example, abbreviation of \"LGBT\". 15:02 2004年31日",
@@ -71,7 +71,7 @@ export const example_conversation1: conversation.Conversation = {
"page_title": "User talk:Menchi/貯藏室二",
"user_text": "Menchi",
"content": ": If Chinese speakers didn't bother to invent a translate and most Chinese decided to just use the English term, then we can too. It sounds like it's the case with LGBT -- that there's no common Chinese translation? Then it's fine. It's kind of like how the word \"OK\" has been absorbed into daily spoken Chinese I guess. --[[User:Menchi|Menchi]] ([[User talk:Menchi|討論頁]])[[前247年|Â]] 08:14 2004年5月31日 (UTC)\n",
- "comment_type": "ADDITION",
+ "type": "ADDITION",
"timestamp": "2004-05-31T08:14:37.000Z",
"parent_id": null,
"cleaned_content": " If Chinese speakers didn't bother to invent a translate and most Chinese decided to just use the English term, then we can too. It sounds like it's the case with LGBT that there's no common Chinese translation? Then it's fine. It's kind of like how the word \"OK\" has been absorbed into daily spoken Chinese I guess. Â 08:14 2004年5月31日 (UTC)",
@@ -87,7 +87,7 @@ export const example_conversation2: conversation.Conversation = {
"content": "== Name == ",
"cleaned_content": "== Name == ",
"indentation": 0,
- "comment_type": "MODIFICATION",
+ "type": "MODIFICATION",
"user_text": "Adamdaley",
"timestamp": "2013-04-16 08:55:31 UTC",
"replyTo_id": "",
@@ -99,7 +99,7 @@ export const example_conversation2: conversation.Conversation = {
"content": " I edited it to the largest \"labor\" uprising and the largest \"organized armed uprising\" since the civil war. They were not in rebellion per se and the race riots of the 60's are clearly a larger uprising (I'm not too sure on armed).",
"cleaned_content": " I edited it to the largest \"labor\" uprising and the largest \"organized armed uprising\" since the civil war. They were not in rebellion per se and the race riots of the 60's are clearly a larger uprising (I'm not too sure on armed).",
"indentation": 0,
- "comment_type": "ADDITION",
+ "type": "ADDITION",
"user_text": "70.151.72.162",
"timestamp": "2015-08-07 17:03:18 UTC",
"replyTo_id": "550613551.0.0",
@@ -111,7 +111,7 @@ export const example_conversation2: conversation.Conversation = {
"content": "Edited it to say \"one of the largest, organized, and well-armed uprisings.\" This seemed like the best course of action. It may be settling but hopefully it resolves the dispute. \u2014\u00a0Preceding [WIKI_LINK: Wikipedia:Signatures@unsigned] comment added by ",
"cleaned_content": "Edited it to say \"one of the largest, organized, and well-armed uprisings.\" This seemed like the best course of action. It may be settling but hopefully it resolves the dispute. \u2014\u00a0Preceding [WIKI_LINK: Wikipedia:Signatures@unsigned] comment added by ",
"indentation": 0,
- "comment_type": "MODIFICATION",
+ "type": "MODIFICATION",
"user_text": "SineBot",
"timestamp": "2015-08-07 17:03:18 UTC",
"replyTo_id": "675014505.416.416",
@@ -127,7 +127,7 @@ export const example_conversation3: conversation.Conversation = {
"content": "foo comment 1 text",
"cleaned_content": "foo comment 1 text",
"indentation": 0,
- "comment_type": "ADDITION",
+ "type": "ADDITION",
"user_text": "70.151.72.162",
"timestamp": "2015-08-07 17:03:18 UTC",
"replyTo_id": null,
@@ -135,7 +135,7 @@ export const example_conversation3: conversation.Conversation = {
"page_title": "talk page title",
},
"675014505.20.416": {
- "comment_type": "ADDITION",
+ "type": "ADDITION",
"content": "foo comment 2 text",
"cleaned_content": "foo comment 2 text",
"indentation": 0,
@@ -150,7 +150,7 @@ export const example_conversation3: conversation.Conversation = {
export const example_conversation4: conversation.Conversation = {
"159766698.24.0": {
- "comment_type": "ADDITION",
+ "type": "ADDITION",
"content": "\nYour recent edit to [WIKI_LINK: Barton on Sea] ([EXTERNAL_LINK: diff]) was reverted by an automated bot. The edit was identified as adding either test edits, [WIKI_LINK: WP:VAND@vandalism], or [WIKI_LINK: WP:SPAM@link spam] to the page or having an inappropriate [WIKI_LINK: WP:ES@edit summary]. If you want to experiment, please use the preview button while editing or consider using the [WIKI_LINK: Wikipedia:Sandbox@sandbox]. If this revert was in error, please contact the bot operator. If you made an edit that removed a large amount of content, try doing smaller edits instead. Thanks! //",
"cleaned_content": "\nYour recent edit to [WIKI_LINK: Barton on Sea] ([EXTERNAL_LINK: diff]) was reverted by an automated bot. The edit was identified as adding either test edits, [WIKI_LINK: WP:VAND@vandalism], or [WIKI_LINK: WP:SPAM@link spam] to the page or having an inappropriate [WIKI_LINK: WP:ES@edit summary]. If you want to experiment, please use the preview button while editing or consider using the [WIKI_LINK: Wikipedia:Sandbox@sandbox]. If this revert was in error, please contact the bot operator. If you made an edit that removed a large amount of content, try doing smaller edits instead. Thanks! //",
"indentation": 0,
@@ -162,7 +162,7 @@ export const example_conversation4: conversation.Conversation = {
"user_text": "VoABot II",
},
"159766698.0.0": {
- "comment_type": "CREATION",
+ "type": "CREATION",
"content": "\n==Regarding your edits to [WIKI_LINK: Barton on Sea]:==",
"cleaned_content": "\n==Regarding your edits to [WIKI_LINK: Barton on Sea]:==",
"indentation": 0,
@@ -181,7 +181,7 @@ export const example_conversation5: conversation.Conversation = {
"page_title": "User talk:Menchi/貯藏室二",
"user_text": "Vegie",
"content": ": Actually, based on my research, I did not find any unifid traslation. All the articles used the english term directly followed a parenthetical explaination. it's not convinent to use a chinese traslation, because it needs around 20 chinese characters to define the meaning of LGBT. I thought it is forbidden here using an english expression as title of a item. So, thanks your response. -- [[User:vegie|vegie]] 19:44 2004年5月31日\n",
- "comment_type": "ADDITION",
+ "type": "ADDITION",
"timestamp": "2004-05-31T10:44:26.000Z",
"parent_id": null,
"cleaned_content": " Actually, based on my research, I did not find any unifid traslation. All the articles used the english term directly followed a parenthetical explaination. it's not convinent to use a chinese traslation, because it needs around 20 chinese characters to define the meaning of LGBT. I thought it is forbidden here using an english expression as title of a item. So, thanks your response. 19:44 2004年5月31日",
@@ -194,7 +194,7 @@ export const example_conversation5: conversation.Conversation = {
"page_title": "User talk:Menchi/貯藏室二",
"user_text": "Sunzx",
"content": ":: 用英文题目应该没问题的. 不过最好在文章里面说明\"此条目名称尚未有统一的中文翻译\"之类的... --[[User:Sunzx|Sunzx]] ([[User_talk:Sunzx|Talk]])[[北京大学| ]] 10:46 2004年5月31日 (UTC)\n",
- "comment_type": "ADDITION",
+ "type": "ADDITION",
"timestamp": "2004-05-31T10:46:48.000Z",
"parent_id": null,
"cleaned_content": " 用英文题目应该没问题的. 不过最好在文章里面说明\"此条目名称尚未有统一的中文翻译\"之类的... 10:46 2004年5月31日 (UTC)",
@@ -207,7 +207,7 @@ export const example_conversation5: conversation.Conversation = {
"page_title": "User talk:Menchi/貯藏室二",
"user_text": "Vegie",
"content": "==Translation Problem==\n",
- "comment_type": "CREATION",
+ "type": "CREATION",
"timestamp": "2004-05-31T06:03:04.000Z",
"parent_id": null,
"cleaned_content": "Translation Problem",
@@ -220,7 +220,7 @@ export const example_conversation5: conversation.Conversation = {
"page_title": "User talk:Menchi/貯藏室二",
"user_text": "Vegie",
"content": "what if I could not find a proper expression for a specail english word, which is using by some chinese website, can i use the english word as title of item? for example, abbreviation of \"LGBT\". --[[User:Wing|Wing]] 15:02 2004年31日\n",
- "comment_type": "DELETION",
+ "type": "DELETION",
"timestamp": "2004-05-31T06:25:49.000Z",
"parent_id": "99858.17811.17782",
"cleaned_content": "what if I could not find a proper expression for a specail english word, which is using by some chinese website, can i use the english word as title of item? for example, abbreviation of \"LGBT\". 15:02 2004年31日",
@@ -233,7 +233,7 @@ export const example_conversation5: conversation.Conversation = {
"page_title": "User talk:Menchi/貯藏室二",
"user_text": "Vegie",
"content": "what if I could not find a proper expression for a specail english word, which is using by some chinese website, can i use the english word as title of item? for example, abbreviation of \"LGBT\". --[[User:Wing|Wing]] 15:02 2004年31日\n",
- "comment_type": "ADDITION",
+ "type": "ADDITION",
"timestamp": "2004-05-31T06:03:04.000Z",
"parent_id": null,
"cleaned_content": "what if I could not find a proper expression for a specail english word, which is using by some chinese website, can i use the english word as title of item? for example, abbreviation of \"LGBT\". 15:02 2004年31日",
@@ -246,7 +246,7 @@ export const example_conversation5: conversation.Conversation = {
"page_title": "User talk:Menchi/貯藏室二",
"user_text": "Menchi",
"content": ": If Chinese speakers didn't bother to invent a translate and most Chinese decided to just use the English term, then we can too. It sounds like it's the case with LGBT -- that there's no common Chinese translation? Then it's fine. It's kind of like how the word \"OK\" has been absorbed into daily spoken Chinese I guess. --[[User:Menchi|Menchi]] ([[User talk:Menchi|討論頁]])[[前247年|Â]] 08:14 2004年5月31日 (UTC)\n",
- "comment_type": "ADDITION",
+ "type": "ADDITION",
"timestamp": "2004-05-31T08:14:37.000Z",
"parent_id": null,
"cleaned_content": " If Chinese speakers didn't bother to invent a translate and most Chinese decided to just use the English term, then we can too. It sounds like it's the case with LGBT that there's no common Chinese translation? Then it's fine. It's kind of like how the word \"OK\" has been absorbed into daily spoken Chinese I guess. Â 08:14 2004年5月31日 (UTC)",
@@ -262,7 +262,7 @@ export const example_conversation6: conversation.Conversation = {
"page_title": "User talk:Menchi/貯藏室二",
"user_text": "Vegie",
"content": "==Translation Problem==\n",
- "comment_type": "CREATION",
+ "type": "CREATION",
"timestamp": "2004-05-31T06:03:04.000Z",
"parent_id": null,
"cleaned_content": "Translation Problem",
@@ -275,7 +275,7 @@ export const example_conversation6: conversation.Conversation = {
"page_title": "User talk:Menchi/貯藏室二",
"user_text": "Vegie",
"content": "what if I could not find a proper expression for a specail english word, which is using by some chinese website, can i use the english word as title of item? for example, abbreviation of \"LGBT\". --[[User:Wing|Wing]] 15:02 2004年31日\n",
- "comment_type": "DELETION",
+ "type": "DELETION",
"timestamp": "2004-05-31T06:25:49.000Z",
"parent_id": "99858.17811.17782",
"cleaned_content": "what if I could not find a proper expression for a specail english word, which is using by some chinese website, can i use the english word as title of item? for example, abbreviation of \"LGBT\". 15:02 2004年31日",
@@ -288,7 +288,7 @@ export const example_conversation6: conversation.Conversation = {
"page_title": "User talk:Menchi/貯藏室二",
"user_text": "Vegie",
"content": "what if I could not find a proper expression for a specail english word, which is using by some chinese website, can i use the english word as title of item? for example, abbreviation of \"LGBT\". --[[User:Wing|Wing]] 15:02 2004年31日\n",
- "comment_type": "ADDITION",
+ "type": "ADDITION",
"timestamp": "2004-05-31T06:03:04.000Z",
"parent_id": null,
"cleaned_content": "what if I could not find a proper expression for a specail english word, which is using by some chinese website, can i use the english word as title of item? for example, abbreviation of \"LGBT\". 15:02 2004年31日",
@@ -301,7 +301,7 @@ export const example_conversation6: conversation.Conversation = {
"page_title": "User talk:Menchi/貯藏室二",
"user_text": "Vegie",
"content": "what if I could not find a proper expression for a specail english word, which is using by some chinese website, can i use the english word as title of item? for example, abbreviation of \"LGBT\". --[[User:Wing|Wing]] 15:02 2004年31日\n",
- "comment_type": "RESTORATION",
+ "type": "RESTORATION",
"timestamp": "2004-05-31T07:04:04.000Z",
"parent_id": "99858.17811.17782",
"cleaned_content": "what if I could not find a proper expression for a specail english word, which is using by some chinese website, can i use the english word as title of item? for example, abbreviation of \"LGBT\". 15:02 2004年31日",
diff --git a/view_conversations/conv-viewer-webapi/README.md b/view_conversations/conv-viewer-webapi/README.md
index f396b9f1..7c02199d 100644
--- a/view_conversations/conv-viewer-webapi/README.md
+++ b/view_conversations/conv-viewer-webapi/README.md
@@ -45,9 +45,9 @@ Before you can deploy, you need to:
1. Copy the `server_config.template.json` file to `build/config/server_config.json`.
2. In the `build/config/server_config.json` file, set these values:
- * `bigQueryProjectId` : The Google Cloud Project ID that contains the BigQuery database.
- * `bigQueryDataSetId` : The name of the dataset in the cloud project.
- * `bigQueryTable` : The name of the table that contains the conversations.
+ * `spannerProjectId` : The Google Cloud Project ID that contains the Spanner database.
+ * `spannerDataSetId` : The name of the dataset in the cloud project.
+ * `spannerTable` : The name of the table that contains the conversations.
TODO(ldixon): in future we'll move to using OAuth and project credentials.
diff --git a/view_conversations/conv-viewer-webapi/server_config.template.json b/view_conversations/conv-viewer-webapi/server_config.template.json
index 9a6e7a9a..77e3de3d 100644
--- a/view_conversations/conv-viewer-webapi/server_config.template.json
+++ b/view_conversations/conv-viewer-webapi/server_config.template.json
@@ -3,8 +3,8 @@
"isProduction": false,
"staticPath": "build/static",
- "cloudProjectId": "wikidetox-viz",
- "spannerInstanceId": "wikiconv",
- "spannerDatabaseName": "convdata",
- "spannerTableName": "zh_20180601_conv"
+ "cloudProjectId": "yourCloudProject",
+ "spannerInstanceId": "yourSpannerInstance",
+ "spannerDatabaseName": "yourSpannerDatabase",
+ "spannerTableName": "yourSpannerTable"
}
diff --git a/view_conversations/conv-viewer-webapi/src/db_types.ts b/view_conversations/conv-viewer-webapi/src/db_types.ts
index 061edf63..42bbec7f 100644
--- a/view_conversations/conv-viewer-webapi/src/db_types.ts
+++ b/view_conversations/conv-viewer-webapi/src/db_types.ts
@@ -128,6 +128,8 @@ class TimestampFieldHandler extends SpannerFieldHandler {
interface HandlerSet { [fieldName:string] : SpannerFieldHandler; };
interface ParsedOutput { [fieldName:string] : string | string[] | Date | number | null; };
+const scoreSubstrings = 'RockV6_1|Smirnoff_2';
+const scoreType = 'score';
const handlers : Array> = [
new StringFieldHandler('id'),
@@ -146,6 +148,7 @@ const handlers : Array> =
new StringFieldHandler('type'),
new IntFieldHandler('user_id'),
new StringFieldHandler('user_text'),
+ new FloatFieldHandler(scoreType),
];
function addHandler(inputHandlers : HandlerSet, handler : SpannerFieldHandler)
@@ -160,11 +163,12 @@ export function parseOutputRows(rows: spanner.ResultRow[]) : T[] {
for (const row of rows) {
const ret: { [fieldName:string] : string | string[] | Date | number | null } = {};
for (const field of row) {
- if(!(field.name in handlerSet)) {
+ const testname = new RegExp(scoreSubstrings).test(field.name) ? scoreType : field.name;
+ if(!(testname in handlerSet)) {
console.error(`Field ${field.name} does not have a handler and so cannot be interpreted.`);
break;
}
- ret[field.name] = handlerSet[field.name].fromSpannerResultField(field.value);
+ ret[field.name] = handlerSet[testname].fromSpannerResultField(field.value);
}
output.push(ret)
}
diff --git a/view_conversations/conv-viewer-webapi/src/routes.ts b/view_conversations/conv-viewer-webapi/src/routes.ts
index f46cb53c..bae9239f 100644
--- a/view_conversations/conv-viewer-webapi/src/routes.ts
+++ b/view_conversations/conv-viewer-webapi/src/routes.ts
@@ -33,22 +33,26 @@ const SEARCH_OP_TYPE =
// escaped quotes.
const SQL_SAFE_STRING =
new runtime_types.RuntimeStringType('SearchBy', /^[^"]+$/);
+const conversationIdIndex = '_by_conversation_id';
+const toxicityIndex = '_by_toxicity';
// TODO(ldixon): consider using passport auth
// for google cloud project.
export function setup(
app: express.Express, conf: config.Config,
spannerDatabase: spanner.Database) {
- let table = `\`${conf.spannerTableName}\``;
+ const table = `\`${conf.spannerTableName}\``;
app.get('/api/conversation-id/:conv_id', async (req, res) => {
try {
- let conv_id: runtime_types.ConversationId =
+ const conv_id: runtime_types.ConversationId =
runtime_types.ConversationId.assert(req.params.conv_id);
+ const index = conf.spannerTableName + conversationIdIndex;
// TODO remove outer try wrapper unless it get used.
+ // Force Spanner using particular indices to speed up performance.
const sqlQuery = `SELECT *
- FROM ${table}
+ FROM ${table}@{FORCE_INDEX=${index}}
WHERE conversation_id="${conv_id}"
LIMIT 100`;
// Query options list:
@@ -69,16 +73,52 @@ export function setup(
}
});
+ app.get('/api/toxicity/:score', async (req, res) => {
+ try {
+ const score: number = runtime_types.assertNumber(req.params.score);
+ const index = conf.spannerTableName + toxicityIndex;
+
+ // TODO remove outer try wrapper unless it get used.
+ const sqlQuery = `SELECT *
+ FROM ${table}@{FORCE_INDEX=${index}}
+ WHERE RockV6_1_TOXICITY <= ${score} and type != "DELETION"
+ ORDER BY RockV6_1_TOXICITY DESC
+ LIMIT 20`;
+ // Query options list:
+ // https://cloud.google.com/spanner/docs/getting-started/nodejs/#query_data_using_sql
+ const query: spanner.Query = {
+ sql: sqlQuery
+ };
+
+ await spannerDatabase.run(query).then(results => {
+ const rows = results[0];
+ res.status(httpcodes.OK).send(JSON.stringify(db_types.parseOutputRows(rows), null, 2));
+ });
+ } catch (e) {
+ console.error(`*** Failed: `, e);
+ res.status(httpcodes.INTERNAL_SERVER_ERROR).send(JSON.stringify({
+ error: e.message
+ }));
+ }
+ });
+
+
+
app.get('/api/comment-id/:comment_id', async (req, res) => {
try {
- let comment_id: runtime_types.CommentId =
+ const comment_id: runtime_types.CommentId =
runtime_types.CommentId.assert(req.params.comment_id);
+ const index = conf.spannerTableName + conversationIdIndex;
+
// TODO remove outer try wrapper unless it get used.
- const sqlQuery = `SELECT *
- FROM ${table}
- WHERE id="${comment_id}"
- LIMIT 100`;
+ // id field is unique.
+ const sqlQuery = `
+ SELECT conv_r.*
+ FROM ${table} conv_l
+ JOIN ${table}@{FORCE_INDEX=${index}} conv_r
+ ON conv_r.conversation_id = conv_l.conversation_id
+ WHERE conv_l.id = "${comment_id}" and conv_r.timestamp <= conv_l.timestamp`;
// Query options list:
// https://cloud.google.com/spanner/docs/getting-started/nodejs/#query_data_using_sql
const query: spanner.Query = {
@@ -87,7 +127,7 @@ export function setup(
await spannerDatabase.run(query).then(results => {
const rows = results[0];
- res.status(httpcodes.OK).send(db_types.parseOutputRows(rows));
+ res.status(httpcodes.OK).send(db_types.parseOutputRows(rows));
});
} catch (e) {
console.error(`*** Failed: `, e);
@@ -99,7 +139,7 @@ export function setup(
app.get('/api/revision-id/:rev_id', async (req, res) => {
try {
- let rev_id: runtime_types.RevisionId =
+ const rev_id: runtime_types.RevisionId =
runtime_types.RevisionId.assert(req.params.rev_id);
// TODO remove outer try wrapper unless it get used.
@@ -115,7 +155,7 @@ export function setup(
await spannerDatabase.run(query).then(results => {
const rows = results[0];
- res.status(httpcodes.OK).send(db_types.parseOutputRows(rows));
+ res.status(httpcodes.OK).send(db_types.parseOutputRows(rows));
});
} catch (e) {
console.error(`*** Failed: `, e);
@@ -127,7 +167,7 @@ export function setup(
app.get('/api/page-id/:page_id', async (req, res) => {
try {
- let page_id: runtime_types.PageId =
+ const page_id: runtime_types.PageId =
runtime_types.PageId.assert(req.params.page_id);
// TODO remove outer try wrapper unless it get used.
@@ -144,7 +184,7 @@ export function setup(
await spannerDatabase.run(query).then(results => {
const rows = results[0];
- res.status(httpcodes.OK).send(db_types.parseOutputRows(rows));
+ res.status(httpcodes.OK).send(db_types.parseOutputRows(rows));
});
} catch (e) {
console.error(`*** Failed: `, e);
@@ -156,13 +196,13 @@ export function setup(
app.get('/api/page-title/:page_title', async (req, res) => {
try {
- let page_title: runtime_types.PageTitleSearch =
+ const page_title: runtime_types.PageTitleSearch =
runtime_types.PageTitleSearch.assert(req.params.page_title);
// TODO remove outer try wrapper unless it get used.
const sqlQuery = `SELECT *
FROM ${table}
- WHERE page_title LIKE "${page_title}"
+ WHERE page_title = "${page_title}"
LIMIT 100`;
// Query options list:
// https://cloud.google.com/spanner/docs/getting-started/nodejs/#query_data_using_sql
@@ -173,7 +213,7 @@ export function setup(
await spannerDatabase.run(query).then(results => {
const rows = results[0];
- res.status(httpcodes.OK).send(db_types.parseOutputRows(rows));
+ res.status(httpcodes.OK).send(db_types.parseOutputRows(rows));
});
} catch (e) {
console.error(`*** Failed: `, e);
@@ -185,19 +225,19 @@ export function setup(
app.get('/api/search/:search_by/:search_op/:search_for', async (req, res) => {
if (!SEARCH_BY_TYPE.isValid(req.params.search_by)) {
- let errorMsg = `Error: Invalid searchBy string: ${req.params.search_by}`
+ const errorMsg = `Error: Invalid searchBy string: ${req.params.search_by}`
console.error(errorMsg);
res.status(httpcodes.BAD_REQUEST).send(JSON.stringify({error: errorMsg}));
return;
}
if (!SEARCH_OP_TYPE.isValid(req.params.search_op)) {
- let errorMsg = `Error: Invalid searchOp string: ${req.params.search_op}`
+ const errorMsg = `Error: Invalid searchOp string: ${req.params.search_op}`
console.error(errorMsg);
res.status(httpcodes.BAD_REQUEST).send(JSON.stringify({error: errorMsg}));
return;
}
if (!SQL_SAFE_STRING.isValid(req.params.search_for)) {
- let errorMsg = `Error: Invalid searchFor string: ${req.params.search_for}`
+ const errorMsg = `Error: Invalid searchFor string: ${req.params.search_for}`
console.error(errorMsg);
res.status(httpcodes.BAD_REQUEST).send(JSON.stringify({error: errorMsg}));
return;
@@ -218,7 +258,7 @@ export function setup(
await spannerDatabase.run(query).then(results => {
const rows = results[0];
- res.status(httpcodes.OK).send(db_types.parseOutputRows(rows));
+ res.status(httpcodes.OK).send(db_types.parseOutputRows(rows));
});
} catch (e) {
console.error(`*** Failed: `, e);
diff --git a/view_conversations/conv-viewer-webapi/src/runtime_types.ts b/view_conversations/conv-viewer-webapi/src/runtime_types.ts
index 123f9a7a..acba70c6 100644
--- a/view_conversations/conv-viewer-webapi/src/runtime_types.ts
+++ b/view_conversations/conv-viewer-webapi/src/runtime_types.ts
@@ -88,3 +88,10 @@ export let PageId = new RuntimeStringType('PageId', /^(\d+)$/);
export let PageTitleSearch =
new RuntimeStringType('PageTitleSearch', /^([^"]+)$/);
+export function assertNumber(score : number) {
+ if isNaN(score) {
+ throw new Error(`Wanted number but got: NaN.`);
+ }
+ return score;
+}
+
diff --git a/view_conversations/conv-viewer-webapp/src/app/app.component.css b/view_conversations/conv-viewer-webapp/src/app/app.component.css
index 4a257c7f..f114455b 100644
--- a/view_conversations/conv-viewer-webapp/src/app/app.component.css
+++ b/view_conversations/conv-viewer-webapp/src/app/app.component.css
@@ -32,4 +32,113 @@
.convid-text {
font-size: 80%;
color: #BBB;
-}
\ No newline at end of file
+}
+
+.debug-pre-output {
+ font-family: monospace;
+ white-space: pre;
+ border: 1px #AAA solid;
+}
+
+.indent {
+ margin-left: 1em;
+}
+
+hr {
+ display: block;
+ height: 1px;
+}
+
+last {
+ color: blue;
+}
+
+.hidden {
+ display: none;
+}
+
+.action {
+ display: flex;
+ flex-direction: row;
+ flex-wrap: nowrap;
+ align-items: flex-start;
+}
+
+.comment {
+ padding: 5px;
+ margin: 2px;
+ margin-left: 10px;
+ flex: auto;
+}
+
+.index {
+ border: 1px solid #CCC;
+ padding: 5px;
+ width: 2em;
+ border-radius: 5px;
+ text-align: center;
+}
+
+.finalcomment {
+ border: 2px dotted #88F;
+}
+
+.content{
+ padding: 5px;
+ background: #FFF;
+ width: 100%;
+ /* border: 1px solid #CCC; */
+}
+
+/* TODO: figure out how to make vertical line that
+ scales with the heigh of the containing element */
+.sep {
+}
+
+.whenandwho {
+ width: 100px;
+ text-align: right;
+ font-size: 80%;
+ flex: none;
+ display: block;
+
+ /* padding-right: 5px;
+ border-right: 2px solid #CCC; */
+ margin-right: 5px;
+}
+
+.author {
+ color: #44B;
+ font-weight: bold;
+}
+
+.anonAuthor {
+ color: Crimson;
+ font-weight: bold;
+}
+
+.timestamp {
+ font-size: 80%;
+ color: #BBB;
+}
+
+.collapseComment{
+ border: 1px solid #CCC;
+ padding: 5px;
+ background: #EEE;
+}
+
+.collapsible-action {
+ background-color: #EEE;
+ color: #444;
+ cursor: pointer;
+ padding: 18px;
+ width: 100%;
+ border-left: 6px solid cornflowerblue;
+ border-bottom: solid grey;
+ text-align: left;
+ outline: none;
+ font-size: 15px;
+}
+
+
diff --git a/view_conversations/conv-viewer-webapp/src/app/app.component.html b/view_conversations/conv-viewer-webapp/src/app/app.component.html
index 40a108a5..00150e85 100644
--- a/view_conversations/conv-viewer-webapp/src/app/app.component.html
+++ b/view_conversations/conv-viewer-webapp/src/app/app.component.html
@@ -4,8 +4,39 @@