4.22.2

* Added: Option to display debugging information in chat when highlight filters are applied to a message.
* Changed: Treat a completely transparent highlight color as removing highlights from a message. This allows rules such as preventing messages from bots (Bot Badge) from being highlighted.
* Fixed: Detect an invalid IndexedDB database and recreate it when necessary.
* API Added: `applyHighlight(msg, priority, color, reason)` method for tokenizers.

package.json

@@ -1,7 +1,7 @@
 {
   "name": "frankerfacez",
   "author": "Dan Salvato LLC",
-  "version": "4.22.1",
+  "version": "4.22.2",
   "description": "FrankerFaceZ is a Twitch enhancement suite.",
   "private": true,
   "license": "Apache-2.0",

src/modules/chat/index.js

@@ -8,7 +8,8 @@ import dayjs from 'dayjs';
 
 import Module from 'utilities/module';
 import {createElement, ManagedStyle} from 'utilities/dom';
-import {timeout, has, glob_to_regex, escape_regex, split_chars, deep_copy} from 'utilities/object';
+import {timeout, has, glob_to_regex, escape_regex, split_chars} from 'utilities/object';
+import {Color} from 'utilities/color';
 
 import Badges from './badges';
 import Emotes from './emotes';
@@ -258,6 +259,16 @@ export default class Chat extends Module {
 			}
 		});
 
+		this.settings.add('chat.filtering.debug', {
+			default: false,
+			ui: {
+				path: 'Chat > Filtering > General >> Behavior',
+				title: 'Display a list of highlight reasons on every chat message for debugging.',
+				component: 'setting-check-box',
+				force_seen: true
+			}
+		});
+
 		this.settings.addUI('chat.filtering.pad-bottom', {
 			path: 'Chat > Filtering > Highlight',
 			sort: 1000,
@@ -414,7 +425,7 @@ export default class Chat extends Module {
 			type: 'array_merge',
 			always_inherit: true,
 			ui: {
-				path: 'Chat > Filtering > Highlight @{"description": "These settings allow you to highlight messages in chat based on their contents. Setting priorities on rules allows you to determine which highlight color should be applied if a message matches multiple rules. Rules with a higher priority take priority over rules with lower priorities."} >> Users',
+				path: 'Chat > Filtering > Highlight @{"description": "These settings allow you to highlight messages in chat based on their contents. Setting priorities on rules allows you to determine which highlight color should be applied if a message matches multiple rules. Rules with a higher priority take priority over rules with lower priorities.\\n\\nYou can also create a rule that removes highlights from messages, preventing lower priority rules from highlighting them, by setting a color with an alpha value of zero. Example: `#00000000`"} >> Users',
 				component: 'basic-terms',
 				colored: true,
 				words: false,
@@ -433,10 +444,10 @@ export default class Chat extends Module {
 				const temp = new Map;
 
 				for(const item of val) {
-					const c = item.c || null,
+					const p = item.p || 0,
-						p = item.p || 0,
 						t = item.t;
 
+					let c = item.c || null;
 					let v = item.v;
 
 					if ( t === 'glob' )
@@ -460,6 +471,12 @@ export default class Chat extends Module {
 						temp.set(p, colors);
 					}
 
+					if ( c ) {
+						const test = Color.RGBA.fromCSS(c);
+						if ( ! test || ! test.a )
+							c = false;
+					}
+
 					if ( colors.has(c) )
 						colors.get(c).push(v);
 					else {
@@ -558,10 +575,16 @@ export default class Chat extends Module {
 				const badges = new Map;
 
 				for(const item of val) {
-					const c = item.c || null,
+					let c = item.c || null;
-						p = item.p || 0,
+					const p = item.p || 0,
 						v = item.v;
 
+					if ( c ) {
+						const test = Color.RGBA.fromCSS(c);
+						if ( ! test || ! test.a )
+							c = false;
+					}
+
 					const existing = badges.get(v);
 					if ( ! existing || existing[0] < p || (c && ! existing[1] && existing[0] <= p) )
 						badges.set(v, [p, c]);
@@ -633,13 +656,13 @@ export default class Chat extends Module {
 					has_non = false;
 
 				for(const item of val) {
-					const c = item.c || null,
+					const p = item.p || 0,
-						p = item.p || 0,
 						highlight = can_highlight && (has(item, 'h') ? item.h : true),
 						sensitive = item.s,
 						t = item.t,
 						word = has(item, 'w') ? item.w : t !== 'raw';
 
+					let c = item.c || null;
 					let v = item.v;
 
 					if ( t === 'glob' )
@@ -668,6 +691,12 @@ export default class Chat extends Module {
 						temp.set(p, colors);
 					}
 
+					if ( c ) {
+						const test = Color.RGBA.fromCSS(c);
+						if ( ! test || ! test.a )
+							c = false;
+					}
+
 					let data = colors.get(c);
 					if ( ! data )
 						colors.set(c, data = [
@@ -1449,6 +1478,43 @@ export default class Chat extends Module {
 	}
 
 
+	applyHighlight(msg, priority, color, reason, use_null_color = false) { // eslint-disable-line class-methods-use-this
+		if ( ! msg )
+			return msg;
+
+		const is_null = msg.mention_priority == null,
+			matched = is_null || priority >= msg.mention_priority,
+			higher = is_null || priority > msg.mention_priority;
+
+		if ( msg.filters )
+			msg.filters.push(`${reason}(${priority})${matched && color === false ? ':remove' : color ? `:${color}` : ''}`);
+
+		if ( matched ) {
+			msg.mention_priority = priority;
+
+			if ( color === false ) {
+				if ( higher ) {
+					msg.mentioned = false;
+					msg.clear_priority = priority;
+					msg.mention_color = msg.highlights = null;
+				}
+
+				return;
+			}
+
+			msg.mentioned = true;
+			if ( ! msg.highlights )
+				msg.highlights = new Set;
+		}
+
+		if ( msg.mentioned && (msg.clear_priority == null || priority >= msg.clear_priority) ) {
+			msg.highlights.add(reason);
+			if ( (color || use_null_color) && (higher || ! msg.mention_color) )
+				msg.mention_color = color;
+		}
+	}
+
+
 	standardizeMessage(msg) { // eslint-disable-line class-methods-use-this
 		if ( ! msg )
 			return msg;

src/modules/chat/tokenizers.jsx

@@ -19,6 +19,35 @@ const EMOTE_CLASS = 'chat-image chat-line__message--emote',
 	MENTION_REGEX = /^(['"*([{<\\/]*)(@)((?:[^\u0000-\u007F]|[\w-])+)(?:\b|$)/; // eslint-disable-line no-control-regex
 
 
+export const FilterTester = {
+	type: 'filter_test',
+	priority: 1000,
+
+	render(token, createElement) {
+		if ( ! token.msg.filters?.length )
+			return null;
+
+		return (<div class="ffz-pill tw-mg-l-1">
+			{ token.msg.filters.join(', ') }
+		</div>);
+	},
+
+	process(tokens, msg) {
+		if ( ! tokens || ! tokens.length || ! this.context.get('chat.filtering.debug') )
+			return tokens;
+
+		msg.filters = [];
+
+		tokens.push({
+			type: 'filter_test',
+			msg
+		});
+
+		return tokens;
+	}
+}
+
+
 // ============================================================================
 // Links
 // ============================================================================
@@ -406,14 +435,8 @@ export const Mentions = {
 						recipient: rlower
 					});
 
-					if ( mentioned ) {
+					if ( mentioned )
-						(msg.highlights = (msg.highlights || new Set())).add('mention');
+						this.applyHighlight(msg, priority, null, 'mention', true);
-						msg.mentioned = true;
-						if ( msg.color_priority == null || priority > msg.color_priority ) {
-							msg.mention_color = null;
-							msg.color_priority = priority;
-						}
-					}
 
 					// Push the remaining text from the token.
 					text.push(segment.substr(match[0].length));
@@ -449,17 +472,8 @@ export const UserHighlights = {
 
 		const u = msg.user;
 		for(const [priority, color, regex] of list) {
-			if ( regex.test(u.login) || regex.test(u.displayName) ) {
+			if ( regex.test(u.login) || regex.test(u.displayName) )
-				(msg.highlights = (msg.highlights || new Set())).add('user');
+				this.applyHighlight(msg, priority, color, 'user');
-				msg.mentioned = true;
-				if ( color ) {
-					if ( msg.color_priority == null || priority > msg.color_priority ) {
-						msg.mention_color = color;
-						msg.color_priority = priority;
-					}
-					return tokens;
-				}
-			}
 		}
 
 		return tokens;
@@ -539,16 +553,8 @@ export const BadgeStuff = {
 
 			if ( highlights && highlights.has(badge) ) {
 				const details = highlights.get(badge);
-				(msg.highlights = (msg.highlights || new Set())).add('badge');
+				if ( Array.isArray(details) && details.length > 1 )
-				msg.mentioned = true;
+					this.applyHighlight(msg, details[0], details[1], 'badge');
-				if ( details[1] ) {
-					if ( msg.color_priority == null || details[0] > msg.color_priority ) {
-						msg.mention_color = details[1];
-						msg.color_priority = details[0];
-					}
-					if ( ! list )
-						return tokens;
-				}
 			}
 		}
 
@@ -612,7 +618,7 @@ export const CustomHighlights = {
 		let had_match = false;
 		if ( data.non ) {
 			for(const [priority, color, regexes] of data.non) {
-				if ( had_match && msg.color_priority != null && msg.color_priority > priority )
+				if ( had_match && msg.mention_priority != null && msg.mention_priority > priority )
 					break;
 
 				let matched = false;
@@ -626,17 +632,8 @@ export const CustomHighlights = {
 				}
 
 				if ( matched ) {
-					(msg.highlights = (msg.highlights || new Set())).add('term');
-					msg.mentioned = true;
 					had_match = true;
-					if ( color ) {
+					this.applyHighlight(msg, priority, color, 'term');
-						if ( msg.color_priority == null || priority > msg.color_priority ) {
-							msg.mention_color = color;
-							msg.color_priority = priority;
-						}
-
-						break;
-					}
 				}
 			}
 		}
@@ -676,12 +673,7 @@ export const CustomHighlights = {
 					if ( idx !== nix )
 						out.push({type: 'text', text: text.slice(idx, nix)});
 
-					(msg.highlights = (msg.highlights || new Set())).add('term');
+					this.applyHighlight(msg, priority, color, 'term');
-					msg.mentioned = true;
-					if ( color && (msg.color_priority == null || priority > msg.color_priority) ) {
-						msg.mention_color = color;
-						msg.color_priority = priority;
-					}
 
 					out.push({
 						type: 'highlight',

src/modules/main_menu/components/badge-term-editor.vue

@@ -44,11 +44,20 @@
 				</select>
 			</div>
 			<div v-if="colored" class="tw-flex-shrink-0 tw-mg-r-05">
-				<color-picker v-if="editing" v-model="edit_data.c" :nullable="true" :show-input="false" />
+				<color-picker
-				<div v-else-if="term.c" class="ffz-color-preview">
+					v-if="editing"
+					v-model="edit_data.c"
+					:nullable="true"
+					:show-input="false"
+					:tooltip="t('settings.term.color.tip', 'Color')"
+				/>
+				<div v-else-if="term.c" class="ffz-color-preview tw-relative tw-tooltip__container">
 					<figure :style="`background-color: ${term.c}`">
 						&nbsp;
 					</figure>
+					<div class="tw-tooltip tw-tooltip--down tw-tooltip--align-right">
+						{{ t('settings.term.color.tip', 'Color') }}
+					</div>
 				</div>
 			</div>
 			<div

src/modules/main_menu/components/term-editor.vue

@@ -27,14 +27,6 @@
 					autocorrect="off"
 				>
 			</div>
-			<div v-if="colored" class="tw-flex-shrink-0 tw-mg-l-05">
-				<color-picker v-if="editing" v-model="edit_data.c" :nullable="true" :show-input="false" />
-				<div v-else-if="term.c" class="ffz-color-preview">
-					<figure :style="`background-color: ${term.c}`">
-						&nbsp;
-					</figure>
-				</div>
-			</div>
 			<div class="tw-flex-shrink-0 tw-mg-x-05">
 				<span v-if="! editing">{{ term_type }}</span>
 				<select
@@ -56,6 +48,23 @@
 					</option>
 				</select>
 			</div>
+			<div v-if="colored" class="tw-flex-shrink-0 tw-mg-r-05">
+				<color-picker
+					v-if="editing"
+					v-model="edit_data.c"
+					:nullable="true"
+					:show-input="false"
+					:tooltip="t('settings.term.color.tip', 'Color')"
+				/>
+				<div v-else-if="term.c" class="ffz-color-preview tw-relative tw-tooltip__container">
+					<figure :style="`background-color: ${term.c}`">
+						&nbsp;
+					</figure>
+					<div class="tw-tooltip tw-tooltip--down tw-tooltip--align-right">
+						{{ t('settings.term.color.tip', 'Color') }}
+					</div>
+				</div>
+			</div>
 			<div
 				v-if="priority"
 				:class="editing ? 'tw-mg-r-05' : 'tw-mg-x-05'"

src/settings/providers.js

@@ -382,8 +382,14 @@ export class IndexedDBProvider extends SettingsProvider {
 				const db = request.result;
 
 				// We have a database, but does it contain anything?
-				const trx = db.transaction(['settings'], 'readonly'),
+				let store;
+				try {
+					const trx = db.transaction(['settings'], 'readonly');
 					store = trx.objectStore('settings');
+				} catch(err) {
+					// This indicates a bad database.
+					s(false);
+				}
 
 				const r2 = store.getAllKeys();
 
@@ -610,7 +616,7 @@ export class IndexedDBProvider extends SettingsProvider {
 
 	// IDB Interaction
 
-	getDB() {
+	getDB(second = false) {
 		if ( this.db )
 			return Promise.resolve(this.db);
 
@@ -651,6 +657,57 @@ export class IndexedDBProvider extends SettingsProvider {
 				if ( this.manager )
 					this.manager.log.info(`Database opened. (After: ${(performance.now() - this._start_time).toFixed(5)}ms)`);
 				this.db = request.result;
+
+				try {
+					const trx = this.db.transaction(['settings', 'blobs'], 'readonly');
+					trx.objectStore('settings');
+					trx.objectStore('blobs');
+				} catch(err) {
+					// If this is an error, the database is in an invalid state.
+					if ( this.manager )
+						this.manager.log.error(`Database in invalid state.`, err);
+
+					try {
+						this.db.close();
+					} catch(e) { /* no-op */ }
+
+					this._onFinish(request);
+					this.db = null;
+
+					if ( second )
+						done(false, err);
+
+					else {
+						// Try deleting the database and making a new one.
+						const delreq = window.indexedDB.deleteDatabase('FFZ');
+						this._onStart(delreq);
+
+						delreq.onerror = e => {
+							if ( this.manager )
+								this.manager.log.error('Error deleting invalid database.', e);
+							done(false, e);
+							this._onFinish(delreq);
+						}
+
+						delreq.onsuccess = () => {
+							if ( this.manager )
+								this.manager.log.info('Deleted invalid database.');
+
+							this._onFinish(delreq);
+							this._listeners = null;
+							this.getDB(true).then(result => {
+								for(const pair of listeners)
+									pair[0](result);
+							}).catch(err => {
+								for(const pair of listeners)
+									pair[1](err);
+							});
+						}
+					}
+
+					return;
+				}
+
 				done(true, this.db);
 				this._onFinish(request);
 			}

src/sites/twitch-twilight/modules/chat/line.js

@@ -83,6 +83,7 @@ export default class ChatLine extends Module {
 		this.chat.context.on('changed:chat.filtering.process-own', this.updateLines, this);
 		this.chat.context.on('changed:chat.timestamp-format', this.updateLines, this);
 		this.chat.context.on('changed:chat.filtering.mention-priority', this.updateLines, this);
+		this.chat.context.on('changed:chat.filtering.debug', this.updateLines, this);
 		this.chat.context.on('changed:__filter:highlight-terms', this.updateLines, this);
 		this.chat.context.on('changed:__filter:highlight-users', this.updateLines, this);
 		this.chat.context.on('changed:__filter:highlight-badges', this.updateLines, this);
@@ -1061,7 +1062,7 @@ other {# messages were deleted by a moderator.}
 			if ( msg ) {
 				msg.ffz_tokens = null;
 				msg.ffz_badges = null;
-				msg.highlights = msg.mentioned = msg.mention_color = msg.color_priority = null;
+				msg.highlights = msg.mentioned = msg.mention_color = msg.mention_priority = msg.clear_priority = null;
 			}
 		}
 
@@ -1070,7 +1071,7 @@ other {# messages were deleted by a moderator.}
 			if ( msg ) {
 				msg.ffz_tokens = null;
 				msg.ffz_badges = null;
-				msg.highlights = msg.mentioned = msg.mention_color = msg.color_priority = null;
+				msg.highlights = msg.mentioned = msg.mention_color = msg.mention_priority = msg.clear_priority = null;
 			}
 		}
 

src/std-components/color-picker.vue

@@ -31,7 +31,11 @@
 				<chrome-picker :disable-alpha="! alpha" :value="colors" @input="onPick" />
 			</div>
 		</div>
-		<div v-else class="tw-relative">
+		<div
+			v-else
+			:class="{'tw-tooltip__container': hasTooltip}"
+			class="tw-relative"
+		>
 			<button
 				class="tw-button tw-button--text ffz-color-preview"
 				@click="togglePicker"
@@ -51,6 +55,15 @@
 			>
 				<chrome-picker :disable-alpha="! alpha" :value="colors" @input="onPick" />
 			</div>
+			<div
+				v-if="! open && hasTooltip"
+				class="tw-tooltip tw-tooltip--down tw-tooltip--align-right"
+			>
+				{{ tooltip }}
+				<div v-if="nullable" class="tw-regular">
+					{{ t('setting.color.nullable', 'Right-Click to Reset') }}
+				</div>
+			</div>
 		</div>
 	</div>
 </template>
@@ -88,6 +101,10 @@ export default {
 			type: Boolean,
 			default: true
 		},
+		tooltip: {
+			type: String,
+			default: null
+		},
 		openUp: {
 			type: Boolean,
 			default: false
@@ -103,6 +120,10 @@ export default {
 	},
 
 	computed: {
+		hasTooltip() {
+			return this.tooltip?.length > 0 || this.nullable
+		},
+
 		colors() {
 			try {
 				return Color.RGBA.fromCSS(this.color || this.default)