diff --git a/package-lock.json b/package-lock.json index a8193f4a..7c8deaf6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "frankerfacez", - "version": "4.20.58", + "version": "4.20.59", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -4695,6 +4695,12 @@ "integrity": "sha512-Pgs951kaMm5GXP7MOvxERINe3gsaVjUWFm+UZPSq9xYriQAksyhg0csnS0KXSNRD5NmNdapXEpjxG49+AKh/ug==", "dev": true }, + "immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=", + "dev": true + }, "import-fresh": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz", @@ -5284,6 +5290,18 @@ "object.assign": "^4.1.0" } }, + "jszip": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.6.0.tgz", + "integrity": "sha512-jgnQoG9LKnWO3mnVNBnfhkh0QknICd1FGSrXcgrl67zioyJ4wgx25o9ZqwNtrROSflGBCGYnJfjrIyRIby1OoQ==", + "dev": true, + "requires": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "set-immediate-shim": "~1.0.1" + } + }, "killable": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/killable/-/killable-1.0.1.tgz", @@ -5306,6 +5324,15 @@ "type-check": "~0.4.0" } }, + "lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "dev": true, + "requires": { + "immediate": "~3.0.5" + } + }, "linkify-it": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-3.0.2.tgz", @@ -7753,6 +7780,12 @@ "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", "dev": true }, + "set-immediate-shim": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz", + "integrity": "sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E=", + "dev": true + }, "set-value": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz", diff --git a/package.json b/package.json index 2d7ff406..147194f6 100755 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "frankerfacez", "author": "Dan Salvato LLC", - "version": "4.20.59", + "version": "4.20.60", "description": "FrankerFaceZ is a Twitch enhancement suite.", "license": "Apache-2.0", "scripts": { @@ -44,6 +44,7 @@ "extract-loader": "^2.0.1", "file-loader": "^4.1.0", "json-loader": "^0.5.7", + "jszip": "^3.6.0", "node-sass": "^4.14.1", "raw-loader": "^3.1.0", "rimraf": "^3.0.2", diff --git a/src/experiments.json b/src/experiments.json index 4b20e7e3..a2d5be28 100644 --- a/src/experiments.json +++ b/src/experiments.json @@ -1,4 +1,12 @@ { + "new_links": { + "name": "New Link Tokenization", + "description": "Update to Twitch's latest link regex. Experiment while this is checked for bugs.", + "groups": [ + {"value": true, "weight": 50}, + {"value": false, "weight": 50} + ] + }, "api_load": { "name": "New API Stress Testing", "description": "Send duplicate requests to the new API server for load testing.", diff --git a/src/modules/chat/tokenizers.jsx b/src/modules/chat/tokenizers.jsx index 4b51e8f2..4d1e8ab0 100644 --- a/src/modules/chat/tokenizers.jsx +++ b/src/modules/chat/tokenizers.jsx @@ -13,6 +13,7 @@ import {CATEGORIES} from './emoji'; const EMOTE_CLASS = 'chat-image chat-line__message--emote', LINK_REGEX = /([^\w@#%\-+=:~])?((?:(https?:\/\/)?(?:[\w@#%\-+=:~]+\.)+[a-z]{2,6}(?:\/[\w./@#%&()\-+=:?~]*)?))([^\w./@#%&()\-+=:?~]|\s|$)/g, + NEW_LINK_REGEX = /(?:(https?:\/\/)?((?:[\w#%\-+=:~]+\.)+[a-z]{2,10}(?:\/[\w./#%&@()\-+=:?~]*)?))/g, //MENTION_REGEX = /([^\w@#%\-+=:~])?(@([^\u0000-\u007F]+|\w+)+)([^\w./@#%&()\-+=:?~]|\s|$)/g; // eslint-disable-line no-control-regex MENTION_REGEX = /^(['"*([{<\\/]*)(@)((?:[^\u0000-\u007F]|[\w-])+)(?:\b|$)/; // eslint-disable-line no-control-regex @@ -146,6 +147,8 @@ export const Links = { if ( ! tokens || ! tokens.length ) return tokens; + const use_new = this.experiments.getAssignment('new_links'); + const out = []; for(const token of tokens) { if ( token.type !== 'text' ) { @@ -154,24 +157,43 @@ export const Links = { } LINK_REGEX.lastIndex = 0; + NEW_LINK_REGEX.lastIndex = 0; const text = token.text; let idx = 0, match; - while((match = LINK_REGEX.exec(text))) { - const nix = match.index + (match[1] ? match[1].length : 0); - if ( idx !== nix ) - out.push({type: 'text', text: text.slice(idx, nix)}); + if ( use_new ) { + while((match = NEW_LINK_REGEX.exec(text))) { + const nix = match.index; + if ( idx !== nix ) + out.push({type: 'text', text: text.slice(idx, nix)}); - const is_mail = ! match[3] && match[2].indexOf('/') === -1 && match[2].indexOf('@') !== -1; + out.push({ + type: 'link', + url: `${match[1] ? '' : 'https://'}${match[0]}`, + is_mail: false, + text: match[0] + }); - out.push({ - type: 'link', - url: (match[3] ? '' : is_mail ? 'mailto:' : 'https://') + match[2], - is_mail, - text: match[2] - }); + idx = nix + match[0].length; + } - idx = nix + match[2].length; + } else { + while((match = LINK_REGEX.exec(text))) { + const nix = match.index + (match[1] ? match[1].length : 0); + if ( idx !== nix ) + out.push({type: 'text', text: text.slice(idx, nix)}); + + const is_mail = ! match[3] && match[2].indexOf('/') === -1 && match[2].indexOf('@') !== -1; + + out.push({ + type: 'link', + url: (match[3] ? '' : is_mail ? 'mailto:' : 'https://') + match[2], + is_mail, + text: match[2] + }); + + idx = nix + match[2].length; + } } if ( idx < text.length ) diff --git a/src/modules/main_menu/components/backup-restore.vue b/src/modules/main_menu/components/backup-restore.vue index 203a0f99..fc52c023 100644 --- a/src/modules/main_menu/components/backup-restore.vue +++ b/src/modules/main_menu/components/backup-restore.vue @@ -66,11 +66,11 @@ export default { this.error = false; this.message = null; - let blob; + let file; try { - const settings = this.item.getFFZ().resolve('settings'), - data = await settings.getFullBackup(); - blob = new Blob([JSON.stringify(data)], {type: 'application/json;charset=utf-8'}); + const settings = this.item.getFFZ().resolve('settings'); + file = await settings.generateBackupFile(); + } catch(err) { this.error_desc = this.t('setting.backup-restore.dump-error', 'Unable to export settings data to JSON.'); this.error = true; @@ -78,7 +78,7 @@ export default { } try { - saveAs(blob, 'ffz-settings.json'); + saveAs(file, file.name); } catch(err) { this.error_desc = this.t('setting.backup-restore.save-error', 'Unable to save.'); } @@ -88,9 +88,22 @@ export default { this.error = false; this.message = null; + let file; + try { + file = await openFile('application/json,application/zip'); + } catch(err) { + this.error_desc = this.t('setting.backup-restore.read-error', 'Unable to read file.'); + this.error = true; + return; + } + + // We might get a different MIME than expected, roll with it. + if ( file.type.toLowerCase().includes('zip') ) + return this.restoreZip(file); + let contents; try { - contents = await readFile(await openFile('application/json')); + contents = await readFile(file); } catch(err) { this.error_desc = this.t('setting.backup-restore.read-error', 'Unable to read file.'); this.error = true; @@ -135,6 +148,101 @@ export default { this.message = this.t('setting.backup-restore.restored', '{count,number} items have been restored. Please refresh this page.', { count: i }); + }, + + async restoreZip(file) { + const JSZip = (await import(/* webpackChunkName: "zip" */ 'jszip')).default; + let input, blobs, data; + + try { + input = await (new JSZip().loadAsync(file)); + + blobs = await input.file('blobs.json').async('text'); + data = await input.file('settings.json').async('text'); + + } catch(err) { + this.error_desc = this.t('setting.backup-restore.zip-error', 'Unable to parse ZIP archive.'); + this.error = true; + } + + try { + blobs = JSON.parse(blobs); + data = JSON.parse(data); + } catch(err) { + this.error_desc = this.t('setting.backup-restore.json-error', 'Unable to parse file as JSON.'); + this.error = true; + return; + } + + if ( ! data || data.version !== 2 ) { + this.error_desc = this.t('setting.backup-restore.old-file', 'This file is invalid or was created in another version of FrankerFaceZ and cannot be loaded.'); + this.error = true; + return; + } + + if ( data.type !== 'full' ) { + this.error_desc = this.t('setting.backup-restore.non-full', 'This file is not a full backup and cannot be restored with this tool.'); + this.error = true; + return; + } + + const settings = this.item.getFFZ().resolve('settings'); + await settings.awaitProvider(); + const provider = settings.provider; + await provider.awaitReady(); + + if ( Object.keys(blobs).length && ! provider.supportsBlobs ) { + this.error_desc = this.t('setting.backup-restore.blob-error', 'This backup contains binary data not supported by the current storage provider. Please change your storage provider in Data Management > Storage >> Provider.'); + this.error = true; + return; + } + + // Attempt to load all the blobs, to make sure they're all valid. + const loaded_blobs = {}; + + for(const [safe_key, data] of Object.entries(blobs)) { + let blob; + if ( data.type === 'file' ) { + blob = await input.file(`blobs/${safe_key}`).async('blob'); // eslint-disable-line no-await-in-loop + blob = new File([blob], data.name, {lastModified: data.modified, type: data.mime}); + } else if ( data.type === 'blob' ) + blob = await input.file(`blobs/${safe_key}`).async('blob'); // eslint-disable-line no-await-in-loop + else if ( data.type === 'ab' ) + blob = await input.file(`blobs/${safe_key}`).async('arraybuffer'); // eslint-disable-line no-await-in-loop + else if ( data.type === 'ui8' ) + blob = await input.file(`blobs/${safe_key}`).async('uint8array'); // eslint-disable-line no-await-in-loop + else { + this.error_desc = this.t('setting.backup-restore.invalid-blob', 'This file contains a binary blob with an invalid type: {type}', data); + this.error = true; + } + + loaded_blobs[data.key] = blob; + } + + // We've loaded all data, let's get this installed. + // Blobs first. + let b = 0; + await provider.clearBlobs(); + + for(const [key, blob] of Object.entries(loaded_blobs)) { + await provider.setBlob(key, blob); // eslint-disable-line no-await-in-loop + b++; + } + + // Settings second. + provider.clear(); + let i = 0; + for(const key of Object.keys(data.values)) { + const val = data.values[key]; + provider.set(key, val); + provider.emit('changed', key, val, false); + i++; + } + + this.message = this.t('setting.backup-restore.zip-restored', '{count,number} items and {blobs,number} binary blobs have been restored. Please refresh this page.', { + count: i, + blobs: b + }); } } } diff --git a/src/modules/main_menu/components/clear-settings.vue b/src/modules/main_menu/components/clear-settings.vue index 953389cc..de529b42 100644 --- a/src/modules/main_menu/components/clear-settings.vue +++ b/src/modules/main_menu/components/clear-settings.vue @@ -134,81 +134,6 @@ export default { } this.running = false; - }, - - async backup() { - this.error = false; - this.message = null; - - let blob; - try { - const settings = this.item.getFFZ().resolve('settings'), - data = await settings.getFullBackup(); - blob = new Blob([JSON.stringify(data)], {type: 'application/json;charset=utf-8'}); - } catch(err) { - this.error_desc = this.t('setting.backup-restore.dump-error', 'Unable to export settings data to JSON.'); - this.error = true; - return; - } - - try { - saveAs(blob, 'ffz-settings.json'); - } catch(err) { - this.error_desc = this.t('setting.backup-restore.save-error', 'Unable to save.'); - } - }, - - async restore() { - this.error = false; - this.message = null; - - let contents; - try { - contents = await readFile(await openFile('application/json')); - } catch(err) { - this.error_desc = this.t('setting.backup-restore.read-error', 'Unable to read file.'); - this.error = true; - return; - } - - let data; - try { - data = JSON.parse(contents); - } catch(err) { - this.error_desc = this.t('setting.backup-restore.json-error', 'Unable to parse file as JSON.'); - this.error = true; - return; - } - - if ( ! data || data.version !== 2 ) { - this.error_desc = this.t('setting.backup-restore.old-file', 'This file is invalid or was created in another version of FrankerFaceZ and cannot be loaded.'); - this.error = true; - return; - } - - if ( data.type !== 'full' ) { - this.error_desc = this.t('setting.backup-restore.non-full', 'This file is not a full backup and cannot be restored with this tool.'); - this.error = true; - return; - } - - const settings = this.item.getFFZ().resolve('settings'), - provider = settings.provider; - - await provider.awaitReady(); - - provider.clear(); - let i = 0; - for(const key of Object.keys(data.values)) { - const val = data.values[key]; - provider.set(key, val); - provider.emit('changed', key, val, false); - i++; - } - - this.message = this.t('setting.backup-restore.restored', '{count,number} items have been restored. Please refresh this page.', { - count: i - }); } } } diff --git a/src/modules/main_menu/components/profile-manager.vue b/src/modules/main_menu/components/profile-manager.vue index c6a6b86d..296a1913 100644 --- a/src/modules/main_menu/components/profile-manager.vue +++ b/src/modules/main_menu/components/profile-manager.vue @@ -276,9 +276,20 @@ export default { async doImport() { this.resetImport(); - let contents; + let file, contents; try { - contents = await readFile(await openFile('application/json')); + file = await openFile('application/json,application/zip'); + + // We might get a different MIME than expected, roll with it. + if ( file.type.toLowerCase().includes('zip') ) { + const JSZip = (await import(/* webpackChunkName: "zip" */ 'jszip')).default, + zip = await (new JSZip().loadAsync(file)); + + contents = await zip.file('settings.json').async('text'); + + } else + contents = await readFile(file); + } catch(err) { this.import_error = true; this.import_error_message = this.t('setting.backup-restore.read-error', 'Unable to read file.'); diff --git a/src/modules/main_menu/components/provider.vue b/src/modules/main_menu/components/provider.vue index e0ca5f2f..43a0b037 100644 --- a/src/modules/main_menu/components/provider.vue +++ b/src/modules/main_menu/components/provider.vue @@ -38,6 +38,9 @@ {{ t('setting.provider.has-data', '(Has Data)') }} + + {{ t('setting.provider.has-blobs', '(Supports Binary Data)') }} +
@@ -100,6 +103,7 @@ export default { const prov = { key, has_data: null, + has_blobs: val.supportsBlobs, i18n_key: `setting.provider.${key}.title`, title: val.title || key, desc_i18n_key: val.description ? `setting.provider.${key}.desc` : null, diff --git a/src/modules/main_menu/components/setting-hotkey.vue b/src/modules/main_menu/components/setting-hotkey.vue index 40f61fef..4e86b271 100644 --- a/src/modules/main_menu/components/setting-hotkey.vue +++ b/src/modules/main_menu/components/setting-hotkey.vue @@ -1,25 +1,47 @@