From 097e49d7978fb1b1a23ae66c55a8f54a6a4b51aa Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Sun, 20 Jun 2021 11:19:59 +0100 Subject: [PATCH] All: Fixes #5051: Fixed error that could prevent a revision from being created, and that would prevent the revision service from processing the rest of the notes --- .eslintignore | 6 + .gitignore | 6 + .../support/big-list-of-naughty-strings.txt | 742 ++++++++++++++++++ packages/lib/models/Note.ts | 2 +- packages/lib/models/Revision.test.js | 92 --- packages/lib/models/Revision.test.ts | 194 +++++ packages/lib/models/Revision.ts | 41 +- ...ervice.test.js => RevisionService.test.ts} | 94 ++- packages/lib/services/RevisionService.ts | 50 +- packages/lib/testing/test-utils.ts | 27 +- 10 files changed, 1099 insertions(+), 155 deletions(-) create mode 100644 packages/app-cli/tests/support/big-list-of-naughty-strings.txt delete mode 100644 packages/lib/models/Revision.test.js create mode 100644 packages/lib/models/Revision.test.ts rename packages/lib/services/{RevisionService.test.js => RevisionService.test.ts} (82%) diff --git a/.eslintignore b/.eslintignore index 0c08a5a3c..51fcb6136 100644 --- a/.eslintignore +++ b/.eslintignore @@ -968,6 +968,9 @@ packages/lib/models/ResourceLocalState.js.map packages/lib/models/Revision.d.ts packages/lib/models/Revision.js packages/lib/models/Revision.js.map +packages/lib/models/Revision.test.d.ts +packages/lib/models/Revision.test.js +packages/lib/models/Revision.test.js.map packages/lib/models/Search.d.ts packages/lib/models/Search.js packages/lib/models/Search.js.map @@ -1088,6 +1091,9 @@ packages/lib/services/ResourceService.test.js.map packages/lib/services/RevisionService.d.ts packages/lib/services/RevisionService.js packages/lib/services/RevisionService.js.map +packages/lib/services/RevisionService.test.d.ts +packages/lib/services/RevisionService.test.js +packages/lib/services/RevisionService.test.js.map packages/lib/services/SettingUtils.d.ts packages/lib/services/SettingUtils.js packages/lib/services/SettingUtils.js.map diff --git a/.gitignore b/.gitignore index 48bca5b7e..58d3c96f2 100644 --- a/.gitignore +++ b/.gitignore @@ -954,6 +954,9 @@ packages/lib/models/ResourceLocalState.js.map packages/lib/models/Revision.d.ts packages/lib/models/Revision.js packages/lib/models/Revision.js.map +packages/lib/models/Revision.test.d.ts +packages/lib/models/Revision.test.js +packages/lib/models/Revision.test.js.map packages/lib/models/Search.d.ts packages/lib/models/Search.js packages/lib/models/Search.js.map @@ -1074,6 +1077,9 @@ packages/lib/services/ResourceService.test.js.map packages/lib/services/RevisionService.d.ts packages/lib/services/RevisionService.js packages/lib/services/RevisionService.js.map +packages/lib/services/RevisionService.test.d.ts +packages/lib/services/RevisionService.test.js +packages/lib/services/RevisionService.test.js.map packages/lib/services/SettingUtils.d.ts packages/lib/services/SettingUtils.js packages/lib/services/SettingUtils.js.map diff --git a/packages/app-cli/tests/support/big-list-of-naughty-strings.txt b/packages/app-cli/tests/support/big-list-of-naughty-strings.txt new file mode 100644 index 000000000..7c27bbd7e --- /dev/null +++ b/packages/app-cli/tests/support/big-list-of-naughty-strings.txt @@ -0,0 +1,742 @@ +# Reserved Strings +# +# Strings which may be used elsewhere in code + +undefined +undef +null +NULL +(null) +nil +NIL +true +false +True +False +TRUE +FALSE +None +hasOwnProperty +then +constructor +\ +\\ + +# Numeric Strings +# +# Strings which can be interpreted as numeric + +0 +1 +1.00 +$1.00 +1/2 +1E2 +1E02 +1E+02 +-1 +-1.00 +-$1.00 +-1/2 +-1E2 +-1E02 +-1E+02 +1/0 +0/0 +-2147483648/-1 +-9223372036854775808/-1 +-0 +-0.0 ++0 ++0.0 +0.00 +0..0 +. +0.0.0 +0,00 +0,,0 +, +0,0,0 +0.0/0 +1.0/0.0 +0.0/0.0 +1,0/0,0 +0,0/0,0 +--1 +- +-. +-, +999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999 +NaN +Infinity +-Infinity +INF +1#INF +-1#IND +1#QNAN +1#SNAN +1#IND +0x0 +0xffffffff +0xffffffffffffffff +0xabad1dea +123456789012345678901234567890123456789 +1,000.00 +1 000.00 +1'000.00 +1,000,000.00 +1 000 000.00 +1'000'000.00 +1.000,00 +1 000,00 +1'000,00 +1.000.000,00 +1 000 000,00 +1'000'000,00 +01000 +08 +09 +2.2250738585072011e-308 + +# Special Characters +# +# ASCII punctuation. All of these characters may need to be escaped in some +# contexts. Divided into three groups based on (US-layout) keyboard position. + +,./;'[]\-= +<>?:"{}|_+ +!@#$%^&*()`~ + +# Non-whitespace C0 controls: U+0001 through U+0008, U+000E through U+001F, +# and U+007F (DEL) +# Often forbidden to appear in various text-based file formats (e.g. XML), +# or reused for internal delimiters on the theory that they should never +# appear in input. +# The next line may appear to be blank or mojibake in some viewers. + + +# Non-whitespace C1 controls: U+0080 through U+0084 and U+0086 through U+009F. +# Commonly misinterpreted as additional graphic characters. +# The next line may appear to be blank, mojibake, or dingbats in some viewers. +€‚ƒ„†‡ˆ‰Š‹ŒŽ‘’“”•–—˜™š›œžŸ + +# Whitespace: all of the characters with category Zs, Zl, or Zp (in Unicode +# version 8.0.0), plus U+0009 (HT), U+000B (VT), U+000C (FF), U+0085 (NEL), +# and U+200B (ZERO WIDTH SPACE), which are in the C categories but are often +# treated as whitespace in some contexts. +# This file unfortunately cannot express strings containing +# U+0000, U+000A, or U+000D (NUL, LF, CR). +# The next line may appear to be blank or mojibake in some viewers. +# The next line may be flagged for "trailing whitespace" in some viewers. + …             ​

  　 + +# Unicode additional control characters: all of the characters with +# general category Cf (in Unicode 8.0.0). +# The next line may appear to be blank or mojibake in some viewers. +­؀؁؂؃؄؅؜۝܏᠎​‌‍‎‏‪‫‬‭‮⁠⁡⁢⁣⁤⁦⁧⁨⁩𑂽𛲠𛲡𛲢𛲣𝅳𝅴𝅵𝅶𝅷𝅸𝅹𝅺󠀁󠀠󠀡󠀢󠀣󠀤󠀥󠀦󠀧󠀨󠀩󠀪󠀫󠀬󠀭󠀮󠀯󠀰󠀱󠀲󠀳󠀴󠀵󠀶󠀷󠀸󠀹󠀺󠀻󠀼󠀽󠀾󠀿󠁀󠁁󠁂󠁃󠁄󠁅󠁆󠁇󠁈󠁉󠁊󠁋󠁌󠁍󠁎󠁏󠁐󠁑󠁒󠁓󠁔󠁕󠁖󠁗󠁘󠁙󠁚󠁛󠁜󠁝󠁞󠁟󠁠󠁡󠁢󠁣󠁤󠁥󠁦󠁧󠁨󠁩󠁪󠁫󠁬󠁭󠁮󠁯󠁰󠁱󠁲󠁳󠁴󠁵󠁶󠁷󠁸󠁹󠁺󠁻󠁼󠁽󠁾󠁿 + +# "Byte order marks", U+FEFF and U+FFFE, each on its own line. +# The next two lines may appear to be blank or mojibake in some viewers. + +￾ + +# Unicode Symbols +# +# Strings which contain common unicode symbols (e.g. smart quotes) + +Ω≈ç√∫˜µ≤≥÷ +åß∂ƒ©˙∆˚¬…æ +œ∑´®†¥¨ˆøπ“‘ +¡™£¢∞§¶•ªº–≠ +¸˛Ç◊ı˜Â¯˘¿ +ÅÍÎÏ˝ÓÔÒÚÆ☃ +Œ„´‰ˇÁ¨ˆØ∏”’ +`⁄€‹›fifl‡°·‚—± +⅛⅜⅝⅞ +ЁЂЃЄЅІЇЈЉЊЋЌЍЎЏАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюя +٠١٢٣٤٥٦٧٨٩ + +# Unicode Subscript/Superscript/Accents +# +# Strings which contain unicode subscripts/superscripts; can cause rendering issues + +⁰⁴⁵ +₀₁₂ +⁰⁴⁵₀₁₂ +ด้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็ ด้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็ ด้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็ + +# Quotation Marks +# +# Strings which contain misplaced quotation marks; can cause encoding errors + +' +" +'' +"" +'"' +"''''"'" +"'"'"''''" + + + + + +# Two-Byte Characters +# +# Strings which contain two-byte characters: can cause rendering issues or character-length issues + +田中さんにあげて下さい +パーティーへ行かないか +和製漢語 +部落格 +사회과학원 어학연구소 +찦차를 타고 온 펲시맨과 쑛다리 똠방각하 +社會科學院語學研究所 +울란바토르 +𠜎𠜱𠝹𠱓𠱸𠲖𠳏 + +# Strings which contain two-byte letters: can cause issues with naïve UTF-16 capitalizers which think that 16 bits == 1 character + +𐐜 𐐔𐐇𐐝𐐀𐐡𐐇𐐓 𐐙𐐊𐐡𐐝𐐓/𐐝𐐇𐐗𐐊𐐤𐐔 𐐒𐐋𐐗 𐐒𐐌 𐐜 𐐡𐐀𐐖𐐇𐐤𐐓𐐝 𐐱𐑂 𐑄 𐐔𐐇𐐝𐐀𐐡𐐇𐐓 𐐏𐐆𐐅𐐤𐐆𐐚𐐊𐐡𐐝𐐆𐐓𐐆 + +# Special Unicode Characters Union +# +# A super string recommended by VMware Inc. Globalization Team: can effectively cause rendering issues or character-length issues to validate product globalization readiness. +# +# 表 CJK_UNIFIED_IDEOGRAPHS (U+8868) +# ポ KATAKANA LETTER PO (U+30DD) +# あ HIRAGANA LETTER A (U+3042) +# A LATIN CAPITAL LETTER A (U+0041) +# 鷗 CJK_UNIFIED_IDEOGRAPHS (U+9DD7) +# Œ LATIN SMALL LIGATURE OE (U+0153) +# é LATIN SMALL LETTER E WITH ACUTE (U+00E9) +# Ｂ FULLWIDTH LATIN CAPITAL LETTER B (U+FF22) +# 逍 CJK_UNIFIED_IDEOGRAPHS (U+900D) +# Ü LATIN SMALL LETTER U WITH DIAERESIS (U+00FC) +# ß LATIN SMALL LETTER SHARP S (U+00DF) +# ª FEMININE ORDINAL INDICATOR (U+00AA) +# ą LATIN SMALL LETTER A WITH OGONEK (U+0105) +# ñ LATIN SMALL LETTER N WITH TILDE (U+00F1) +# 丂 CJK_UNIFIED_IDEOGRAPHS (U+4E02) +# 㐀 CJK Ideograph Extension A, First (U+3400) +# 𠀀 CJK Ideograph Extension B, First (U+20000) + +表ポあA鷗ŒéＢ逍Üߪąñ丂㐀𠀀 + +# Changing length when lowercased +# +# Characters which increase in length (2 to 3 bytes) when lowercased +# Credit: https://twitter.com/jifa/status/625776454479970304 + +Ⱥ +Ⱦ + +# Japanese Emoticons +# +# Strings which consists of Japanese-style emoticons which are popular on the web + +ヽ༼ຈل͜ຈ༽ﾉ ヽ༼ຈل͜ຈ༽ﾉ +(｡◕ ∀ ◕｡) +｀ｨ(´∀｀∩ +__ﾛ(,_,*) +・(￣∀￣)・:*: +ﾟ･✿ヾ╲(｡◕‿◕｡)╱✿･ﾟ +,。・:*:・゜’( ☻ ω ☻ )。・:*:・゜’ +(╯°□°）╯︵ ┻━┻) +(ﾉಥ益ಥ）ﾉ ┻━┻ +┬─┬ノ( º _ ºノ) +( ͡° ͜ʖ ͡°) +¯\_(ツ)_/¯ + +# Emoji +# +# Strings which contain Emoji; should be the same behavior as two-byte characters, but not always + +😍 +👩🏽 +👨‍🦰 👨🏿‍🦰 👨‍🦱 👨🏿‍🦱 🦹🏿‍♂️ +👾 🙇 💁 🙅 🙆 🙋 🙎 🙍 +🐵 🙈 🙉 🙊 +❤️ 💔 💌 💕 💞 💓 💗 💖 💘 💝 💟 💜 💛 💚 💙 +✋🏿 💪🏿 👐🏿 🙌🏿 👏🏿 🙏🏿 +👨‍👩‍👦 👨‍👩‍👧‍👦 👨‍👨‍👦 👩‍👩‍👧 👨‍👦 👨‍👧‍👦 👩‍👦 👩‍👧‍👦 +🚾 🆒 🆓 🆕 🆖 🆗 🆙 🏧 +0️⃣ 1️⃣ 2️⃣ 3️⃣ 4️⃣ 5️⃣ 6️⃣ 7️⃣ 8️⃣ 9️⃣ 🔟 + +# Regional Indicator Symbols +# +# Regional Indicator Symbols can be displayed differently across +# fonts, and have a number of special behaviors + +🇺🇸🇷🇺🇸 🇦🇫🇦🇲🇸 +🇺🇸🇷🇺🇸🇦🇫🇦🇲 +🇺🇸🇷🇺🇸🇦 + +# Unicode Numbers +# +# Strings which contain unicode numbers; if the code is localized, it should see the input as numeric + +１２３ +١٢٣ + +# Right-To-Left Strings +# +# Strings which contain text that should be rendered RTL if possible (e.g. Arabic, Hebrew) + +ثم نفس سقطت وبالتحديد،, جزيرتي باستخدام أن دنو. إذ هنا؟ الستار وتنصيب كان. أهّل ايطاليا، بريطانيا-فرنسا قد أخذ. سليمان، إتفاقية بين ما, يذكر الحدود أي بعد, معاملة بولندا، الإطلاق عل إيو. +בְּרֵאשִׁית, בָּרָא אֱלֹהִים, אֵת הַשָּׁמַיִם, וְאֵת הָאָרֶץ +הָיְתָהtestالصفحات التّحول +﷽ +ﷺ +مُنَاقَشَةُ سُبُلِ اِسْتِخْدَامِ اللُّغَةِ فِي النُّظُمِ الْقَائِمَةِ وَفِيم يَخُصَّ التَّطْبِيقَاتُ الْحاسُوبِيَّةُ، +الكل في المجمو عة (5) + +# Ogham Text +# +# The only unicode alphabet to use a space which isn't empty but should still act like a space. + +᚛ᚄᚓᚐᚋᚒᚄ ᚑᚄᚂᚑᚏᚅ᚜ +᚛                 ᚜ + +# Trick Unicode +# +# Strings which contain unicode with unusual properties (e.g. Right-to-left override) (c.f. http://www.unicode.org/charts/PDF/U2000.pdf) + +‪‪test‪ +‫test‫ +
test
 +test⁠test‫ +⁦test⁧ + +# Zalgo Text +# +# Strings which contain "corrupted" text. The corruption will not appear in non-HTML text, however. (via http://www.eeemo.net) + +Ṱ̺̺̕o͞ ̷i̲̬͇̪͙n̝̗͕v̟̜̘̦͟o̶̙̰̠kè͚̮̺̪̹̱̤ ̖t̝͕̳̣̻̪͞h̼͓̲̦̳̘̲e͇̣̰̦̬͎ ̢̼̻̱̘h͚͎͙̜̣̲ͅi̦̲̣̰̤v̻͍e̺̭̳̪̰-m̢iͅn̖̺̞̲̯̰d̵̼̟͙̩̼̘̳ ̞̥̱̳̭r̛̗̘e͙p͠r̼̞̻̭̗e̺̠̣͟s̘͇̳͍̝͉e͉̥̯̞̲͚̬͜ǹ̬͎͎̟̖͇̤t͍̬̤͓̼̭͘ͅi̪̱n͠g̴͉ ͏͉ͅc̬̟h͡a̫̻̯͘o̫̟̖͍̙̝͉s̗̦̲.̨̹͈̣ +̡͓̞ͅI̗̘̦͝n͇͇͙v̮̫ok̲̫̙͈i̖͙̭̹̠̞n̡̻̮̣̺g̲͈͙̭͙̬͎ ̰t͔̦h̞̲e̢̤ ͍̬̲͖f̴̘͕̣è͖ẹ̥̩l͖͔͚i͓͚̦͠n͖͍̗͓̳̮g͍ ̨o͚̪͡f̘̣̬ ̖̘͖̟͙̮c҉͔̫͖͓͇͖ͅh̵̤̣͚͔á̗̼͕ͅo̼̣̥s̱͈̺̖̦̻͢.̛̖̞̠̫̰ +̗̺͖̹̯͓Ṯ̤͍̥͇͈h̲́e͏͓̼̗̙̼̣͔ ͇̜̱̠͓͍ͅN͕͠e̗̱z̘̝̜̺͙p̤̺̹͍̯͚e̠̻̠͜r̨̤͍̺̖͔̖̖d̠̟̭̬̝͟i̦͖̩͓͔̤a̠̗̬͉̙n͚͜ ̻̞̰͚ͅh̵͉i̳̞v̢͇ḙ͎͟-҉̭̩̼͔m̤̭̫i͕͇̝̦n̗͙ḍ̟ ̯̲͕͞ǫ̟̯̰̲͙̻̝f ̪̰̰̗̖̭̘͘c̦͍̲̞͍̩̙ḥ͚a̮͎̟̙͜ơ̩̹͎s̤.̝̝ ҉Z̡̖̜͖̰̣͉̜a͖̰͙̬͡l̲̫̳͍̩g̡̟̼̱͚̞̬ͅo̗͜.̟ +̦H̬̤̗̤͝e͜ ̜̥̝̻͍̟́w̕h̖̯͓o̝͙̖͎̱̮ ҉̺̙̞̟͈W̷̼̭a̺̪͍į͈͕̭͙̯̜t̶̼̮s̘͙͖̕ ̠̫̠B̻͍͙͉̳ͅe̵h̵̬͇̫͙i̹͓̳̳̮͎̫̕n͟d̴̪̜̖ ̰͉̩͇͙̲͞ͅT͖̼͓̪͢h͏͓̮̻e̬̝̟ͅ ̤̹̝W͙̞̝͔͇͝ͅa͏͓͔̹̼̣l̴͔̰̤̟͔ḽ̫.͕ +Z̮̞̠͙͔ͅḀ̗̞͈̻̗Ḷ͙͎̯̹̞͓G̻O̭̗̮ + +# Unicode Upsidedown +# +# Strings which contain unicode with an "upsidedown" effect (via http://www.upsidedowntext.com) + +˙ɐnbᴉlɐ ɐuƃɐɯ ǝɹolop ʇǝ ǝɹoqɐl ʇn ʇunpᴉpᴉɔuᴉ ɹodɯǝʇ poɯsnᴉǝ op pǝs 'ʇᴉlǝ ƃuᴉɔsᴉdᴉpɐ ɹnʇǝʇɔǝsuoɔ 'ʇǝɯɐ ʇᴉs ɹolop ɯnsdᴉ ɯǝɹo˥ +00˙Ɩ$- + +# Unicode font +# +# Strings which contain bold/italic/etc. versions of normal characters + +Ｔｈｅ ｑｕｉｃｋ ｂｒｏｗｎ ｆｏｘ ｊｕｍｐｓ ｏｖｅｒ ｔｈｅ ｌａｚｙ ｄｏｇ +𝐓𝐡𝐞 𝐪𝐮𝐢𝐜𝐤 𝐛𝐫𝐨𝐰𝐧 𝐟𝐨𝐱 𝐣𝐮𝐦𝐩𝐬 𝐨𝐯𝐞𝐫 𝐭𝐡𝐞 𝐥𝐚𝐳𝐲 𝐝𝐨𝐠 +𝕿𝖍𝖊 𝖖𝖚𝖎𝖈𝖐 𝖇𝖗𝖔𝖜𝖓 𝖋𝖔𝖝 𝖏𝖚𝖒𝖕𝖘 𝖔𝖛𝖊𝖗 𝖙𝖍𝖊 𝖑𝖆𝖟𝖞 𝖉𝖔𝖌 +𝑻𝒉𝒆 𝒒𝒖𝒊𝒄𝒌 𝒃𝒓𝒐𝒘𝒏 𝒇𝒐𝒙 𝒋𝒖𝒎𝒑𝒔 𝒐𝒗𝒆𝒓 𝒕𝒉𝒆 𝒍𝒂𝒛𝒚 𝒅𝒐𝒈 +𝓣𝓱𝓮 𝓺𝓾𝓲𝓬𝓴 𝓫𝓻𝓸𝔀𝓷 𝓯𝓸𝔁 𝓳𝓾𝓶𝓹𝓼 𝓸𝓿𝓮𝓻 𝓽𝓱𝓮 𝓵𝓪𝔃𝔂 𝓭𝓸𝓰 +𝕋𝕙𝕖 𝕢𝕦𝕚𝕔𝕜 𝕓𝕣𝕠𝕨𝕟 𝕗𝕠𝕩 𝕛𝕦𝕞𝕡𝕤 𝕠𝕧𝕖𝕣 𝕥𝕙𝕖 𝕝𝕒𝕫𝕪 𝕕𝕠𝕘 +𝚃𝚑𝚎 𝚚𝚞𝚒𝚌𝚔 𝚋𝚛𝚘𝚠𝚗 𝚏𝚘𝚡 𝚓𝚞𝚖𝚙𝚜 𝚘𝚟𝚎𝚛 𝚝𝚑𝚎 𝚕𝚊𝚣𝚢 𝚍𝚘𝚐 +⒯⒣⒠ ⒬⒰⒤⒞⒦ ⒝⒭⒪⒲⒩ ⒡⒪⒳ ⒥⒰⒨⒫⒮ ⒪⒱⒠⒭ ⒯⒣⒠ ⒧⒜⒵⒴ ⒟⒪⒢ + +# Script Injection +# +# Strings which attempt to invoke a benign script injection; shows vulnerability to XSS + + +<script>alert('1');</script> + + +"> +'> +> + +< / script >< script >alert(8)< / script > + onfocus=JaVaSCript:alert(9) autofocus +" onfocus=JaVaSCript:alert(10) autofocus +' onfocus=JaVaSCript:alert(11) autofocus +＜script＞alert(12)＜/script＞ +ript>alert(13)ript> +--> +";alert(15);t=" +';alert(16);t=' +JavaSCript:alert(17) +;alert(18); +src=JaVaSCript:prompt(19) +">javascript:alert(25); +javascript:alert(26); +javascript:alert(27); +javascript:alert(28); +javascript:alert(29); +javascript:alert(30); +javascript:alert(31); +'`"><\x3Cscript>javascript:alert(32) +'`"><\x00script>javascript:alert(33) +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +`"'> +`"'> +`"'> +`"'> +`"'> +`"'> +`"'> +`"'> +`"'> +`"'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +XXX + + + +<a href=http://foo.bar/#x=`y></a><img alt="`><img src=x:x onerror=javascript:alert(203)></a>"> +<!--[if]><script>javascript:alert(204)</script --> +<!--[if<img src=x onerror=javascript:alert(205)//]> --> +<script src="/\%(jscript)s"></script> +<script src="\\%(jscript)s"></script> +<IMG """><SCRIPT>alert("206")</SCRIPT>"> +<IMG SRC=javascript:alert(String.fromCharCode(50,48,55))> +<IMG SRC=# onmouseover="alert('208')"> +<IMG SRC= onmouseover="alert('209')"> +<IMG onmouseover="alert('210')"> +<IMG SRC=javascript:alert('211')> +<IMG SRC=javascript:alert('212')> +<IMG SRC=javascript:alert('213')> +<IMG SRC="jav   ascript:alert('214');"> +<IMG SRC="jav ascript:alert('215');"> +<IMG SRC="jav ascript:alert('216');"> +<IMG SRC="jav ascript:alert('217');"> +perl -e 'print "<IMG SRC=java\0script:alert(\"218\")>";' > out +<IMG SRC="   javascript:alert('219');"> +<SCRIPT/XSS SRC="http://ha.ckers.org/xss.js"></SCRIPT> +<BODY onload!#$%&()*~+-_.,:;?@[/|\]^`=alert("220")> +<SCRIPT/SRC="http://ha.ckers.org/xss.js"></SCRIPT> +<<SCRIPT>alert("221");//<</SCRIPT> +<SCRIPT SRC=http://ha.ckers.org/xss.js?< B > +<SCRIPT SRC=//ha.ckers.org/.j> +<IMG SRC="javascript:alert('222')" +<iframe src=http://ha.ckers.org/scriptlet.html < +\";alert('223');// +<u oncopy=alert()> Copy me</u> +<i onwheel=alert(224)> Scroll over me </i> +<plaintext> +http://a/%%30%30 +</textarea><script>alert(225)</script> + +# SQL Injection +# +# Strings which can cause a SQL injection if inputs are not sanitized + +1;DROP TABLE users +1'; DROP TABLE users-- 1 +' OR 1=1 -- 1 +' OR '1'='1 +'; EXEC sp_MSForEachTable 'DROP TABLE ?'; -- + +% +_ + +# Server Code Injection +# +# Strings which can cause user to run code on server as a privileged user (c.f. https://news.ycombinator.com/item?id=7665153) + +- +-- +--version +--help +$USER +/dev/null; touch /tmp/blns.fail ; echo +`touch /tmp/blns.fail` +$(touch /tmp/blns.fail) +@{[system "touch /tmp/blns.fail"]} + +# Command Injection (Ruby) +# +# Strings which can call system commands within Ruby/Rails applications + +eval("puts 'hello world'") +System("ls -al /") +`ls -al /` +Kernel.exec("ls -al /") +Kernel.exit(1) +%x('ls -al /') + +# XXE Injection (XML) +# +# String which can reveal system files when parsed by a badly configured XML parser + +<?xml version="1.0" encoding="ISO-8859-1"?><!DOCTYPE foo [ <!ELEMENT foo ANY ><!ENTITY xxe SYSTEM "file:///etc/passwd" >]><foo>&xxe;</foo> + +# Unwanted Interpolation +# +# Strings which can be accidentally expanded into different strings if evaluated in the wrong context, e.g. used as a printf format string or via Perl or shell eval. Might expose sensitive data from the program doing the interpolation, or might just represent the wrong string. + +$HOME +$ENV{'HOME'} +%d +%s%s%s%s%s +{0} +%*.*s +%@ +%n +File:/// + +# File Inclusion +# +# Strings which can cause user to pull in files that should not be a part of a web server + +../../../../../../../../../../../etc/passwd%00 +../../../../../../../../../../../etc/hosts + +# Known CVEs and Vulnerabilities +# +# Strings that test for known vulnerabilities + +() { 0; }; touch /tmp/blns.shellshock1.fail; +() { _; } >_[$($())] { touch /tmp/blns.shellshock2.fail; } +<<< %s(un='%s') = %u ++++ATH0 + +# MSDOS/Windows Special Filenames +# +# Strings which are reserved characters in MSDOS/Windows + +CON +PRN +AUX +CLOCK$ +NUL +A: +ZZ: +COM1 +LPT1 +LPT2 +LPT3 +COM2 +COM3 +COM4 + +# IRC specific strings +# +# Strings that may occur on IRC clients that make security products freak out + +DCC SEND STARTKEYLOGGER 0 0 0 + +# Scunthorpe Problem +# +# Innocuous strings which may be blocked by profanity filters (https://en.wikipedia.org/wiki/Scunthorpe_problem) + +Scunthorpe General Hospital +Penistone Community Church +Lightwater Country Park +Jimmy Clitheroe +Horniman Museum +shitake mushrooms +RomansInSussex.co.uk +http://www.cum.qc.ca/ +Craig Cockburn, Software Specialist +Linda Callahan +Dr. Herman I. Libshitz +magna cum laude +Super Bowl XXX +medieval erection of parapets +evaluate +mocha +expression +Arsenal canal +classic +Tyson Gay +Dick Van Dyke +basement + +# Human injection +# +# Strings which may cause human to reinterpret worldview + +If you're reading this, you've been in a coma for almost 20 years now. We're trying a new technique. We don't know where this message will end up in your dream, but we hope it works. Please wake up, we miss you. + +# Terminal escape codes +# +# Strings which punish the fools who use cat/type on this file + +Roses are red, violets are blue. Hope you enjoy terminal hue +But now...for my greatest trick... +The quick brown fox... [Beeeep] + +# iOS Vulnerabilities +# +# Strings which crashed iMessage in various versions of iOS + +Powerلُلُصّبُلُلصّبُررً ॣ ॣh ॣ ॣ冗 +🏳0🌈️ +జ్ఞ‌ా + +# Persian special characters +# +# This is a four characters string which includes Persian special characters (گچپژ) + +گچپژ + +# jinja2 injection +# +# first one is supposed to raise "MemoryError" exception +# second, obviously, prints contents of /etc/passwd + +{% print 'x' * 64 * 1024**3 %} +{{ "".__class__.__mro__[2].__subclasses__()[40]("/etc/passwd").read() }} diff --git a/packages/lib/models/Note.ts b/packages/lib/models/Note.ts index 5832ba333..a5efb9481 100644 --- a/packages/lib/models/Note.ts +++ b/packages/lib/models/Note.ts @@ -633,7 +633,7 @@ export default class Note extends BaseItem { return n.updated_time < date; } - static async save(o: NoteEntity, options: any = null) { + public static async save(o: NoteEntity, options: any = null): Promise<NoteEntity> { const isNew = this.isNew(o, options); // If true, this is a provisional note - it will be saved permanently diff --git a/packages/lib/models/Revision.test.js b/packages/lib/models/Revision.test.js deleted file mode 100644 index d36971191..000000000 --- a/packages/lib/models/Revision.test.js +++ /dev/null @@ -1,92 +0,0 @@ -const { setupDatabaseAndSynchronizer, switchClient } = require('../testing/test-utils.js'); -const Note = require('../models/Note').default; -const Revision = require('../models/Revision').default; - -describe('models_Revision', function() { - - beforeEach(async (done) => { - await setupDatabaseAndSynchronizer(1); - await switchClient(1); - done(); - }); - - it('should create patches of text and apply it', (async () => { - const note1 = await Note.save({ body: 'my note\nsecond line' }); - - const patch = Revision.createTextPatch(note1.body, 'my new note\nsecond line'); - const merged = Revision.applyTextPatch(note1.body, patch); - - expect(merged).toBe('my new note\nsecond line'); - })); - - it('should create patches of objects and apply it', (async () => { - const oldObject = { - one: '123', - two: '456', - three: '789', - }; - - const newObject = { - one: '123', - three: '999', - }; - - const patch = Revision.createObjectPatch(oldObject, newObject); - const merged = Revision.applyObjectPatch(oldObject, patch); - - expect(JSON.stringify(merged)).toBe(JSON.stringify(newObject)); - })); - - it('should move target revision to the top', (async () => { - const revs = [ - { id: '123' }, - { id: '456' }, - { id: '789' }, - ]; - - let newRevs; - newRevs = Revision.moveRevisionToTop({ id: '456' }, revs); - expect(newRevs[0].id).toBe('123'); - expect(newRevs[1].id).toBe('789'); - expect(newRevs[2].id).toBe('456'); - - newRevs = Revision.moveRevisionToTop({ id: '789' }, revs); - expect(newRevs[0].id).toBe('123'); - expect(newRevs[1].id).toBe('456'); - expect(newRevs[2].id).toBe('789'); - })); - - it('should create patch stats', (async () => { - const tests = [ - { - patch: `@@ -625,16 +625,48 @@ - rrupted download -+%0A- %5B %5D Fix mobile screen options`, - expected: [-0, +32], - }, - { - patch: `@@ -564,17 +564,17 @@ - ages%0A- %5B -- -+x - %5D Check `, - expected: [-1, +1], - }, - { - patch: `@@ -1022,56 +1022,415 @@ - .%0A%0A# -- How to view a note history%0A%0AWhile all the apps -+%C2%A0How does it work?%0A%0AAll the apps save a version of the modified notes every 10 minutes. - %0A%0A# `, - expected: [-(19 + 27 + 2), 17 + 67 + 4], - }, - ]; - - for (const test of tests) { - const stats = Revision.patchStats(test.patch); - expect(stats.removed).toBe(-test.expected[0]); - expect(stats.added).toBe(test.expected[1]); - } - })); - -}); diff --git a/packages/lib/models/Revision.test.ts b/packages/lib/models/Revision.test.ts new file mode 100644 index 000000000..280e46943 --- /dev/null +++ b/packages/lib/models/Revision.test.ts @@ -0,0 +1,194 @@ +import { expectNotThrow, naughtyStrings, setupDatabaseAndSynchronizer, switchClient } from '../testing/test-utils'; +import Note from '../models/Note'; +import Revision from '../models/Revision'; + +describe('models/Revision', function() { + + beforeEach(async (done) => { + await setupDatabaseAndSynchronizer(1); + await switchClient(1); + done(); + }); + + it('should create patches of text and apply it', (async () => { + const note1 = await Note.save({ body: 'my note\nsecond line' }); + + const patch = Revision.createTextPatch(note1.body, 'my new note\nsecond line'); + const merged = Revision.applyTextPatch(note1.body, patch); + + expect(merged).toBe('my new note\nsecond line'); + })); + + it('should check if it is an empty revision', async () => { + const testCases = [ + [false, { + title_diff: '', + body_diff: '', + metadata_diff: '{"new":{"id":"aaa"},"deleted":[]}', + }], + [true, { + title_diff: '', + body_diff: '', + metadata_diff: '', + }], + [true, { + title_diff: '[]', + body_diff: '', + metadata_diff: '{"new":{},"deleted":[]}', + }], + [true, { + title_diff: '', + body_diff: '[]', + metadata_diff: '{"new":{},"deleted":[]}', + }], + [false, { + title_diff: '[{"diffs":[[1,"hello"]],"start1":0,"start2":0,"length1":0,"length2":5}]', + body_diff: '[]', + metadata_diff: '{"new":{},"deleted":[]}', + }], + ]; + + for (const t of testCases) { + const [expected, input] = t; + expect(Revision.isEmptyRevision(input as any)).toBe(expected); + } + }); + + it('should not fail to create revisions on naughty strings', (async () => { + // Previously this pattern would fail: + // - Create a patch between an empty string and smileys + // - Use that patch on the empty string to get back the smileys + // - Create a patch between those smileys and new smileys + // https://github.com/JackuB/diff-match-patch/issues/22 + + const nss = await naughtyStrings(); + + // First confirm that it indeed fails with the legacy approach. + let errorCount = 0; + + for (let i = 0; i < nss.length - 1; i++) { + const ns1 = nss[i]; + const ns2 = nss[i + 1]; + try { + const patchText = Revision.createTextPatchLegacy('', ns1); + const patchedText = Revision.applyTextPatchLegacy('', patchText); + Revision.createTextPatchLegacy(patchedText, ns2); + } catch (error) { + errorCount++; + } + } + + expect(errorCount).toBe(10); + + // Now feed the naughty list again but using the new approach. In that + // case it should work fine. + await expectNotThrow(async () => { + for (let i = 0; i < nss.length - 1; i++) { + const ns1 = nss[i]; + const ns2 = nss[i + 1]; + const patchText = Revision.createTextPatch('', ns1); + const patchedText = Revision.applyTextPatch('', patchText); + Revision.createTextPatch(patchedText, ns2); + } + }); + })); + + it('should successfully handle legacy patches', async () => { + // The code should handle applying a series of new style patches and + // legacy patches, and the correct text should be recovered at the end. + const changes = [ + '', + 'one', + 'one three', + 'one two three', + ]; + + const patches = [ + Revision.createTextPatch(changes[0], changes[1]), + Revision.createTextPatchLegacy(changes[1], changes[2]), + Revision.createTextPatch(changes[2], changes[3]), + ]; + + // Sanity check - verify that the patches are as expected + expect(patches[0].substr(0, 2)).toBe('[{'); // New + expect(patches[1].substr(0, 2)).toBe('@@'); // Legacy + expect(patches[2].substr(0, 2)).toBe('[{'); // New + + let finalString = Revision.applyTextPatch(changes[0], patches[0]); + finalString = Revision.applyTextPatch(finalString, patches[1]); + finalString = Revision.applyTextPatch(finalString, patches[2]); + + expect(finalString).toBe('one two three'); + }); + + it('should create patches of objects and apply it', (async () => { + const oldObject = { + one: '123', + two: '456', + three: '789', + }; + + const newObject = { + one: '123', + three: '999', + }; + + const patch = Revision.createObjectPatch(oldObject, newObject); + const merged = Revision.applyObjectPatch(oldObject, patch); + + expect(JSON.stringify(merged)).toBe(JSON.stringify(newObject)); + })); + + it('should move target revision to the top', (async () => { + const revs = [ + { id: '123' }, + { id: '456' }, + { id: '789' }, + ]; + + let newRevs; + newRevs = Revision.moveRevisionToTop({ id: '456' }, revs); + expect(newRevs[0].id).toBe('123'); + expect(newRevs[1].id).toBe('789'); + expect(newRevs[2].id).toBe('456'); + + newRevs = Revision.moveRevisionToTop({ id: '789' }, revs); + expect(newRevs[0].id).toBe('123'); + expect(newRevs[1].id).toBe('456'); + expect(newRevs[2].id).toBe('789'); + })); + + it('should create patch stats', (async () => { + const tests = [ + { + patch: `@@ -625,16 +625,48 @@ + rrupted download ++%0A- %5B %5D Fix mobile screen options`, + expected: [-0, +32], + }, + { + patch: `@@ -564,17 +564,17 @@ + ages%0A- %5B +- ++x + %5D Check `, + expected: [-1, +1], + }, + { + patch: `@@ -1022,56 +1022,415 @@ + .%0A%0A# +- How to view a note history%0A%0AWhile all the apps ++%C2%A0How does it work?%0A%0AAll the apps save a version of the modified notes every 10 minutes. + %0A%0A# `, + expected: [-(19 + 27 + 2), 17 + 67 + 4], + }, + ]; + + for (const test of tests) { + const stats = Revision.patchStats(test.patch); + expect(stats.removed).toBe(-test.expected[0]); + expect(stats.added).toBe(test.expected[1]); + } + })); + +}); diff --git a/packages/lib/models/Revision.ts b/packages/lib/models/Revision.ts index 91fcb513f..11ff38045 100644 --- a/packages/lib/models/Revision.ts +++ b/packages/lib/models/Revision.ts @@ -17,17 +17,54 @@ export default class Revision extends BaseItem { return BaseModel.TYPE_REVISION; } - static createTextPatch(oldText: string, newText: string) { + public static createTextPatchLegacy(oldText: string, newText: string): string { return dmp.patch_toText(dmp.patch_make(oldText, newText)); } - static applyTextPatch(text: string, patch: string) { + public static createTextPatch(oldText: string, newText: string): string { + return JSON.stringify(dmp.patch_make(oldText, newText)); + } + + public static applyTextPatchLegacy(text: string, patch: string): string { patch = dmp.patch_fromText(patch); const result = dmp.patch_apply(patch, text); if (!result || !result.length) throw new Error('Could not apply patch'); return result[0]; } + private static isLegacyPatch(patch: string): boolean { + return patch && patch.indexOf('@@') === 0; + } + + private static isNewPatch(patch: string): boolean { + if (!patch) return true; + return patch.indexOf('[{') === 0; + } + + public static applyTextPatch(text: string, patch: string): string { + if (this.isLegacyPatch(patch)) { + return this.applyTextPatchLegacy(text, patch); + } else { + const result = dmp.patch_apply(JSON.parse(patch), text); + if (!result || !result.length) throw new Error('Could not apply patch'); + return result[0]; + } + } + + public static isEmptyRevision(rev: RevisionEntity): boolean { + if (this.isLegacyPatch(rev.title_diff) && rev.title_diff) return false; + if (this.isLegacyPatch(rev.body_diff) && rev.body_diff) return false; + + if (this.isNewPatch(rev.title_diff) && rev.title_diff && rev.title_diff !== '[]') return false; + if (this.isNewPatch(rev.body_diff) && rev.body_diff && rev.body_diff !== '[]') return false; + + const md = rev.metadata_diff ? JSON.parse(rev.metadata_diff) : {}; + if (md.new && Object.keys(md.new).length) return false; + if (md.deleted && Object.keys(md.deleted).length) return false; + + return true; + } + static createObjectPatch(oldObject: any, newObject: any) { if (!oldObject) oldObject = {}; diff --git a/packages/lib/services/RevisionService.test.js b/packages/lib/services/RevisionService.test.ts similarity index 82% rename from packages/lib/services/RevisionService.test.js rename to packages/lib/services/RevisionService.test.ts index 709a43b69..423378363 100644 --- a/packages/lib/services/RevisionService.test.js +++ b/packages/lib/services/RevisionService.test.ts @@ -1,13 +1,11 @@ -/* eslint-disable no-unused-vars, @typescript-eslint/no-unused-vars, prefer-const */ - -const time = require('../time').default; -const { revisionService, setupDatabaseAndSynchronizer, switchClient } = require('../testing/test-utils.js'); -const Setting = require('../models/Setting').default; -const Note = require('../models/Note').default; -const ItemChange = require('../models/ItemChange').default; -const Revision = require('../models/Revision').default; -const BaseModel = require('../BaseModel').default; -const RevisionService = require('../services/RevisionService').default; +import time from '../time'; +import { revisionService, setupDatabaseAndSynchronizer, switchClient } from '../testing/test-utils'; +import Setting from '../models/Setting'; +import Note from '../models/Note'; +import ItemChange from '../models/ItemChange'; +import Revision from '../models/Revision'; +import BaseModel from '../BaseModel'; +import RevisionService from '../services/RevisionService'; describe('services_Revision', function() { @@ -25,7 +23,7 @@ describe('services_Revision', function() { await service.collectRevisions(); await Note.save({ id: n1_v1.id, title: 'hello', author: 'testing' }); await service.collectRevisions(); - const n1_v2 = await Note.save({ id: n1_v1.id, title: 'hello welcome', author: '' }); + await Note.save({ id: n1_v1.id, title: 'hello welcome', author: '' }); await service.collectRevisions(); const revisions = await Revision.allByType(BaseModel.TYPE_NOTE, n1_v1.id); @@ -49,6 +47,46 @@ describe('services_Revision', function() { expect(revisions2.length).toBe(0); })); + // ---------------------------------------------------------------------- + // This is to verify that the revision service continues processing + // revisions even when it fails on one note. However, now that the + // diff-match-patch bug is fixed, it's not possible to create notes that + // would make the process fail. Keeping the test anyway in case such case + // comes up again. + // ---------------------------------------------------------------------- + + // it('should handle corrupted strings', (async () => { + // const service = new RevisionService(); + + // // Silence the logger because the revision service is going to print + // // errors. + // // Logger.globalLogger.enabled = false; + + // const n1 = await Note.save({ body: '' }); + // await service.collectRevisions(); + // await Note.save({ id: n1.id, body: naughtyStrings[152] }); // REV 1 + // await service.collectRevisions(); + // await Note.save({ id: n1.id, body: naughtyStrings[153] }); // FAIL (Should have been REV 2) + // await service.collectRevisions(); + + // // Because it fails, only one revision was generated. The second was skipped. + // expect((await Revision.all()).length).toBe(1); + + // // From this point, note 1 will always fail because of a + // // diff-match-patch bug: + // // https://github.com/JackuB/diff-match-patch/issues/22 + // // It will throw "URI malformed". But it shouldn't prevent other notes + // // from getting revisions. + + // const n2 = await Note.save({ body: '' }); + // await service.collectRevisions(); + // await Note.save({ id: n2.id, body: 'valid' }); // REV 2 + // await service.collectRevisions(); + // expect((await Revision.all()).length).toBe(2); + + // Logger.globalLogger.enabled = true; + // })); + it('should delete old revisions (1 note, 2 rev)', (async () => { const service = new RevisionService(); @@ -59,7 +97,7 @@ describe('services_Revision', function() { const time_v1 = Date.now(); await time.msleep(100); - const n1_v2 = await Note.save({ id: n1_v1.id, title: 'hello welcome' }); + await Note.save({ id: n1_v1.id, title: 'hello welcome' }); await service.collectRevisions(); expect((await Revision.allByType(BaseModel.TYPE_NOTE, n1_v1.id)).length).toBe(2); @@ -81,12 +119,12 @@ describe('services_Revision', function() { const time_v1 = Date.now(); await time.msleep(100); - const n1_v2 = await Note.save({ id: n1_v1.id, title: 'one two' }); + await Note.save({ id: n1_v1.id, title: 'one two' }); await service.collectRevisions(); const time_v2 = Date.now(); await time.msleep(100); - const n1_v3 = await Note.save({ id: n1_v1.id, title: 'one two three' }); + await Note.save({ id: n1_v1.id, title: 'one two three' }); await service.collectRevisions(); { @@ -124,8 +162,8 @@ describe('services_Revision', function() { const time_n2_v1 = Date.now(); await time.msleep(100); - const n1_v2 = await Note.save({ id: n1_v1.id, title: 'note 1 (v2)' }); - const n2_v2 = await Note.save({ id: n2_v1.id, title: 'note 2 (v2)' }); + await Note.save({ id: n1_v1.id, title: 'note 1 (v2)' }); + await Note.save({ id: n2_v1.id, title: 'note 2 (v2)' }); await service.collectRevisions(); expect((await Revision.all()).length).toBe(4); @@ -167,9 +205,9 @@ describe('services_Revision', function() { const noteId = n1_v1.id; const rev1 = await service.createNoteRevision_(n1_v1); const n1_v2 = await Note.save({ id: noteId, title: 'hello Paul' }); - const rev2 = await service.createNoteRevision_(n1_v2, rev1.id); + await service.createNoteRevision_(n1_v2, rev1.id); const n1_v3 = await Note.save({ id: noteId, title: 'hello John' }); - const rev3 = await service.createNoteRevision_(n1_v3, rev1.id); + await service.createNoteRevision_(n1_v3, rev1.id); const revisions = await Revision.allByType(BaseModel.TYPE_NOTE, noteId); expect(revisions.length).toBe(3); @@ -311,7 +349,7 @@ describe('services_Revision', function() { const n1_v1 = await Note.save({ id: n1_v0.id, title: 'hello' }); await revisionService().collectRevisions(); // REV 1 await time.sleep(0.1); - const n1_v2 = await Note.save({ id: n1_v1.id, title: 'hello welcome' }); + await Note.save({ id: n1_v1.id, title: 'hello welcome' }); await revisionService().collectRevisions(); // REV 2 await time.sleep(0.1); @@ -340,7 +378,7 @@ describe('services_Revision', function() { const timeRev1 = Date.now(); await time.msleep(100); - const n1_v2 = await Note.save({ id: n1_v1.id, title: 'hello welcome' }); + await Note.save({ id: n1_v1.id, title: 'hello welcome' }); await revisionService().collectRevisions(); // REV 2 expect((await Revision.all()).length).toBe(2); @@ -364,7 +402,7 @@ describe('services_Revision', function() { const timeRev1 = Date.now(); await time.msleep(100); - const n1_v2 = await Note.save({ id: n1_v1.id, title: 'hello welcome' }); + await Note.save({ id: n1_v1.id, title: 'hello welcome' }); await revisionService().collectRevisions(); // REV 2 expect((await Revision.all()).length).toBe(2); @@ -385,11 +423,11 @@ describe('services_Revision', function() { it('should not create a revision if the note has not changed', (async () => { const n1_v0 = await Note.save({ title: '' }); - const n1_v1 = await Note.save({ id: n1_v0.id, title: 'hello' }); + await Note.save({ id: n1_v0.id, title: 'hello' }); await revisionService().collectRevisions(); // REV 1 expect((await Revision.all()).length).toBe(1); - const n1_v2 = await Note.save({ id: n1_v0.id, title: 'hello' }); + await Note.save({ id: n1_v0.id, title: 'hello' }); await revisionService().collectRevisions(); // Note has not changed (except its timestamp) so don't create a revision expect((await Revision.all()).length).toBe(1); })); @@ -399,12 +437,12 @@ describe('services_Revision', function() { // places so make sure it is saved correctly with the revision const n1_v0 = await Note.save({ title: '' }); - const n1_v1 = await Note.save({ id: n1_v0.id, title: 'hello' }); + await Note.save({ id: n1_v0.id, title: 'hello' }); await revisionService().collectRevisions(); // REV 1 expect((await Revision.all()).length).toBe(1); const userUpdatedTime = Date.now() - 1000 * 60 * 60; - const n1_v2 = await Note.save({ id: n1_v0.id, title: 'hello', updated_time: Date.now(), user_updated_time: userUpdatedTime }, { autoTimestamp: false }); + await Note.save({ id: n1_v0.id, title: 'hello', updated_time: Date.now(), user_updated_time: userUpdatedTime }, { autoTimestamp: false }); await revisionService().collectRevisions(); // Only the user timestamp has changed, but that needs to be saved const revisions = await Revision.all(); @@ -416,20 +454,20 @@ describe('services_Revision', function() { it('should not create a revision if there is already a recent one', (async () => { const n1_v0 = await Note.save({ title: '' }); - const n1_v1 = await Note.save({ id: n1_v0.id, title: 'hello' }); + await Note.save({ id: n1_v0.id, title: 'hello' }); await revisionService().collectRevisions(); // REV 1 const timeRev1 = Date.now(); await time.sleep(2); const timeRev2 = Date.now(); - const n1_v2 = await Note.save({ id: n1_v0.id, title: 'hello 2' }); + await Note.save({ id: n1_v0.id, title: 'hello 2' }); await revisionService().collectRevisions(); // REV 2 expect((await Revision.all()).length).toBe(2); const interval = Date.now() - timeRev1 + 1; Setting.setValue('revisionService.intervalBetweenRevisions', interval); - const n1_v3 = await Note.save({ id: n1_v0.id, title: 'hello 3' }); + await Note.save({ id: n1_v0.id, title: 'hello 3' }); await revisionService().collectRevisions(); // No rev because time since last rev is less than the required 'interval between revisions' expect(Date.now() - interval < timeRev2).toBe(true); // check the computer is not too slow for this test expect((await Revision.all()).length).toBe(2); diff --git a/packages/lib/services/RevisionService.ts b/packages/lib/services/RevisionService.ts index b13dd58d4..f8836647a 100644 --- a/packages/lib/services/RevisionService.ts +++ b/packages/lib/services/RevisionService.ts @@ -9,10 +9,13 @@ import shim from '../shim'; import BaseService from './BaseService'; import { _ } from '../locale'; import { ItemChangeEntity, NoteEntity, RevisionEntity } from './database/types'; +import Logger from '../Logger'; const { substrWithEllipsis } = require('../string-utils'); const { sprintf } = require('sprintf-js'); const { wrapError } = require('../errorUtils'); +const logger = Logger.create('RevisionService'); + export default class RevisionService extends BaseService { public static instance_: RevisionService; @@ -60,18 +63,7 @@ export default class RevisionService extends BaseService { return md; } - isEmptyRevision_(rev: RevisionEntity) { - if (rev.title_diff) return false; - if (rev.body_diff) return false; - - const md = JSON.parse(rev.metadata_diff); - if (md.new && Object.keys(md.new).length) return false; - if (md.deleted && Object.keys(md.deleted).length) return false; - - return true; - } - - async createNoteRevision_(note: NoteEntity, parentRevId: string = null) { + public async createNoteRevision_(note: NoteEntity, parentRevId: string = null): Promise<RevisionEntity> { try { const parentRev = parentRevId ? await Revision.load(parentRevId) : await Revision.latestRevision(BaseModel.TYPE_NOTE, note.id); @@ -100,7 +92,7 @@ export default class RevisionService extends BaseService { output.metadata_diff = Revision.createObjectPatch(merged.metadata, noteMd); } - if (this.isEmptyRevision_(output)) return null; + if (Revision.isEmptyRevision(output)) return null; return Revision.save(output); } catch (error) { @@ -109,7 +101,7 @@ export default class RevisionService extends BaseService { } } - async collectRevisions() { + public async collectRevisions() { if (this.isCollecting_) return; this.isCollecting_ = true; @@ -153,11 +145,11 @@ export default class RevisionService extends BaseService { if (oldNote && oldNote.updated_time < this.oldNoteCutOffDate_()) { // This is where we save the original version of this old note const rev = await this.createNoteRevision_(oldNote); - if (rev) this.logger().debug(sprintf('RevisionService::collectRevisions: Saved revision %s (old note)', rev.id)); + if (rev) logger.debug(sprintf('RevisionService::collectRevisions: Saved revision %s (old note)', rev.id)); } const rev = await this.createNoteRevision_(note); - if (rev) this.logger().debug(sprintf('RevisionService::collectRevisions: Saved revision %s (Last rev was more than %d ms ago)', rev.id, Setting.value('revisionService.intervalBetweenRevisions'))); + if (rev) logger.debug(sprintf('RevisionService::collectRevisions: Saved revision %s (Last rev was more than %d ms ago)', rev.id, Setting.value('revisionService.intervalBetweenRevisions'))); doneNoteIds.push(noteId); this.isOldNotesCache_[noteId] = false; } @@ -168,7 +160,7 @@ export default class RevisionService extends BaseService { const revExists = await Revision.revisionExists(BaseModel.TYPE_NOTE, note.id, note.updated_time); if (!revExists) { const rev = await this.createNoteRevision_(note); - if (rev) this.logger().debug(sprintf('RevisionService::collectRevisions: Saved revision %s (for deleted note)', rev.id)); + if (rev) logger.debug(sprintf('RevisionService::collectRevisions: Saved revision %s (for deleted note)', rev.id)); } doneNoteIds.push(noteId); } @@ -181,9 +173,15 @@ export default class RevisionService extends BaseService { // One or more revisions are encrypted - stop processing for now // and these revisions will be processed next time the revision // collector runs. - this.logger().info('RevisionService::collectRevisions: One or more revision was encrypted. Processing was stopped but will resume later when the revision is decrypted.', error); + logger.info('RevisionService::collectRevisions: One or more revision was encrypted. Processing was stopped but will resume later when the revision is decrypted.', error); } else { - this.logger().error('RevisionService::collectRevisions:', error); + // Note that, for now, if any revision creation fails, the whole + // process fails. This is on purpose because if we keep on + // processing, whatever caused the error will be in the past + // changes (before revisionService.lastProcessedChangeId) and + // will never be processed again. Now that the diff-match-patch + // issue is fixed, there should be no such error anyway. + logger.error('RevisionService::collectRevisions:', error); } } @@ -192,7 +190,7 @@ export default class RevisionService extends BaseService { this.isCollecting_ = false; - this.logger().info(`RevisionService::collectRevisions: Created revisions for ${doneNoteIds.length} notes`); + logger.info(`RevisionService::collectRevisions: Created revisions for ${doneNoteIds.length} notes`); } async deleteOldRevisions(ttl: number) { @@ -266,23 +264,23 @@ export default class RevisionService extends BaseService { this.maintenanceCalls_.push(true); try { const startTime = Date.now(); - this.logger().info('RevisionService::maintenance: Starting...'); + logger.info('RevisionService::maintenance: Starting...'); if (!Setting.value('revisionService.enabled')) { - this.logger().info('RevisionService::maintenance: Service is disabled'); + logger.info('RevisionService::maintenance: Service is disabled'); // We do as if we had processed all the latest changes so that they can be cleaned up // later on by ItemChangeUtils.deleteProcessedChanges(). Setting.setValue('revisionService.lastProcessedChangeId', await ItemChange.lastChangeId()); await this.deleteOldRevisions(Setting.value('revisionService.ttlDays') * 24 * 60 * 60 * 1000); } else { - this.logger().info('RevisionService::maintenance: Service is enabled'); + logger.info('RevisionService::maintenance: Service is enabled'); await this.collectRevisions(); await this.deleteOldRevisions(Setting.value('revisionService.ttlDays') * 24 * 60 * 60 * 1000); - this.logger().info(`RevisionService::maintenance: Done in ${Date.now() - startTime}ms`); + logger.info(`RevisionService::maintenance: Done in ${Date.now() - startTime}ms`); } } catch (error) { - this.logger().error('RevisionService::maintenance:', error); + logger.error('RevisionService::maintenance:', error); } finally { this.maintenanceCalls_.pop(); } @@ -294,7 +292,7 @@ export default class RevisionService extends BaseService { if (collectRevisionInterval === null) collectRevisionInterval = 1000 * 60 * 10; - this.logger().info(`RevisionService::runInBackground: Starting background service with revision collection interval ${collectRevisionInterval}`); + logger.info(`RevisionService::runInBackground: Starting background service with revision collection interval ${collectRevisionInterval}`); this.maintenanceTimer1_ = shim.setTimeout(() => { void this.maintenance(); diff --git a/packages/lib/testing/test-utils.ts b/packages/lib/testing/test-utils.ts index c608dc2e8..e93afb428 100644 --- a/packages/lib/testing/test-utils.ts +++ b/packages/lib/testing/test-utils.ts @@ -17,7 +17,7 @@ import FileApiDriverJoplinServer from '../file-api-driver-joplinServer'; import OneDriveApi from '../onedrive-api'; import SyncTargetOneDrive from '../SyncTargetOneDrive'; import JoplinDatabase from '../JoplinDatabase'; -const fs = require('fs-extra'); +import * as fs from 'fs-extra'; const { DatabaseDriverNode } = require('../database-driver-node.js'); import Folder from '../models/Folder'; import Note from '../models/Note'; @@ -101,8 +101,8 @@ const supportDir = `${oldTestDir}/support`; const dataDir = `${oldTestDir}/test data/${suiteName_}`; const profileDir = `${dataDir}/profile`; -fs.mkdirpSync(logDir, 0o755); -fs.mkdirpSync(baseTempDir, 0o755); +fs.mkdirpSync(logDir); +fs.mkdirpSync(baseTempDir); fs.mkdirpSync(dataDir); fs.mkdirpSync(profileDir); @@ -392,10 +392,10 @@ async function setupDatabaseAndSynchronizer(id: number, options: any = null) { DecryptionWorker.instance_ = null; await fs.remove(resourceDir(id)); - await fs.mkdirp(resourceDir(id), 0o755); + await fs.mkdirp(resourceDir(id)); await fs.remove(pluginDir(id)); - await fs.mkdirp(pluginDir(id), 0o755); + await fs.mkdirp(pluginDir(id)); if (!synchronizers_[id]) { const SyncTargetClass = SyncTargetRegistry.classById(syncTargetId_); @@ -512,7 +512,7 @@ async function initFileApi() { let fileApi = null; if (syncTargetId_ == SyncTargetRegistry.nameToId('filesystem')) { fs.removeSync(syncDir); - fs.mkdirpSync(syncDir, 0o755); + fs.mkdirpSync(syncDir); fileApi = new FileApi(syncDir, new FileApiDriverLocal()); } else if (syncTargetId_ == SyncTargetRegistry.nameToId('memory')) { fileApi = new FileApi('/root', new FileApiDriverMemory()); @@ -788,6 +788,21 @@ async function waitForFolderCount(count: number) { } } +let naughtyStrings_: string[] = null; +export async function naughtyStrings() { + if (naughtyStrings_) return naughtyStrings_; + const t = await fs.readFile(`${supportDir}/big-list-of-naughty-strings.txt`, 'utf8'); + const lines = t.split('\n'); + naughtyStrings_ = []; + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed) continue; + if (trimmed.indexOf('#') === 0) continue; + naughtyStrings_.push(line); + } + return naughtyStrings_; +} + // TODO: Update for Jest // function mockDate(year, month, day, tick) {