diff --git a/.travis.yml b/.travis.yml index 0c010b7a..a63184a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,8 @@ language: node_js node_js: - '10' - '11' + - '12' + - '13' env: - MYSQL_HOST='localhost' MYSQL_PASSWORD='' MYSQL_USERNAME='root' MYSQL_DATABASE='pigallery2_test' services: diff --git a/backend/model/threading/DiskMangerWorker.ts b/backend/model/threading/DiskMangerWorker.ts index ba59c8b4..40682415 100644 --- a/backend/model/threading/DiskMangerWorker.ts +++ b/backend/model/threading/DiskMangerWorker.ts @@ -57,6 +57,39 @@ export class DiskMangerWorker { return path.basename(name); } + public static excludeDir(name: string,relativeDirectoryName: string, absoluteDirectoryName: string) { + const absoluteName=path.normalize(path.join(absoluteDirectoryName,name)); + const relativeName=path.normalize(path.join(relativeDirectoryName,name)); + + for (let j = 0; j < Config.Server.indexing.excludeFolderList.length; j++) { + const exclude=Config.Server.indexing.excludeFolderList[j]; + + if (exclude.startsWith('/')) { + if (exclude==absoluteName) { + return true; + } + } else if (exclude.includes('/')) { + if (path.normalize(exclude)==relativeName) { + return true; + } + } else { + if (exclude==name) { + return true; + } + } + } + + for (let j = 0; j < Config.Server.indexing.excludeFileList.length; j++) { + const exclude=Config.Server.indexing.excludeFileList[j]; + + if (fs.existsSync(path.join(absoluteName,exclude))) { + return true; + } + } + + return false; + } + public static scanDirectory(relativeDirectoryName: string, maxPhotos: number = null, photosOnly: boolean = false): Promise { return new Promise((resolve, reject) => { relativeDirectoryName = this.normalizeDirPath(relativeDirectoryName); @@ -90,6 +123,9 @@ export class DiskMangerWorker { if (photosOnly === true) { continue; } + if (DiskMangerWorker.excludeDir(file,relativeDirectoryName,absoluteDirectoryName)) { + continue; + } const d = await DiskMangerWorker.scanDirectory(path.join(relativeDirectoryName, file), Config.Server.indexing.folderPreviewSize, true ); diff --git a/benchmark/Benchmarks.js b/benchmark/Benchmarks.js new file mode 100644 index 00000000..7d924c53 --- /dev/null +++ b/benchmark/Benchmarks.js @@ -0,0 +1,304 @@ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (_) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +var SQLConnection_1 = require("../backend/model/sql/SQLConnection"); +var Config_1 = require("../common/config/private/Config"); +var IPrivateConfig_1 = require("../common/config/private/IPrivateConfig"); +var ObjectManagers_1 = require("../backend/model/ObjectManagers"); +var DiskMangerWorker_1 = require("../backend/model/threading/DiskMangerWorker"); +var IndexingManager_1 = require("../backend/model/sql/IndexingManager"); +var SearchManager_1 = require("../backend/model/sql/SearchManager"); +var fs = require("fs"); +var AutoCompleteItem_1 = require("../common/entities/AutoCompleteItem"); +var Utils_1 = require("../common/Utils"); +var GalleryManager_1 = require("../backend/model/sql/GalleryManager"); +var BMIndexingManager = /** @class */ (function (_super) { + __extends(BMIndexingManager, _super); + function BMIndexingManager() { + return _super !== null && _super.apply(this, arguments) || this; + } + BMIndexingManager.prototype.saveToDB = function (scannedDirectory) { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_a) { + return [2 /*return*/, _super.prototype.saveToDB.call(this, scannedDirectory)]; + }); + }); + }; + return BMIndexingManager; +}(IndexingManager_1.IndexingManager)); +exports.BMIndexingManager = BMIndexingManager; +var Benchmarks = /** @class */ (function () { + function Benchmarks(RUNS, dbPath) { + var _this = this; + this.RUNS = RUNS; + this.dbPath = dbPath; + this.resetDB = function () { return __awaiter(_this, void 0, void 0, function () { + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, SQLConnection_1.SQLConnection.close()]; + case 1: + _a.sent(); + if (fs.existsSync(this.dbPath)) { + fs.unlinkSync(this.dbPath); + } + Config_1.Config.Server.database.type = IPrivateConfig_1.DatabaseType.sqlite; + Config_1.Config.Server.database.sqlite.storage = this.dbPath; + return [4 /*yield*/, ObjectManagers_1.ObjectManagers.InitSQLManagers()]; + case 2: + _a.sent(); + return [2 /*return*/]; + } + }); + }); }; + } + Benchmarks.prototype.bmSaveDirectory = function () { + return __awaiter(this, void 0, void 0, function () { + var dir, im; + var _this = this; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, this.resetDB()]; + case 1: + _a.sent(); + return [4 /*yield*/, DiskMangerWorker_1.DiskMangerWorker.scanDirectory('./')]; + case 2: + dir = _a.sent(); + im = new BMIndexingManager(); + return [4 /*yield*/, this.benchmark(function () { return im.saveToDB(dir); }, function () { return _this.resetDB(); })]; + case 3: return [2 /*return*/, _a.sent()]; + } + }); + }); + }; + Benchmarks.prototype.bmScanDirectory = function () { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, this.benchmark(function () { return DiskMangerWorker_1.DiskMangerWorker.scanDirectory('./'); })]; + case 1: return [2 /*return*/, _a.sent()]; + } + }); + }); + }; + Benchmarks.prototype.bmListDirectory = function () { + return __awaiter(this, void 0, void 0, function () { + var gm; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + gm = new GalleryManager_1.GalleryManager(); + return [4 /*yield*/, this.setupDB()]; + case 1: + _a.sent(); + Config_1.Config.Server.indexing.reIndexingSensitivity = IPrivateConfig_1.ReIndexingSensitivity.low; + return [4 /*yield*/, this.benchmark(function () { return gm.listDirectory('./'); })]; + case 2: return [2 /*return*/, _a.sent()]; + } + }); + }); + }; + Benchmarks.prototype.bmAllSearch = function (text) { + return __awaiter(this, void 0, void 0, function () { + var types, results, sm, _loop_1, this_1, i; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, this.setupDB()]; + case 1: + _a.sent(); + types = Utils_1.Utils.enumToArray(AutoCompleteItem_1.SearchTypes).map(function (a) { return a.key; }).concat([null]); + results = []; + sm = new SearchManager_1.SearchManager(); + _loop_1 = function (i) { + var _a, _b, _c; + return __generator(this, function (_d) { + switch (_d.label) { + case 0: + _b = (_a = results).push; + _c = {}; + return [4 /*yield*/, this_1.benchmark(function () { return sm.search(text, types[i]); })]; + case 1: + _b.apply(_a, [(_c.result = _d.sent(), _c.searchType = types[i], _c)]); + return [2 /*return*/]; + } + }); + }; + this_1 = this; + i = 0; + _a.label = 2; + case 2: + if (!(i < types.length)) return [3 /*break*/, 5]; + return [5 /*yield**/, _loop_1(i)]; + case 3: + _a.sent(); + _a.label = 4; + case 4: + i++; + return [3 /*break*/, 2]; + case 5: return [2 /*return*/, results]; + } + }); + }); + }; + Benchmarks.prototype.bmInstantSearch = function (text) { + return __awaiter(this, void 0, void 0, function () { + var sm; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, this.setupDB()]; + case 1: + _a.sent(); + sm = new SearchManager_1.SearchManager(); + return [4 /*yield*/, this.benchmark(function () { return sm.instantSearch(text); })]; + case 2: return [2 /*return*/, _a.sent()]; + } + }); + }); + }; + Benchmarks.prototype.bmAutocomplete = function (text) { + return __awaiter(this, void 0, void 0, function () { + var sm; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, this.setupDB()]; + case 1: + _a.sent(); + sm = new SearchManager_1.SearchManager(); + return [4 /*yield*/, this.benchmark(function () { return sm.autocomplete(text); })]; + case 2: return [2 /*return*/, _a.sent()]; + } + }); + }); + }; + Benchmarks.prototype.benchmark = function (fn, beforeEach, afterEach) { + if (beforeEach === void 0) { beforeEach = null; } + if (afterEach === void 0) { afterEach = null; } + return __awaiter(this, void 0, void 0, function () { + var scanned, start, skip, i, startSkip, endSkip, startSkip, endSkip, end, duration; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, fn()]; + case 1: + scanned = _a.sent(); + start = process.hrtime(); + skip = 0; + i = 0; + _a.label = 2; + case 2: + if (!(i < this.RUNS)) return [3 /*break*/, 8]; + if (!beforeEach) return [3 /*break*/, 4]; + startSkip = process.hrtime(); + return [4 /*yield*/, beforeEach()]; + case 3: + _a.sent(); + endSkip = process.hrtime(startSkip); + skip += (endSkip[0] * 1000 + endSkip[1] / 1000000); + _a.label = 4; + case 4: return [4 /*yield*/, fn()]; + case 5: + _a.sent(); + if (!afterEach) return [3 /*break*/, 7]; + startSkip = process.hrtime(); + return [4 /*yield*/, afterEach()]; + case 6: + _a.sent(); + endSkip = process.hrtime(startSkip); + skip += (endSkip[0] * 1000 + endSkip[1] / 1000000); + _a.label = 7; + case 7: + i++; + return [3 /*break*/, 2]; + case 8: + end = process.hrtime(start); + duration = (end[0] * 1000 + end[1] / 1000000) / this.RUNS; + if (!scanned) { + return [2 /*return*/, { + duration: duration + }]; + } + if (Array.isArray(scanned)) { + return [2 /*return*/, { + duration: duration, + items: scanned.length + }]; + } + return [2 /*return*/, { + duration: duration, + media: scanned.media.length, + directories: scanned.directories.length + }]; + } + }); + }); + }; + Benchmarks.prototype.setupDB = function () { + return __awaiter(this, void 0, void 0, function () { + var im, dir; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + im = new BMIndexingManager(); + return [4 /*yield*/, this.resetDB()]; + case 1: + _a.sent(); + return [4 /*yield*/, DiskMangerWorker_1.DiskMangerWorker.scanDirectory('./')]; + case 2: + dir = _a.sent(); + return [4 /*yield*/, im.saveToDB(dir)]; + case 3: + _a.sent(); + return [2 /*return*/]; + } + }); + }); + }; + return Benchmarks; +}()); +exports.Benchmarks = Benchmarks; +//# sourceMappingURL=Benchmarks.js.map \ No newline at end of file diff --git a/benchmark/Benchmarks.js.map b/benchmark/Benchmarks.js.map new file mode 100644 index 00000000..039550d0 --- /dev/null +++ b/benchmark/Benchmarks.js.map @@ -0,0 +1 @@ +{"version":3,"file":"Benchmarks.js","sourceRoot":"","sources":["Benchmarks.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,oEAAiE;AACjE,0DAAuD;AACvD,0EAA4F;AAC5F,kEAA+D;AAC/D,gFAA6E;AAC7E,wEAAqE;AACrE,oEAAiE;AACjE,uBAAyB;AACzB,wEAAgE;AAChE,yCAAsC;AACtC,sEAAmE;AAUnE;IAAuC,qCAAe;IAAtD;;IAKA,CAAC;IAHc,oCAAQ,GAArB,UAAsB,gBAA8B;;;gBAClD,sBAAO,iBAAM,QAAQ,YAAC,gBAAgB,CAAC,EAAC;;;KACzC;IACH,wBAAC;AAAD,CAAC,AALD,CAAuC,iCAAe,GAKrD;AALY,8CAAiB;AAO9B;IAEE,oBAAmB,IAAY,EAAS,MAAc;QAAtD,iBAEC;QAFkB,SAAI,GAAJ,IAAI,CAAQ;QAAS,WAAM,GAAN,MAAM,CAAQ;QA0F9C,YAAO,GAAG;;;4BAChB,qBAAM,6BAAa,CAAC,KAAK,EAAE,EAAA;;wBAA3B,SAA2B,CAAC;wBAC5B,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE;4BAC9B,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;yBAC5B;wBACD,eAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,GAAG,6BAAY,CAAC,MAAM,CAAC;wBAClD,eAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC;wBACpD,qBAAM,+BAAc,CAAC,eAAe,EAAE,EAAA;;wBAAtC,SAAsC,CAAC;;;;aACxC,CAAC;IAhGF,CAAC;IAEK,oCAAe,GAArB;;;;;;4BACE,qBAAM,IAAI,CAAC,OAAO,EAAE,EAAA;;wBAApB,SAAoB,CAAC;wBACT,qBAAM,mCAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,EAAA;;wBAAhD,GAAG,GAAG,SAA0C;wBAChD,EAAE,GAAG,IAAI,iBAAiB,EAAE,CAAC;wBAC5B,qBAAM,IAAI,CAAC,SAAS,CAAC,cAAM,OAAA,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAhB,CAAgB,EAAE,cAAM,OAAA,KAAI,CAAC,OAAO,EAAE,EAAd,CAAc,CAAC,EAAA;4BAAzE,sBAAO,SAAkE,EAAC;;;;KAC3E;IAEK,oCAAe,GAArB;;;;4BACS,qBAAM,IAAI,CAAC,SAAS,CAAC,cAAM,OAAA,mCAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,EAApC,CAAoC,CAAC,EAAA;4BAAvE,sBAAO,SAAgE,EAAC;;;;KACzE;IAEK,oCAAe,GAArB;;;;;;wBACQ,EAAE,GAAG,IAAI,+BAAc,EAAE,CAAC;wBAChC,qBAAM,IAAI,CAAC,OAAO,EAAE,EAAA;;wBAApB,SAAoB,CAAC;wBACrB,eAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,qBAAqB,GAAG,sCAAqB,CAAC,GAAG,CAAC;wBAClE,qBAAM,IAAI,CAAC,SAAS,CAAC,cAAM,OAAA,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,EAAtB,CAAsB,CAAC,EAAA;4BAAzD,sBAAO,SAAkD,EAAC;;;;KAC3D;IAEK,gCAAW,GAAjB,UAAkB,IAAY;;;;;4BAC5B,qBAAM,IAAI,CAAC,OAAO,EAAE,EAAA;;wBAApB,SAAoB,CAAC;wBACf,KAAK,GAAG,aAAK,CAAC,WAAW,CAAC,8BAAW,CAAC,CAAC,GAAG,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,CAAC,GAAG,EAAL,CAAK,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;wBACtE,OAAO,GAA2D,EAAE,CAAC;wBACrE,EAAE,GAAG,IAAI,6BAAa,EAAE,CAAC;4CACtB,CAAC;;;;;wCACR,KAAA,CAAA,KAAA,OAAO,CAAA,CAAC,IAAI,CAAA;;wCAAU,qBAAM,OAAK,SAAS,CAAC,cAAM,OAAA,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAzB,CAAyB,CAAC,EAAA;;wCAA3E,eAAc,SAAM,GAAE,SAAqD,EAAE,aAAU,GAAE,KAAK,CAAC,CAAC,CAAC,OAAE,CAAC;;;;;;wBAD7F,CAAC,GAAG,CAAC;;;6BAAE,CAAA,CAAC,GAAG,KAAK,CAAC,MAAM,CAAA;sDAAvB,CAAC;;;;;wBAAwB,CAAC,EAAE,CAAA;;4BAGrC,sBAAO,OAAO,EAAC;;;;KAChB;IAEK,oCAAe,GAArB,UAAsB,IAAY;;;;;4BAChC,qBAAM,IAAI,CAAC,OAAO,EAAE,EAAA;;wBAApB,SAAoB,CAAC;wBACf,EAAE,GAAG,IAAI,6BAAa,EAAE,CAAC;wBACxB,qBAAM,IAAI,CAAC,SAAS,CAAC,cAAM,OAAA,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,EAAtB,CAAsB,CAAC,EAAA;4BAAzD,sBAAO,SAAkD,EAAC;;;;KAC3D;IAEK,mCAAc,GAApB,UAAqB,IAAY;;;;;4BAC/B,qBAAM,IAAI,CAAC,OAAO,EAAE,EAAA;;wBAApB,SAAoB,CAAC;wBACf,EAAE,GAAG,IAAI,6BAAa,EAAE,CAAC;wBACxB,qBAAM,IAAI,CAAC,SAAS,CAAC,cAAM,OAAA,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,EAArB,CAAqB,CAAC,EAAA;4BAAxD,sBAAO,SAAiD,EAAC;;;;KAC1D;IAGa,8BAAS,GAAvB,UAAwB,EAAsE,EACtE,UAAqC,EACrC,SAAoC;QADpC,2BAAA,EAAA,iBAAqC;QACrC,0BAAA,EAAA,gBAAoC;;;;;4BAC1C,qBAAM,EAAE,EAAE,EAAA;;wBAApB,OAAO,GAAG,SAAU;wBACpB,KAAK,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;wBAC3B,IAAI,GAAG,CAAC,CAAC;wBACJ,CAAC,GAAG,CAAC;;;6BAAE,CAAA,CAAC,GAAG,IAAI,CAAC,IAAI,CAAA;6BACvB,UAAU,EAAV,wBAAU;wBACN,SAAS,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;wBACnC,qBAAM,UAAU,EAAE,EAAA;;wBAAlB,SAAkB,CAAC;wBACb,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;wBAC1C,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC;;4BAErD,qBAAM,EAAE,EAAE,EAAA;;wBAAV,SAAU,CAAC;6BACP,SAAS,EAAT,wBAAS;wBACL,SAAS,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;wBACnC,qBAAM,SAAS,EAAE,EAAA;;wBAAjB,SAAiB,CAAC;wBACZ,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;wBAC1C,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC;;;wBAZxB,CAAC,EAAE,CAAA;;;wBAe5B,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;wBAC5B,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC;wBAEhE,IAAI,CAAC,OAAO,EAAE;4BACZ,sBAAO;oCACL,QAAQ,EAAE,QAAQ;iCACnB,EAAC;yBACH;wBAED,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;4BAC1B,sBAAO;oCACL,QAAQ,EAAE,QAAQ;oCAClB,KAAK,EAAE,OAAO,CAAC,MAAM;iCACtB,EAAC;yBACH;wBAED,sBAAO;gCACL,QAAQ,EAAE,QAAQ;gCAClB,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,MAAM;gCAC3B,WAAW,EAAE,OAAO,CAAC,WAAW,CAAC,MAAM;6BACxC,EAAC;;;;KACH;IAYa,4BAAO,GAArB;;;;;;wBACQ,EAAE,GAAG,IAAI,iBAAiB,EAAE,CAAC;wBACnC,qBAAM,IAAI,CAAC,OAAO,EAAE,EAAA;;wBAApB,SAAoB,CAAC;wBACT,qBAAM,mCAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,EAAA;;wBAAhD,GAAG,GAAG,SAA0C;wBACtD,qBAAM,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAA;;wBAAtB,SAAsB,CAAC;;;;;KACxB;IAEH,iBAAC;AAAD,CAAC,AA7GD,IA6GC;AA7GY,gCAAU"} \ No newline at end of file diff --git a/benchmark/index.js b/benchmark/index.js new file mode 100644 index 00000000..13ca4c02 --- /dev/null +++ b/benchmark/index.js @@ -0,0 +1,145 @@ +"use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (_) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +var _this = this; +Object.defineProperty(exports, "__esModule", { value: true }); +var Config_1 = require("../common/config/private/Config"); +var path = require("path"); +var ProjectPath_1 = require("../backend/ProjectPath"); +var Benchmarks_1 = require("./Benchmarks"); +var AutoCompleteItem_1 = require("../common/entities/AutoCompleteItem"); +var Utils_1 = require("../common/Utils"); +var DiskMangerWorker_1 = require("../backend/model/threading/DiskMangerWorker"); +var config = require(path.join(__dirname, 'config.json')); +Config_1.Config.Server.imagesFolder = config.path; +var dbPath = path.join(__dirname, 'test.db'); +ProjectPath_1.ProjectPath.reset(); +var RUNS = 50; +var resultsText = ''; +var printLine = function (text) { + resultsText += text + '\n'; +}; +var printHeader = function () { return __awaiter(_this, void 0, void 0, function () { + var dt, dir; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + dt = new Date(); + printLine('## PiGallery2 v' + require('./../package.json').version + + ', ' + Utils_1.Utils.zeroPrefix(dt.getDate(), 2) + + '.' + Utils_1.Utils.zeroPrefix(dt.getMonth() + 1, 2) + + '.' + dt.getFullYear()); + printLine('**System**: ' + config.system); + return [4 /*yield*/, DiskMangerWorker_1.DiskMangerWorker.scanDirectory('./')]; + case 1: + dir = _a.sent(); + printLine('**Gallery**: directories: ' + + dir.directories.length + + ' media: ' + dir.media.length + + // @ts-ignore + ', faces: ' + dir.media.reduce(function (p, c) { return p + (c.metadata.faces || []).length; }, 0)); + return [2 /*return*/]; + } + }); +}); }; +var printTableHeader = function () { + printLine('| action | action details | average time | details |'); + printLine('|:------:|:--------------:|:------------:|:-------:|'); +}; +var printResult = function (result, action, actionDetails) { + if (actionDetails === void 0) { actionDetails = ''; } + console.log('benchmarked: ' + action); + var details = '-'; + if (result.items) { + details = 'items: ' + result.items; + } + if (result.media) { + details = 'media: ' + result.media + ', directories:' + result.directories; + } + printLine('| ' + action + ' | ' + actionDetails + + ' | ' + (result.duration).toFixed(1) + 'ms | ' + details + ' |'); +}; +var run = function () { return __awaiter(_this, void 0, void 0, function () { + var start, bm, _a, _b, _c, _d, _e; + return __generator(this, function (_f) { + switch (_f.label) { + case 0: + start = Date.now(); + bm = new Benchmarks_1.Benchmarks(RUNS, dbPath); + // header + return [4 /*yield*/, printHeader()]; + case 1: + // header + _f.sent(); + printTableHeader(); + _a = printResult; + return [4 /*yield*/, bm.bmScanDirectory()]; + case 2: + _a.apply(void 0, [_f.sent(), 'Scanning directory']); + _b = printResult; + return [4 /*yield*/, bm.bmSaveDirectory()]; + case 3: + _b.apply(void 0, [_f.sent(), 'Saving directory']); + _c = printResult; + return [4 /*yield*/, bm.bmListDirectory()]; + case 4: + _c.apply(void 0, [_f.sent(), 'Listing Directory']); + return [4 /*yield*/, bm.bmAllSearch('a')]; + case 5: + (_f.sent()).forEach(function (res) { + if (res.searchType !== null) { + printResult(res.result, 'searching', '`a` as `' + AutoCompleteItem_1.SearchTypes[res.searchType] + '`'); + } + else { + printResult(res.result, 'searching', '`a` as `any`'); + } + }); + _d = printResult; + return [4 /*yield*/, bm.bmInstantSearch('a')]; + case 6: + _d.apply(void 0, [_f.sent(), 'instant search', '`a`']); + _e = printResult; + return [4 /*yield*/, bm.bmAutocomplete('a')]; + case 7: + _e.apply(void 0, [_f.sent(), 'auto complete', '`a`']); + console.log(resultsText); + console.log('run for : ' + ((Date.now() - start)).toFixed(1) + 'ms'); + return [2 /*return*/]; + } + }); +}); }; +run(); +//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/benchmark/index.js.map b/benchmark/index.js.map new file mode 100644 index 00000000..bd5bb96e --- /dev/null +++ b/benchmark/index.js.map @@ -0,0 +1 @@ +{"version":3,"file":"index.js","sourceRoot":"","sources":["index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,iBA6EA;;AA7EA,0DAAuD;AACvD,2BAA6B;AAC7B,sDAAmD;AACnD,2CAAyD;AACzD,wEAAgE;AAChE,yCAAsC;AACtC,gFAA6E;AAE7E,IAAM,MAAM,GAAqC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC,CAAC;AAC9F,eAAM,CAAC,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC;AACzC,IAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;AAC/C,yBAAW,CAAC,KAAK,EAAE,CAAC;AACpB,IAAM,IAAI,GAAG,EAAE,CAAC;AAEhB,IAAI,WAAW,GAAG,EAAE,CAAC;AACrB,IAAM,SAAS,GAAG,UAAC,IAAY;IAC7B,WAAW,IAAI,IAAI,GAAG,IAAI,CAAC;AAC7B,CAAC,CAAC;AAEF,IAAM,WAAW,GAAG;;;;;gBACZ,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC;gBACtB,SAAS,CAAC,iBAAiB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC,OAAO;oBAChE,IAAI,GAAG,aAAK,CAAC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;oBACxC,GAAG,GAAG,aAAK,CAAC,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;oBAC5C,GAAG,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;gBAC1B,SAAS,CAAC,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC9B,qBAAM,mCAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,EAAA;;gBAAhD,GAAG,GAAG,SAA0C;gBACtD,SAAS,CAAC,4BAA4B;oBACpC,GAAG,CAAC,WAAW,CAAC,MAAM;oBACtB,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM;oBAC7B,aAAa;oBACb,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,UAAC,CAAC,EAAE,CAAC,IAAK,OAAA,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,EAAnC,CAAmC,EAAE,CAAC,CAAC,CAAC,CAAC;;;;KACrF,CAAC;AAGF,IAAM,gBAAgB,GAAG;IACvB,SAAS,CAAC,sDAAsD,CAAC,CAAC;IAClE,SAAS,CAAC,sDAAsD,CAAC,CAAC;AACpE,CAAC,CAAC;AACF,IAAM,WAAW,GAAG,UAAC,MAAuB,EAAE,MAAc,EAAE,aAA0B;IAA1B,8BAAA,EAAA,kBAA0B;IACtF,OAAO,CAAC,GAAG,CAAC,eAAe,GAAG,MAAM,CAAC,CAAC;IACtC,IAAI,OAAO,GAAG,GAAG,CAAC;IAClB,IAAI,MAAM,CAAC,KAAK,EAAE;QAChB,OAAO,GAAG,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC;KACpC;IACD,IAAI,MAAM,CAAC,KAAK,EAAE;QAChB,OAAO,GAAG,SAAS,GAAG,MAAM,CAAC,KAAK,GAAG,gBAAgB,GAAG,MAAM,CAAC,WAAW,CAAC;KAC5E;IACD,SAAS,CAAC,IAAI,GAAG,MAAM,GAAG,KAAK,GAAG,aAAa;QAC7C,KAAK,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC;AACrE,CAAC,CAAC;AAEF,IAAM,GAAG,GAAG;;;;;gBACJ,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACnB,EAAE,GAAG,IAAI,uBAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gBAExC,SAAS;gBACT,qBAAM,WAAW,EAAE,EAAA;;gBADnB,SAAS;gBACT,SAAmB,CAAC;gBACpB,gBAAgB,EAAE,CAAC;gBACnB,KAAA,WAAW,CAAA;gBAAC,qBAAM,EAAE,CAAC,eAAe,EAAE,EAAA;;gBAAtC,kBAAY,SAA0B,EAAE,oBAAoB,EAAC,CAAC;gBAC9D,KAAA,WAAW,CAAA;gBAAC,qBAAM,EAAE,CAAC,eAAe,EAAE,EAAA;;gBAAtC,kBAAY,SAA0B,EAAE,kBAAkB,EAAC,CAAC;gBAC5D,KAAA,WAAW,CAAA;gBAAC,qBAAM,EAAE,CAAC,eAAe,EAAE,EAAA;;gBAAtC,kBAAY,SAA0B,EAAE,mBAAmB,EAAC,CAAC;gBAC5D,qBAAM,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,EAAA;;gBAA1B,CAAC,SAAyB,CAAC,CAAC,OAAO,CAAC,UAAA,GAAG;oBACrC,IAAI,GAAG,CAAC,UAAU,KAAK,IAAI,EAAE;wBAC3B,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,UAAU,GAAG,8BAAW,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;qBACtF;yBAAM;wBACL,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,cAAc,CAAC,CAAC;qBACtD;gBACH,CAAC,CAAC,CAAC;gBACH,KAAA,WAAW,CAAA;gBAAC,qBAAM,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,EAAA;;gBAAzC,kBAAY,SAA6B,EAAE,gBAAgB,EAAE,KAAK,EAAC,CAAC;gBACpE,KAAA,WAAW,CAAA;gBAAC,qBAAM,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,EAAA;;gBAAxC,kBAAY,SAA4B,EAAE,eAAe,EAAE,KAAK,EAAC,CAAC;gBAClE,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;gBACzB,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;;;;KACtE,CAAC;AAEF,GAAG,EAAE,CAAC"} \ No newline at end of file diff --git a/common/config/private/IPrivateConfig.ts b/common/config/private/IPrivateConfig.ts index 16b95e14..6cab0f43 100644 --- a/common/config/private/IPrivateConfig.ts +++ b/common/config/private/IPrivateConfig.ts @@ -55,6 +55,8 @@ export interface IndexingConfig { folderPreviewSize: number; cachedFolderTimeout: number; // Do not rescans the folder if seems ok reIndexingSensitivity: ReIndexingSensitivity; + excludeFolderList: string[] + excludeFileList: string[] } export interface ThreadingConfig { diff --git a/common/config/private/PrivateConfigClass.ts b/common/config/private/PrivateConfigClass.ts index fd150324..f39e2f11 100644 --- a/common/config/private/PrivateConfigClass.ts +++ b/common/config/private/PrivateConfigClass.ts @@ -59,7 +59,9 @@ export class PrivateConfigClass extends PublicConfigClass implements IPrivateCon indexing: { folderPreviewSize: 2, cachedFolderTimeout: 1000 * 60 * 60, - reIndexingSensitivity: ReIndexingSensitivity.low + reIndexingSensitivity: ReIndexingSensitivity.low, + excludeFolderList: [], + excludeFileList: [] }, duplicates: { listingLimit: 1000 diff --git a/docker/alpine/Dockerfile b/docker/alpine/Dockerfile new file mode 100644 index 00000000..dc6bebe8 --- /dev/null +++ b/docker/alpine/Dockerfile @@ -0,0 +1,25 @@ +FROM node:alpine +RUN apk add python build-base +COPY . /build +WORKDIR /build +RUN set -x && npm install --unsafe-perm && npm run build-release && \ + cd /build/release && npm install --unsafe-perm +RUN mkdir -p /build/release/data/config && \ + mkdir -p /build/release/data/db && \ + mkdir -p /build/release/data/image && \ + mkdir -p /build/release/data/TEMP && \ + cd /build/release && node backend/server.js && \ + sed -i 's/demo/data/g' config.json && sed -i 's@sqlite\.db@data/db/sqlite\.db@' config.json && \ + mv /build/release/config.json /build/release/data/config/config.json + +FROM node:alpine +WORKDIR /app +ENTRYPOINT ["npm", "start"] +EXPOSE 80 +ENV NODE_ENV=production +COPY --from=0 /build/release /app +RUN ln -s /app/data/config/config.json config.json +VOLUME ["/app/data/config", "/app/data/db", "/app/data/images", "/app/data/TEMP"] +HEALTHCHECK --interval=15s --timeout=5s --retries=4 --start-period=30s \ + CMD wget --quiet --tries=1 --no-check-certificate --spider \ + http://localhost:80 || exit 1 diff --git a/frontend/app/ui/settings/indexing/indexing.settings.component.html b/frontend/app/ui/settings/indexing/indexing.settings.component.html index 025fa112..27cb7981 100644 --- a/frontend/app/ui/settings/indexing/indexing.settings.component.html +++ b/frontend/app/ui/settings/indexing/indexing.settings.component.html @@ -49,6 +49,36 @@ +
+ +
+ + + Folders to exclude from indexing
+ ';' separated strings. If an entry starts with '/' it is treated as an absolute path. If it doesn't start with '/' but contains a '/', the path is relative to the image directory. If it doesn't contain a '/', any folder with this name will be excluded. + +
+
+ +
+ +
+ + + FFiles that mark a folder to be excluded from indexing
+ ';' separated strings. Any folder that contains a file with this name will be excluded from indexing. + +
+
+