From 88bd7d67dcd62dd2b1df1155b8ff269ee80c797d Mon Sep 17 00:00:00 2001 From: Mickael Kerjean Date: Tue, 11 Oct 2022 01:45:34 +1100 Subject: [PATCH] feature (tags): revamp data model and storage --- client/helpers/cache.js | 63 ++++++++++------- client/helpers/index.js | 6 +- client/helpers/path.js | 5 ++ client/index.js | 4 +- client/model/files.js | 37 +++++----- client/model/tags.js | 134 +++++++++++++++++++------------------ client/pages/tagspage.js | 1 - client/pages/tagspage.scss | 2 +- 8 files changed, 141 insertions(+), 111 deletions(-) diff --git a/client/helpers/cache.js b/client/helpers/cache.js index 5b85a44e..6d699093 100644 --- a/client/helpers/cache.js +++ b/client/helpers/cache.js @@ -1,19 +1,22 @@ "use strict"; -const DB_VERSION = 3; +const DB_VERSION = 4; const FILE_PATH = "file_path"; const FILE_CONTENT = "file_content"; +const FILE_TAG = "file_tag"; function DataFromIndexedDB() { this.db = null; this.FILE_PATH = FILE_PATH; this.FILE_CONTENT = FILE_CONTENT; + this.FILE_TAG = FILE_TAG; return this._init(); } function DataFromMemory() { this.data = {}; this.FILE_PATH = FILE_PATH; this.FILE_CONTENT = FILE_CONTENT; + this.FILE_TAG = FILE_TAG; return this._init(); } @@ -32,13 +35,21 @@ DataFromIndexedDB.prototype._init = function() { // we've change the primary key to be a (path,share) db.deleteObjectStore(FILE_PATH); db.deleteObjectStore(FILE_CONTENT); + } else if (event.oldVersion == 3) { + // we've added a FILE_TAG to store tag related data and update + // keyPath to have "backend" + db.deleteObjectStore(FILE_PATH); + db.deleteObjectStore(FILE_CONTENT); } - store = db.createObjectStore(FILE_PATH, { keyPath: ["share", "path"] }); - store.createIndex("idx_path", ["share", "path"], { unique: true }); + store = db.createObjectStore(FILE_PATH, { keyPath: ["backend", "share", "path"] }); + store.createIndex("idx_path", ["backend", "share", "path"], { unique: true }); - store = db.createObjectStore(FILE_CONTENT, { keyPath: ["share", "path"] }); - store.createIndex("idx_path", ["share", "path"], { unique: true }); + store = db.createObjectStore(FILE_CONTENT, { keyPath: ["backend", "share", "path"] }); + store.createIndex("idx_path", ["backend", "share", "path"], { unique: true }); + + store = db.createObjectStore(FILE_TAG, { keyPath: ["backend", "share"] }); + store.createIndex("idx_path", ["backend", "share"], { unique: true }); }; this.db = new Promise((done, err) => { @@ -56,7 +67,7 @@ DataFromMemory.prototype._init = function() { * Fetch a record using its path, can be either a file path or content */ DataFromIndexedDB.prototype.get = function(type, key) { - if (type !== FILE_PATH && type !== FILE_CONTENT) return Promise.reject(); + if (type !== FILE_PATH && type !== FILE_CONTENT && type !== FILE_TAG) return Promise.reject(); return this.db.then((db) => { const tx = db.transaction(type, "readonly"); @@ -71,7 +82,7 @@ DataFromIndexedDB.prototype.get = function(type, key) { }); }; DataFromMemory.prototype.get = function(type, key) { - if (type !== FILE_PATH && type !== FILE_CONTENT) return Promise.reject(); + if (type !== FILE_PATH && type !== FILE_CONTENT && type !== FILE_TAG) return Promise.reject(); const data = this.data[type] || null; if (data === null) { @@ -87,7 +98,7 @@ DataFromMemory.prototype.get = function(type, key) { }; DataFromIndexedDB.prototype.update = function(type, key, fn, exact = true) { - if (type !== FILE_PATH && type !== FILE_CONTENT) return Promise.reject(); + if (type !== FILE_PATH && type !== FILE_CONTENT && type !== FILE_TAG) return Promise.reject(); return this.db.then((db) => { const tx = db.transaction(type, "readwrite"); @@ -113,7 +124,7 @@ DataFromIndexedDB.prototype.update = function(type, key, fn, exact = true) { }; DataFromMemory.prototype.update = function(type, key, fn, exact = true) { - if (type !== FILE_PATH && type !== FILE_CONTENT) return Promise.reject(); + if (type !== FILE_PATH && type !== FILE_CONTENT && type !== FILE_TAG) return Promise.reject(); const data = this.data[type]; if (data === undefined) { @@ -134,7 +145,7 @@ DataFromMemory.prototype.update = function(type, key, fn, exact = true) { }; DataFromIndexedDB.prototype.upsert = function(type, key, fn) { - if (type !== FILE_PATH && type !== FILE_CONTENT) return Promise.reject(); + if (type !== FILE_PATH && type !== FILE_CONTENT && type !== FILE_TAG) return Promise.reject(); return this.db.then((db) => { const tx = db.transaction(type, "readwrite"); @@ -154,7 +165,7 @@ DataFromIndexedDB.prototype.upsert = function(type, key, fn) { }); }; DataFromMemory.prototype.upsert = function(type, key, fn) { - if (type !== FILE_PATH && type !== FILE_CONTENT) return Promise.reject(); + if (type !== FILE_PATH && type !== FILE_CONTENT && type !== FILE_TAG) return Promise.reject(); const db = this.data[type] || null; if (db === null) { @@ -167,7 +178,7 @@ DataFromMemory.prototype.upsert = function(type, key, fn) { }; DataFromIndexedDB.prototype.add = function(type, key, data) { - if (type !== FILE_PATH && type !== FILE_CONTENT) return Promise.reject(); + if (type !== FILE_PATH && type !== FILE_CONTENT && type !== FILE_TAG) return Promise.reject(); return this.db.then((db) => { return new Promise((done, error) => { @@ -180,7 +191,7 @@ DataFromIndexedDB.prototype.add = function(type, key, data) { }).catch(() => Promise.resolve()); }; DataFromMemory.prototype.add = function(type, key, data) { - if (type !== FILE_PATH && type !== FILE_CONTENT) return Promise.reject(); + if (type !== FILE_PATH && type !== FILE_CONTENT && type !== FILE_TAG) return Promise.reject(); if (this.data[type] === undefined) { this.data[type] = {}; @@ -190,7 +201,7 @@ DataFromMemory.prototype.add = function(type, key, data) { }; DataFromIndexedDB.prototype.remove = function(type, key, exact = true) { - if (type !== FILE_PATH && type !== FILE_CONTENT) return Promise.reject(); + if (type !== FILE_PATH && type !== FILE_CONTENT && type !== FILE_TAG) return Promise.reject(); return this.db.then((db) => { const tx = db.transaction(type, "readwrite"); @@ -223,7 +234,7 @@ DataFromIndexedDB.prototype.remove = function(type, key, exact = true) { }).catch(() => Promise.resolve()); }; DataFromMemory.prototype.remove = function(type, key, exact = true) { - if (type !== FILE_PATH && type !== FILE_CONTENT) return Promise.reject(); + if (type !== FILE_PATH && type !== FILE_CONTENT && type !== FILE_TAG) return Promise.reject(); const data = this.data[type] || null; if (data === null) { @@ -242,7 +253,7 @@ DataFromMemory.prototype.remove = function(type, key, exact = true) { }; DataFromIndexedDB.prototype.fetchAll = function(fn, type = FILE_PATH, key) { - if (type !== FILE_PATH && type !== FILE_CONTENT) return Promise.reject(); + if (type !== FILE_PATH && type !== FILE_CONTENT && type !== FILE_TAG) return Promise.reject(); return this.db.then((db) => { const tx = db.transaction([type], "readonly"); @@ -272,7 +283,7 @@ DataFromIndexedDB.prototype.fetchAll = function(fn, type = FILE_PATH, key) { }).catch(() => Promise.resolve()); }; DataFromMemory.prototype.fetchAll = function(fn, type = FILE_PATH, key) { - if (type !== FILE_PATH && type !== FILE_CONTENT) return Promise.reject(); + if (type !== FILE_PATH && type !== FILE_CONTENT && type !== FILE_TAG) return Promise.reject(); const data = this.data[type] || null; if (data === null) { @@ -295,6 +306,8 @@ DataFromIndexedDB.prototype.destroy = function() { this.db.then((db) => { purgeAll(db, FILE_PATH); purgeAll(db, FILE_CONTENT); + // We keep FILE_TAG as this was user generated and potentially frustrating + // for users if they were to lose this }); done(); @@ -311,11 +324,15 @@ DataFromMemory.prototype.destroy = function() { return Promise.resolve(); }; -export let cache = new DataFromMemory(); -if ("indexedDB" in window && window.indexedDB !== null) { - const request = indexedDB.open("_indexedDB", 1); - request.onsuccess = (e) => { +export let cache = null; + +export function setup_cache() { + cache = new DataFromMemory(); + if ("indexedDB" in window && window.indexedDB !== null) { cache = new DataFromIndexedDB(); - indexedDB.deleteDatabase("_indexedDB"); - }; + return new Promise((done) => { + cache.db.then(() => done()); + }); + } + return Promise.resolve(); } diff --git a/client/helpers/index.js b/client/helpers/index.js index 318e204d..3325ea64 100644 --- a/client/helpers/index.js +++ b/client/helpers/index.js @@ -5,10 +5,10 @@ export { export { opener } from "./mimetype"; export { debounce, throttle } from "./backpressure"; export { event } from "./events"; -export { cache } from "./cache"; +export { cache, setup_cache } from "./cache"; export { - pathBuilder, basename, dirname, absoluteToRelative, filetype, currentShare, - findParams, appendShareToUrl, + pathBuilder, basename, dirname, absoluteToRelative, filetype, + currentShare, currentBackend, findParams, appendShareToUrl, } from "./path"; export { memory } from "./memory"; export { prepare } from "./navigate"; diff --git a/client/helpers/path.js b/client/helpers/path.js index fabd8125..ebaf5bfe 100644 --- a/client/helpers/path.js +++ b/client/helpers/path.js @@ -40,6 +40,11 @@ export function currentShare() { return findParams("share"); } +export function currentBackend() { + return ""; +} + + export function findParams(p) { return new window.URL(location.href).searchParams.get(p) || ""; } diff --git a/client/index.js b/client/index.js index ec7e72ac..6ec7ad65 100644 --- a/client/index.js +++ b/client/index.js @@ -3,7 +3,7 @@ import ReactDOM from "react-dom"; import Router from "./router"; import { Config, Log } from "./model/"; -import { http_get } from "./helpers/ajax"; +import { http_get, setup_cache } from "./helpers/"; import load from "little-loader"; import "./assets/css/reset.scss"; @@ -40,7 +40,7 @@ window.addEventListener("DOMContentLoaded", () => { return Promise.resolve(); } - Promise.all([Config.refresh(), setup_xdg_open(), translation()]).then(() => { + Promise.all([Config.refresh(), setup_xdg_open(), translation(), setup_cache()]).then(() => { const timeSinceBoot = new Date() - window.initTime; if (window.CONFIG.name) document.title = window.CONFIG.name; if (timeSinceBoot >= 1500) { diff --git a/client/model/files.js b/client/model/files.js index 043bc943..8224b2b6 100644 --- a/client/model/files.js +++ b/client/model/files.js @@ -2,7 +2,7 @@ import { http_get, http_post, http_options, prepare, basename, dirname, pathBuilder, - currentShare, appendShareToUrl, + currentShare, currentBackend, appendShareToUrl, } from "../helpers/"; import { Observable } from "rxjs/Observable"; @@ -46,8 +46,9 @@ class FileSystem { return http_get(url).then((response) => { response = fileMiddleware(response, path, show_hidden); - return cache.upsert(cache.FILE_PATH, [currentShare(), path], (_files) => { + return cache.upsert(cache.FILE_PATH, [currentBackend(), currentShare(), path], (_files) => { const store = Object.assign({ + backend: currentBackend(), share: currentShare(), status: "ok", path: path, @@ -97,7 +98,7 @@ class FileSystem { } _ls_from_cache(path, _record_access = false) { - return cache.get(cache.FILE_PATH, [currentShare(), path]).then((response) => { + return cache.get(cache.FILE_PATH, [currentBackend(), currentShare(), path]).then((response) => { if (!response || !response.results) return null; if (this.current_path === path) { this.obs && this.obs.next({ @@ -110,7 +111,7 @@ class FileSystem { }).then((e) => { requestAnimationFrame(() => { if (_record_access === true) { - cache.upsert(cache.FILE_PATH, [currentShare(), path], (response) => { + cache.upsert(cache.FILE_PATH, [currentBackend(), currentShare(), path], (response) => { if (!response || !response.results) return null; if (this.current_path === path) { this.obs && this.obs.next({ @@ -136,9 +137,9 @@ class FileSystem { this._ls_from_cache(dirname(path)) : Promise.resolve(res)) .then(() => http_post(url)) .then((res) => { - return cache.remove(cache.FILE_CONTENT, [currentShare(), path]) - .then(cache.remove(cache.FILE_CONTENT, [currentShare(), path], false)) - .then(cache.remove(cache.FILE_PATH, [currentShare(), dirname(path)], false)) + return cache.remove(cache.FILE_CONTENT, [currentBackend(), currentShare(), path]) + .then(cache.remove(cache.FILE_CONTENT, [currentBackend(), currentShare(), path], false)) + .then(cache.remove(cache.FILE_PATH, [currentBackend(), currentShare(), dirname(path)], false)) .then(this._remove(path, "loading")) .then((res) => this.current_path === dirname(path) ? this._ls_from_cache(dirname(path)) : Promise.resolve(res)); @@ -158,8 +159,9 @@ class FileSystem { if (this.is_binary(res) === true) { return Promise.reject({ code: "BINARY_FILE" }); } - return cache.upsert(cache.FILE_CONTENT, [currentShare(), path], (response) => { + return cache.upsert(cache.FILE_CONTENT, [currentBackend(), currentShare(), path], (response) => { const file = response ? response : { + backend: currentBackend(), share: currentShare(), path: path, last_update: null, @@ -245,8 +247,9 @@ class FileSystem { return this._replace(destination_path, null, "loading") .then(() => origin_path !== destination_path ? this._remove(origin_path, "loading") : Promise.resolve()) - .then(() => cache.add(cache.FILE_PATH, [currentShare(), destination_path], { + .then(() => cache.add(cache.FILE_PATH, [currentBackend(), currentShare(), destination_path], { path: destination_path, + backend: currentBackend(), share: currentShare(), results: [], access_count: 0, @@ -353,11 +356,11 @@ class FileSystem { .then(() => this._replace(destination_path, null, "loading")) .then(() => this._refresh(origin_path, destination_path)) .then(() => { - cache.update(cache.FILE_PATH, [currentShare(), origin_path], (data) => { + cache.update(cache.FILE_PATH, [currentBackend(), currentShare(), origin_path], (data) => { data.path = data.path.replace(origin_path, destination_path); return data; }, false); - cache.update(cache.FILE_CONTENT, [currentShare(), origin_path], (data) => { + cache.update(cache.FILE_CONTENT, [currentBackend(), currentShare(), origin_path], (data) => { data.path = data.path.replace(origin_path, destination_path); return data; }, false); @@ -389,7 +392,7 @@ class FileSystem { if (value.access_count >= 1 && value.path !== "/") { data.push(value); } - }, cache.FILE_PATH, [currentShare(), "/"]).then(() => { + }, cache.FILE_PATH, [currentBackend(), currentShare(), "/"]).then(() => { return Promise.resolve( data .sort((a, b) => a.access_count > b.access_count? -1 : 1) @@ -410,9 +413,10 @@ class FileSystem { }); function update_cache(result) { - return cache.upsert(cache.FILE_CONTENT, [currentShare(), path], (response) => { + return cache.upsert(cache.FILE_CONTENT, [currentBackend(), currentShare(), path], (response) => { if (!response) { response = { + backend: currentBackend(), share: currentShare(), path: path, last_access: null, @@ -437,7 +441,7 @@ class FileSystem { } _replace(path, icon, icon_previous) { - return cache.update(cache.FILE_PATH, [currentShare(), dirname(path)], function(res) { + return cache.update(cache.FILE_PATH, [currentBackend(), currentShare(), dirname(path)], function(res) { res.results = res.results.map((file) => { if (file.name === basename(path) && file.icon == icon_previous) { if (!icon) { @@ -453,10 +457,11 @@ class FileSystem { }); } _add(path, icon) { - return cache.upsert(cache.FILE_PATH, [currentShare(), dirname(path)], (res) => { + return cache.upsert(cache.FILE_PATH, [currentBackend(), currentShare(), dirname(path)], (res) => { if (!res || !res.results) { res = { path: path, + backend: currentBackend(), share: currentShare(), results: [], access_count: 0, @@ -475,7 +480,7 @@ class FileSystem { }); } _remove(path, previous_icon) { - return cache.update(cache.FILE_PATH, [currentShare(), dirname(path)], function(res) { + return cache.update(cache.FILE_PATH, [currentBackend(), currentShare(), dirname(path)], function(res) { if (!res) return null; res.results = res.results.filter((file) => { return file.name === basename(path) && file.icon == previous_icon ? false : true; diff --git a/client/model/tags.js b/client/model/tags.js index db73afb5..fe1316b3 100644 --- a/client/model/tags.js +++ b/client/model/tags.js @@ -1,62 +1,52 @@ -let DB = { - tags: { - "Bookmark": ["/home/user/Documents/", "/home/user/Documents/projects/"], - "Customer": ["/home/user/Documents/projects/customers/"], - "wiki": ["/home/user/Documents/test.txt"], - "mit": ["/home/user/Documents/projects/customers/mit/"], - "dhl": ["/home/user/Documents/projects/customers/dhl/"], - "powerstone": ["/home/user/Documents/projects/customers/powerstone/"], - "accounting": [ - "/home/user/Documents/projects/customers/mit/accounting/", - "/home/user/Documents/projects/customers/dhl/accounting/", - "/home/user/Documents/projects/customers/powerstone/accounting/", - ] - }, - weight: { // for sorting - "Bookmark": 2, - }, - share: null, - backend: "__hash__", -}; +import { cache, currentShare, currentBackend } from "../helpers/"; class TagManager { all(tagPath = "/", maxSize = -1) { - return Promise.resolve([]); // TODO: Remove this when ready + // return Promise.resolve([]); // TODO: Remove this when ready - if (tagPath == "/") { - const scoreFn = (acc, el) => (acc + el.replace(/[^\/]/g, "").length); - const tags = Object.keys(DB.tags).sort((a, b) => { - if (DB.tags[a].length === DB.tags[b].length) { - return DB.tags[a].reduce(scoreFn, 0) - DB.tags[b].reduce(scoreFn, 0); - } - return DB.tags[a].length < DB.tags[b].length ? 1 : -1; - }); - if(tags.length === 0) { - return Promise.resolve(["Bookmark"]); - } else if(tags.length >= 5) { - return Promise.resolve(["All"].concat(tags.slice(0, 5))); + return cache.get(cache.FILE_TAG, [currentBackend(), currentShare()]).then((DB) => { + if (DB === null) { + return []; } - return Promise.resolve(tags); - } - return Promise.resolve([ - // "Bookmark", "wiki", "B", "C", "D", "E", "F" - ]); + + if (tagPath == "/") { + const scoreFn = (acc, el) => (acc + el.replace(/[^\/]/g, "").length); + const tags = Object.keys(DB.tags).sort((a, b) => { + if (DB.tags[a].length === DB.tags[b].length) { + return DB.tags[a].reduce(scoreFn, 0) - DB.tags[b].reduce(scoreFn, 0); + } + return DB.tags[a].length < DB.tags[b].length ? 1 : -1; + }); + if(tags.length === 0) { + return ["Bookmark"]; + } else if(tags.length >= 5) { + return ["All"].concat(tags.slice(0, 5)); + } + return tags; + } + return [ + "Bookmark", "wiki", "B", "C", "D", "E", "F" + ]; + }); } files(tagPath) { const tags = this._tagPathStringToArray(tagPath, false); if (tags.length === 0) return Promise.resolve([]); - else if(tags.length > 1) return Promise.resolve([]); // TODO + else if (tags.length > 1) return Promise.resolve([]); // TODO - switch(tags[0]) { - case "All": - return this.all() - .then((tags) => (tags.reduce((acc, el) => { - return DB.tags[el] ? acc.concat(DB.tags[el]) : acc; - }, []))); - default: - return Promise.resolve(DB.tags[tags[0]] || []); - } + return cache.get(cache.FILE_TAG, [currentBackend(), currentShare()]).then((DB) => { + if(!DB) return []; + switch(tags[0]) { + case "All": + return this.all() + .then((tags) => (tags.reduce((acc, el) => { + return DB.tags[el] ? acc.concat(DB.tags[el]) : acc; + }, []))); + default: + return Promise.resolve(DB.tags[tags[0]] || []); + } + }); } _tagPathStringToArray(tagPathString, removeFirst = true) { @@ -66,31 +56,45 @@ class TagManager { } addTagToFile(tag, path) { - if(Object.keys(DB.tags).indexOf(tag) === -1) { - DB.tags[tag] = []; - } - if(!DB.tags[tag].indexOf(path) === -1) { - DB.tags[tag].push(path); - } + return cache.upsert(cache.FILE_TAG, [currentBackend(), currentShare()], (DB) => { + if(Object.keys(DB.tags).indexOf(tag) === -1) { + DB.tags[tag] = []; + } + if(!DB.tags[tag].indexOf(path) === -1) { + DB.tags[tag].push(path); + } + return DB; + }); } removeTagFromFile(tag, path) { - if(!DB.tags[tag]) return; - const idx = DB.tags[tag].indexOf(path); - DB.tags[tag].splice(idx, 1); + return cache.upsert(cache.FILE_TAG, [currentBackend(), currentShare()], (DB) => { + if(!DB.tags[tag]) return; + const idx = DB.tags[tag].indexOf(path); + DB.tags[tag].splice(idx, 1); + if (DB.tags[tag].length === 0) { + delete DB.tags[tag]; + delete DB.weight[tag]; + } + return DB; + }); } - import(_DB) { - DB = _DB; - return new Promise((done) => { - setTimeout(() => { - done(); - }, 5000); - }) + import(DB) { + return cache.upsert(cache.FILE_TAG, [currentBackend(), currentShare()], () => { + return DB; + }); } export() { - return Promise.resolve(DB); + const key = [currentBackend(), currentShare()]; + return cache.get(cache.FILE_TAG, key) + .then((a) => { + if (a === null) { + return {tags: {}, weight: {}, share: key[1], backend: key[0]} + } + return a; + }); } } diff --git a/client/pages/tagspage.js b/client/pages/tagspage.js index 845ec70b..bc4617de 100644 --- a/client/pages/tagspage.js +++ b/client/pages/tagspage.js @@ -81,7 +81,6 @@ export function TagsPageComponent({ match }) { setRefresh(refresh + 1); }).catch((err) => { setLoading(false); - notify.send(err, "error") }); }; reader.readAsText($input.files[0]); diff --git a/client/pages/tagspage.scss b/client/pages/tagspage.scss index 6b08f069..78d717c8 100644 --- a/client/pages/tagspage.scss +++ b/client/pages/tagspage.scss @@ -14,7 +14,7 @@ overflow-x: hidden!important; -webkit-overflow-scrolling: touch; .component_submenu { - margin: 10px; + margin: 25px 0 0 0; h1 { font-weight: 100; text-transform: uppercase;