Backup_Repos/filestash
1
0
Fork 0
mirror of https://github.com/mickael-kerjean/filestash synced 2026-01-03 22:33:08 +01:00
Code Issues Projects Releases Wiki Activity

chore (rewrite): cache handling in fs

Browse source
This commit is contained in:
MickaelK 2024-05-28 00:39:14 +10:00
parent 422856343c
commit 76b44f3c18
12 changed files with 256 additions and 155 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
public/assets/components/decorator_shell_filemanager.css
View file

@ -13,6 +13,7 @@
font-size: 0.95rem;
opacity: 0.8;
padding-left: 10px;
text-transform: capitalize;
}
.component_filemanager_shell .component_sidebar > div {
min-height: 50px;

48
public/assets/pages/filespage/cache.js
View file

@ -112,7 +112,7 @@ class IndexDBCache extends ICache {
if (exact !== true) {
let request = store.openCursor(IDBKeyRange.bound(
[key[0], key[1], key[2]],
[key[0], key[1], key[2]+'\uFFFF'.repeat(5000)],
[key[0], key[1], key[2]+"\u{FFFF}".repeat(5000)],
true, true,
));
await new Promise((done, err) => {
@ -177,22 +177,42 @@ export function clearCache(path) {
}
export async function init() {
cache = new InMemoryCache();
if (!("indexedDB" in window)) return initCacheState();
const setup_cache = () => {
cache = new InMemoryCache();
if (!("indexedDB" in window)) return initCacheState();
cache = new IndexDBCache();
return Promise.all([cache.db, initCacheState()]).catch((err) => {
if (err === "INDEXEDDB_NOT_SUPPORTED") {
// Firefox in private mode act like if it supports indexedDB but
// is throwing that string as an error if you try to use it ...
// so we fallback with our basic ram cache
cache = new DataFromMemory();
return initCacheState();
}
throw err;
});
cache = new IndexDBCache();
return Promise.all([cache.db, initCacheState()]).catch((err) => {
if (err === "INDEXEDDB_NOT_SUPPORTED") {
// Firefox in private mode act like if it supports indexedDB but
// is throwing that string as an error if you try to use it ...
// so we fallback with our basic ram cache
cache = new DataFromMemory();
return initCacheState();
}
throw err;
});
}
const setup_session = () => {
return Promise.resolve();
};
return Promise.all([setup_cache(), setup_session()]);
}
export default function() {
return cache;
};
let backendID = "na";
export function currentBackend() {
return backendID;
}
export function currentShare() {
return new window.URL(location.href).searchParams.get("share") || "";
}
function initCacheState() {
return Promise.resolve();
}

13
public/assets/pages/filespage/cache_state.js
View file

@ -1,13 +0,0 @@
let backendID = "na";
export function currentBackend() {
return backendID;
}
export function currentShare() {
return new window.URL(location.href).searchParams.get("share") || "";
}
export async function init() {
// TODO: init session with backendID;
}

120
public/assets/pages/filespage/state_filemutate.js → public/assets/pages/filespage/cache_transcient.js
View file

@ -1,6 +1,16 @@
import { onDestroy } from "../../lib/skeleton/index.js";
import rxjs from "../../lib/rx.js";
import fscache from "./cache.js";
import { extractPath } from "./helper.js";
/*
* The transcient cache is used to rerender the list of files in a particular location. That's used
* when the user is doing either of a touch, mkdir, rm or mv. It is split onto 2 parts:
* - the virtualFiles$: which are things we want to display in addition to what is currently
* visible on the screen
* - the mutationFiles$: which are things already on the screen which we need to mutate. For
* example when we want a particular file to show a loading spinner
*/
const virtualFiles$ = new rxjs.BehaviorSubject({
// "/tmp/": [],
@ -10,7 +20,7 @@ const mutationFiles$ = new rxjs.BehaviorSubject({
// "/home/": [{ name: "test", fn: (file) => file, ...]
});
export function touch(path) {
export function touch(ajax$, path) {
const [basepath, filename] = extractPath(path);
const file = {
name: filename,
@ -25,10 +35,10 @@ export function touch(path) {
const onSuccess = async () => {
removeLoading(virtualFiles$, basepath, filename);
onDestroy(() => statePop(virtualFiles$, basepath, filename));
await fscache().update(basepath, ({ files, ...rest }) => {
if (Array.isArray(files) === false) return rest;
return { files: files.concat([file]), ...rest };
});
await fscache().update(basepath, ({ files, ...rest }) => ({
files: files.concat([file]),
...rest,
}));
};
const onFailure = (err, caught) => {
statePop(virtualFiles$, basepath, filename);
@ -36,14 +46,13 @@ export function touch(path) {
rxjs.mergeMap(() => rxjs.EMPTY),
);
};
return rxjs.of(null).pipe(
rxjs.delay(2000),
return ajax$.pipe(
rxjs.mergeMap(onSuccess),
rxjs.catchError(onFailure),
);
}
export function mkdir(path) {
export function mkdir(ajax$, path) {
const [basepath, dirname] = extractPath(path);
const file = {
name: dirname,
@ -58,15 +67,10 @@ export function mkdir(path) {
const onSuccess = async () => {
removeLoading(virtualFiles$, basepath, dirname);
onDestroy(() => statePop(virtualFiles$, basepath, dirname));
await fscache().update(basepath, ({ files, ...rest }) => {
if (Array.isArray(files) === false) return rest;
return { files: files.concat([file]), ...rest };
});
// TODO: remove cache if path not found
// await fscache().store(path, {
// files: [],
// permissions: {},
// });
await fscache().update(basepath, ({ files, ...rest }) => ({
files: files.concat([file]),
...rest,
}));
};
const onFailure = () => {
statePop(virtualFiles$, basepath, dirname);
@ -74,41 +78,44 @@ export function mkdir(path) {
rxjs.mergeMap(() => rxjs.EMPTY),
);
};
return rxjs.of(null).pipe(
rxjs.delay(2000),
return ajax$.pipe(
rxjs.mergeMap(onSuccess),
rxjs.catchError(onFailure),
);
}
export function save(path, size) {
export function save(ajax$, path, size) {
const [basepath, filename] = extractPath(path);
stateAdd(virtualFiles$, basepath, {
const file = {
name: dirname,
type: "file",
size,
time: new Date().getTime(),
};
stateAdd(virtualFiles$, basepath, {
...file,
loading: true,
});
const onSuccess = () => {
const onSuccess = async () => {
removeLoading(virtualFiles$, basepath, filename);
onDestroy(() => statePop(virtualFiles$, basepath, filename));
// TODO: update cache
await fscache().update(basepath, ({ files, ...rest }) => ({
files: files.concat([file]),
...rest,
}));
};
const onFailure = () => {
statePop(virtualFiles$, basepath, dirname);
return rxjs.EMPTY;
};
return rxjs.of(null).pipe(
rxjs.delay(2000),
return ajax$.pipe(
rxjs.tap(onSuccess),
rxjs.catchError(onFailure),
);
}
export function rm(...paths) {
export function rm(ajax$, ...paths) {
if (paths.length === 0) return rxjs.of(null);
const arr = new Array(paths.length * 2);
let basepath = null;
for (let i=0;i<paths.length;i++) {
@ -116,7 +123,6 @@ export function rm(...paths) {
if (i === 0) basepath = arr[2*i];
else if (basepath !== arr[2*i]) throw new Error("NOT_IMPLEMENTED");
}
stateAdd(mutationFiles$, basepath, {
name: basepath,
fn: (file) => {
@ -129,7 +135,6 @@ export function rm(...paths) {
return file;
},
});
const onSuccess = async () => {
stateAdd(mutationFiles$, basepath, {
name: basepath,
@ -142,31 +147,42 @@ export function rm(...paths) {
});
onDestroy(() => statePop(mutationFiles$, basepath, basepath));
await Promise.all(paths.map((path) => fscache().remove(path, false)));
await fscache().update(basepath, ({ files, ...rest }) => ({
files: files.filter(({ name }) => {
for (let i=0;i<arr.length;i+=2) {
if (name === arr[i+1]) {
return false;
}
}
return true;
}),
...rest,
}));
};
const onFailure = () => {
stateAdd(mutationFiles$, basepath, {
name: basepath,
fn: (file) => {
for (let i=0;i<arr.length;i+=2) {
if (file.name === arr[i+1]) delete file.loading;
if (file.name === arr[i+1]) {
delete file.loading;
delete file.last;
}
}
return file;
},
});
return rxjs.EMPTY;
};
return rxjs.of(null).pipe(
rxjs.delay(1000),
return ajax$.pipe(
rxjs.tap(onSuccess),
rxjs.catchError(onFailure),
);
}
export function mv(fromPath, toPath) {
export function mv(ajax$, fromPath, toPath) {
const [fromBasepath, fromName] = extractPath(fromPath);
const [toBasepath, toName] = extractPath(toPath);
let type = null;
if (fromBasepath === toBasepath) {
stateAdd(mutationFiles$, fromBasepath, {
@ -197,7 +213,8 @@ export function mv(fromPath, toPath) {
type,
});
}
const onSuccess = () => {
const onSuccess = async () => {
console.log(fromPath, toPath)
fscache().remove(fromPath, false);
if (fromBasepath === toBasepath) {
stateAdd(mutationFiles$, fromBasepath, {
@ -207,6 +224,17 @@ export function mv(fromPath, toPath) {
return file;
},
});
await fscache().update(fromBasepath, ({ files, ...rest }) => {
return {
files: files.map((file) => {
if (file.name === fromName) {
file.name = toName;
}
return file;
}),
...rest,
};
});
} else {
stateAdd(mutationFiles$, fromBasepath, {
name: fromName,
@ -219,21 +247,19 @@ export function mv(fromPath, toPath) {
}
};
const onFailure = () => {
if (fromBasepath === toBasepath) {
// TODO
} else {
// TODO
statePop(mutationFiles$, fromBasepath, fromName);
if (fromBasepath !== toBasepath) {
statePop(virtualFiles$, toBasepath, toName);
}
return rxjs.EMPTY;
};
return rxjs.of(null).pipe(
rxjs.delay(1000),
return ajax$.pipe(
rxjs.tap(onSuccess),
rxjs.catchError(onFailure),
);
}
export function middlewareLs(path) {
export function ls(path) {
return rxjs.pipe(
// case1: virtual files = additional files we want to see displayed in the UI
rxjs.switchMap(({ files, ...res }) => virtualFiles$.pipe(rxjs.mergeMap((virtualFiles) => {
@ -260,17 +286,9 @@ export function middlewareLs(path) {
}
return rxjs.of({ ...res, files });
}))),
// rxjs.tap(({ files }) => console.log(files)),
);
}
export function extractPath(path) {
path = path.replace(new RegExp("/$"), "");
const p = path.split("/");
const filename = p.pop();
return [p.join("/") + "/", filename];
}
function stateAdd(behavior, path, obj) {
let arr = behavior.value[path];
if (!arr) arr = [];

29
public/assets/pages/filespage/ctrl_filesystem.js
View file

File diff suppressed because one or more lines are too long

2
public/assets/pages/filespage/ctrl_newitem.js
View file

@ -5,8 +5,8 @@ import { animate } from "../../lib/animate.js";
import { loadCSS } from "../../helpers/loader.js";
import { getAction$, setAction } from "./state_newthing.js";
import { mkdir, touch } from "./state_filemutate.js";
import { currentPath } from "./helper.js";
import { mkdir, touch } from "./model_files.js";
export default async function(render) {
const $node = createElement(`

1
public/assets/pages/filespage/ctrl_submenu.css
View file

@ -53,6 +53,7 @@
}
[is="component_submenu"] .component_submenu .action.right {
border-radius: 5px;
white-space: nowrap;
}
[is="component_submenu"] .component_submenu .action.right button[data-bind="clear"] {
display: flex;

29
public/assets/pages/filespage/ctrl_submenu.js
View file

@ -19,14 +19,14 @@ import componentDelete from "./modal_delete.js";
import { getSelection$, clearSelection, lengthSelection, expandSelection } from "./state_selection.js";
import { getAction$, setAction } from "./state_newthing.js";
import { setState, getState$ } from "./state_config.js";
import { rm, mv, extractPath } from "./state_filemutate.js";
import { getPermission, calculatePermission } from "./model_acl.js";
import { clearCache } from "./cache.js";
import { currentPath } from "./helper.js";
import { getPermission, calculatePermission } from "./model_acl.js";
import { rm, mv } from "./model_files.js";
import { currentPath, extractPath } from "./helper.js";
const modalOpt = {
withButtonsRight: "OK",
withButtonsLeft: "CANCEL",
withButtonsRight: t("OK"),
withButtonsLeft: t("CANCEL"),
};
export default async function(render) {
@ -297,7 +297,6 @@ function componentRight(render) {
rxjs.mergeMap(async (show) => {
const $input = qs($page, "input");
const $searchImg = qs($page, "img");
const hide_left_side = document.body.clientWidth < 500;
if (show) {
$page.classList.add("hover");
$input.value = "";
@ -305,11 +304,9 @@ function componentRight(render) {
$searchImg.setAttribute("src", "data:image/svg+xml;base64," + ICONS.CROSS);
$searchImg.setAttribute("alt", "close");
if (hide_left_side) {
const $listOfButtons = $page.parentElement.firstElementChild.children
for (let $item of $listOfButtons) {
$item.classList.add("hidden");
}
const $listOfButtons = $page.parentElement.firstElementChild.children
for (let $item of $listOfButtons) {
$item.classList.add("hidden");
}
setAction(null); // reset new file, new folder
await animate($input, {
@ -326,12 +323,10 @@ function componentRight(render) {
time: 100,
});
$input.classList.add("hidden");
if (hide_left_side) {
const $listOfButtons = $page.parentElement.firstElementChild.children
for (let $item of $listOfButtons) {
$item.classList.remove("hidden");
animate($item, { time: 100, keyframes: slideXIn(5) })
}
const $listOfButtons = $page.parentElement.firstElementChild.children
for (let $item of $listOfButtons) {
$item.classList.remove("hidden");
animate($item, { time: 100, keyframes: slideXIn(5) })
}
setState("search", "");
}

7
public/assets/pages/filespage/helper.js
View file

@ -4,6 +4,13 @@ export function currentPath() {
return decodeURIComponent(location.pathname.replace(new RegExp("^/files"), ""));
}
export function extractPath(path) {
path = path.replace(new RegExp("/$"), "");
const p = path.split("/");
const filename = p.pop();
return [p.join("/") + "/", filename];
}
export function sort(files, type, order) {
switch(type) {
case "name": return sortByName(files, order);

6
public/assets/pages/filespage/model_acl.js
View file

@ -21,6 +21,8 @@ export function calculatePermission(path, action) {
}
}
export function getPermission() {
return perms$.asObservable();
export function getPermission(path) {
return perms$.asObservable().pipe(
rxjs.map((perms) => perms[path]),
);
}

133
public/assets/pages/filespage/model_files.js
View file

@ -3,58 +3,111 @@ import ajax from "../../lib/ajax.js";
import { setPermissions } from "./model_acl.js";
import fscache from "./cache.js";
import {
rm as cacheRm, mv as cacheMv, save as cacheSave,
touch as cacheTouch, mkdir as cacheMkdir, ls as middlewareLs,
} from "./cache_transcient.js";
/*
* ls is in the hot path. To make it look faster, we keep a cache of its results locally
* and refresh the screen twice, a first time with the result of the cache and another time
* with the fresh data.
* The naive approach would be to make an API call and refresh the screen after an action
* is made but that give a poor UX. Instead, we rely on 2 layers of caching:
* - the indexedDB cache that stores the part of the filesystem we've already visited. That way
* we can make navigation feel instant by first returning what's in the cache first and only
* refresh the screen if our cache is out of date.
* - the transcient cache which is used whenever the user do something. For example, when creating
* a file we have 3 actions being done:
* 1. a new file is shown in the UI but with a loading spinner
* 2. the api call is made
* 3. the new file is being persisted in the screen if the API call is a success
*/
export function touch(path) {
const ajax$ = rxjs.of(null).pipe(
rxjs.delay(1000),
// rxjs.tap(() => {
// throw new Error("NOOOO");
// }),
rxjs.delay(1000),
);
return cacheTouch(ajax$, path);
}
export function mkdir(path) {
const ajax$ = rxjs.of(null).pipe(
rxjs.delay(1000),
// rxjs.tap(() => {
// throw new Error("NOOOO");
// }),
rxjs.delay(1000),
);
return cacheMkdir(ajax$, path);
}
export function rm(...paths) {
const ajax$ = rxjs.of(null).pipe(
rxjs.delay(1000),
// rxjs.tap(() => {
// throw new Error("NOOOO");
// }),
rxjs.delay(1000),
);
return cacheRm(ajax$, ...paths);
}
export function mv(from, to) {
const ajax$ = rxjs.of(null).pipe(rxjs.delay(1000));
return cacheMv(ajax$, from, to);
}
export function save(path) { // TODO
return rxjs.of(null).pipe(rxjs.delay(1000));
}
export function ls(path) {
const lsFromCache = (path) => rxjs.from(fscache().get(path));
const lsFromHttp = (path) => ajax({
url: `/api/files/ls?path=${path}`,
responseType: "json"
}).pipe(
rxjs.map(({ responseJSON }) => ({
files: responseJSON.results,
permissions: responseJSON.permissions,
})),
rxjs.tap((data) => fscache().store(path, data)),
);
return rxjs.combineLatest(
lsFromCache(path),
rxjs.merge(
rxjs.of(null),
rxjs.merge(rxjs.of(null), rxjs.fromEvent(window, "keydown").pipe( // "r" shorcut
rxjs.filter((e) => e.keyCode === 82 && document.activeElement.tagName !== "INPUT"),
)).pipe(
rxjs.switchMap(() => lsFromHttp(path)),
rxjs.tap(({ permissions }) => setPermissions(path, permissions)),
),
)
).pipe(rxjs.mergeMap(([cache, http]) => {
if (http && cache) {
let shouldRefresh = false;
if (http.files.length !== cache.files.length) return rxjs.of(http);
else if (JSON.stringify(http.permissions) !== JSON.stringify(cache.permissions)) return rxjs.of(http);
for (let i=0; i<http.files.length; i++) {
if (http.files[i].type !== cache.files[i].type ||
http.files[i].name !== cache.files[i].name) {
return rxjs.of(http);
)).pipe(rxjs.switchMap(() => lsFromHttp(path))),
),
).pipe(
rxjs.mergeMap(([cache, http]) => {
if (http) return rxjs.of(http);
if (cache) return rxjs.of(cache);
return rxjs.EMPTY;
}),
rxjs.distinctUntilChanged((prev, curr) => {
let refresh = false;
if (prev.files.length !== curr.files.length) refresh = true;
else if (JSON.stringify(prev.permissions) !== JSON.stringify(curr.permissions)) refresh = true;
else {
for (let i=0; i<curr.files.length; i++) {
if (curr.files[i].type !== prev.files[i].type ||
curr.files[i].size !== prev.files[i].size ||
curr.files[i].name !== prev.files[i].name) {
refresh = true;
break;
}
}
}
}
if (http) return rxjs.of(http);
if (cache) return rxjs.of(cache);
return rxjs.EMPTY;
}));
}
function lsFromCache(path) {
return rxjs.from(fscache().get(path));
}
function lsFromHttp(path) {
return ajax({
url: `/api/files/ls?path=${path}`,
responseType: "json"
}).pipe(
// rxjs.delay(1000),
rxjs.map(({ responseJSON }) => ({
files: responseJSON.results,
permissions: responseJSON.permissions,
})),
rxjs.tap((data) => fscache().store(path, data)),
return !refresh;
}),
rxjs.tap(({ permissions }) => setPermissions(path, permissions)),
middlewareLs(path),
);
}

22
public/assets/pages/filespage/thing.js
View file

@ -2,6 +2,8 @@ import { createElement, createFragment } from "../../lib/skeleton/index.js";
import { qs } from "../../lib/dom.js";
import assert from "../../lib/assert.js";
import { extractPath } from "./helper.js";
import { mv } from "./model_files.js";
import { files$ } from "./ctrl_filesystem.js";
import { addSelection, isSelected } from "./state_selection.js";
@ -12,7 +14,7 @@ const IMAGE = {
};
const $tmpl = createElement(`
<a href="__TEMPLATE__" class="component_thing no-select" draggable="true" data-link>
<a href="__TEMPLATE__" class="component_thing no-select" draggable="false" data-link>
<div class="component_checkbox"><input name="select" type="checkbox"><span class="indicator"></span></div>
<img class="component_icon" draggable="false" src="__TEMPLATE__" alt="directory">
<div class="info_extension"><span></span></div>
@ -42,6 +44,7 @@ export function createThing({
view = "",
n = 0,
read_only = false,
permissions = {},
}) {
const $thing = $tmpl.cloneNode(true);
assert.type($thing, window.HTMLElement);
@ -87,9 +90,9 @@ export function createThing({
}
const checked = isSelected(n);
if (permissions && permissions.can_move !== false) $thing.setAttribute("draggable", "true");
$thing.classList.add(checked ? "selected" : "not-selected");
$checkbox.firstElementChild.checked = checked;
$checkbox.onclick = (e) => {
e.preventDefault();
e.stopPropagation();
@ -112,9 +115,20 @@ export function createThing({
$thing.ondragleave = () => {
$thing.classList.remove("hover");
};
$thing.ondrop = (e) => {
$thing.ondrop = async (e) => {
$thing.classList.remove("hover");
console.log("DROPPED!", e.dataTransfer.getData("path"));
const from = e.dataTransfer.getData("path");
let to = path;
if (from === to) return;
const isDir = (p) => new RegExp("/$").test(p);
if (isDir(to)) {
const [fromBasepath, fromName] = extractPath(from);
to += fromName;
if (isDir(from)) to += "/";
}
await mv(from, to).toPromise();
};
return $thing;
}