This folder is empty
+There is nothing here
diff --git a/client/pages/filespage/thing-new.js b/client/pages/filespage/thing-new.js index 41a3ed0c..9601b747 100644 --- a/client/pages/filespage/thing-new.js +++ b/client/pages/filespage/thing-new.js @@ -15,7 +15,7 @@ export class NewThing extends React.Component { type: null, message: null, icon: null, - search_enabled: "ServiceWorker" in window ? true : false, + search_enabled: CONFIG.enable_search || false, search_input_visible: false, search_keyword: "" }; diff --git a/server/common/cache.go b/server/common/cache.go index 76190d99..d3bd33bf 100644 --- a/server/common/cache.go +++ b/server/common/cache.go @@ -79,7 +79,7 @@ func NewQuickCache(arg ...time.Duration) AppCache { type KeyValueStore struct { cache map[string]interface{} - sync.RWMutex + sync.Mutex } func NewKeyValueStore() KeyValueStore { @@ -87,15 +87,16 @@ func NewKeyValueStore() KeyValueStore { } func (this KeyValueStore) Get(key string) interface{} { - this.RLock() - defer this.RUnlock() - return this.cache[key] + this.Lock() + val := this.cache[key] + this.Unlock() + return val } func (this *KeyValueStore) Set(key string, value interface{}) { this.Lock() - defer this.Unlock() this.cache[key] = value + this.Unlock() } func (this *KeyValueStore) Clear() { diff --git a/server/common/config.go b/server/common/config.go index c5c454d4..27a77421 100644 --- a/server/common/config.go +++ b/server/common/config.go @@ -78,12 +78,6 @@ func NewConfiguration() Configuration { Form{ Title: "features", Form: []Form{ - Form{ - Title: "search", - Elmnts: []FormElement{ - FormElement{Name: "enable", Type: "boolean", Default: true, Description: "Enable/Disable the search feature"}, - }, - }, Form{ Title: "share", Elmnts: []FormElement{ diff --git a/server/common/constants.go b/server/common/constants.go index 2e9fdff6..a64781f5 100644 --- a/server/common/constants.go +++ b/server/common/constants.go @@ -1,11 +1,18 @@ package common +import ( + "os" + "path/filepath" +) + const ( APP_VERSION = "v0.4" - CONFIG_PATH = "data/config/" PLUGIN_PATH = "data/plugin/" - LOG_PATH = "data/log/" - TMP_PATH = "data/tmp/" + LOG_PATH = "data/state/log/" + CONFIG_PATH = "data/state/config/" + DB_PATH = "data/state/db/" + FTS_PATH = "data/state/db/search/" + TMP_PATH = "data/cache/tmp/" COOKIE_NAME_AUTH = "auth" COOKIE_NAME_PROOF = "proof" COOKIE_NAME_ADMIN = "admin" @@ -16,6 +23,15 @@ const ( URL_SETUP = "/admin/setup" ) +func init(){ + os.MkdirAll(filepath.Join(GetCurrentDir(), LOG_PATH), os.ModePerm) + os.MkdirAll(filepath.Join(GetCurrentDir(), FTS_PATH), os.ModePerm) + os.MkdirAll(filepath.Join(GetCurrentDir(), CONFIG_PATH), os.ModePerm) + os.RemoveAll(filepath.Join(GetCurrentDir(), TMP_PATH)) + os.MkdirAll(filepath.Join(GetCurrentDir(), TMP_PATH), os.ModePerm) +} + + var ( BUILD_NUMBER string SECRET_KEY string @@ -24,7 +40,6 @@ var ( SECRET_KEY_DERIVATE_FOR_USER string ) - /* * Improve security by calculating derivative of the secret key to restrict the attack surface * in the worst case scenario with one compromise secret key diff --git a/server/common/log.go b/server/common/log.go index ca0cf184..bc9f7d30 100644 --- a/server/common/log.go +++ b/server/common/log.go @@ -13,7 +13,6 @@ var logfile *os.File func init(){ var err error logPath := filepath.Join(GetCurrentDir(), LOG_PATH) - os.MkdirAll(logPath, os.ModePerm) logfile, err = os.OpenFile(filepath.Join(logPath, "access.log"), os.O_APPEND|os.O_WRONLY|os.O_CREATE, os.ModePerm) if err != nil { slog.Printf("ERROR log file: %+v", err) diff --git a/server/common/types.go b/server/common/types.go index 5d0940fb..a42096b5 100644 --- a/server/common/types.go +++ b/server/common/types.go @@ -23,7 +23,8 @@ type File struct { FName string `json:"name"` FType string `json:"type"` FTime int64 `json:"time"` - FSize int64 `json:"size"` + FSize int64 `json:"size"` + FPath string `json:"path,omitempty"` CanRename *bool `json:"can_rename,omitempty"` CanMove *bool `json:"can_move_directory,omitempty"` CanDelete *bool `json:"can_delete,omitempty"` diff --git a/server/ctrl/config.go b/server/ctrl/config.go index 652d6dd3..8a958677 100644 --- a/server/ctrl/config.go +++ b/server/ctrl/config.go @@ -12,7 +12,7 @@ import ( var ( logpath = filepath.Join(GetCurrentDir(), LOG_PATH, "access.log") - cachepath = filepath.Join(GetCurrentDir(), CONFIG_PATH, "config.json") + configpath = filepath.Join(GetCurrentDir(), CONFIG_PATH, "config.json") pluginpath = filepath.Join(GetCurrentDir(), PLUGIN_PATH) ) @@ -73,7 +73,7 @@ func PrivateConfigHandler(ctx App, res http.ResponseWriter, req *http.Request) { func PrivateConfigUpdateHandler(ctx App, res http.ResponseWriter, req *http.Request) { b, _ := ioutil.ReadAll(req.Body) b = PrettyPrint(b) - file, err := os.Create(cachepath) + file, err := os.Create(configpath) if err != nil { SendErrorResult(res, err) return diff --git a/server/ctrl/export.go b/server/ctrl/export.go index 3eac1e74..2956aba0 100644 --- a/server/ctrl/export.go +++ b/server/ctrl/export.go @@ -14,13 +14,6 @@ import ( "strings" ) - -var EXPORT_PATH string -func init() { - EXPORT_PATH = GetAbsolutePath(TMP_PATH) - os.RemoveAll(EXPORT_PATH) - os.MkdirAll(EXPORT_PATH, os.ModePerm) -} func FileExport(ctx App, res http.ResponseWriter, req *http.Request) { http.SetCookie(res, &http.Cookie{ Name: "download", @@ -41,7 +34,7 @@ func FileExport(ctx App, res http.ResponseWriter, req *http.Request) { return } - var tmpPath string = EXPORT_PATH + "/export_" + QuickString(10) + var tmpPath string = GetAbsolutePath(TMP_PATH) + "/export_" + QuickString(10) var cmd *exec.Cmd var emacsPath string var outPath string diff --git a/server/ctrl/files.go b/server/ctrl/files.go index 9680993d..25eb8f9c 100644 --- a/server/ctrl/files.go +++ b/server/ctrl/files.go @@ -22,15 +22,11 @@ type FileInfo struct { Time int64 `json:"time"` } -const FileCachePath = "data/cache/tmp/" - var FileCache AppCache func init() { FileCache = NewAppCache() - cachePath := filepath.Join(GetCurrentDir(), FileCachePath) - os.RemoveAll(cachePath) - os.MkdirAll(cachePath, os.ModePerm) + cachePath := filepath.Join(GetCurrentDir(), TMP_PATH) FileCache.OnEvict(func(key string, value interface{}) { os.RemoveAll(filepath.Join(cachePath, key)) }) @@ -39,7 +35,7 @@ func init() { func FileLs(ctx App, res http.ResponseWriter, req *http.Request) { if model.CanRead(&ctx) == false { if model.CanUpload(&ctx) == false { - SendErrorResult(res, NewError("Permission denied", 403)) + SendErrorResult(res, ErrPermissionDenied) return } SendSuccessResults(res, make([]FileInfo, 0)) @@ -50,6 +46,7 @@ func FileLs(ctx App, res http.ResponseWriter, req *http.Request) { SendErrorResult(res, err) return } + model.SProc.Append(&ctx, path) // ping the search indexer entries, err := ctx.Backend.Ls(path) if err != nil { @@ -179,7 +176,7 @@ func FileCat(ctx App, res http.ResponseWriter, req *http.Request) { } } } else { - tmpPath := filepath.Join(GetCurrentDir(), FileCachePath, "file_" + QuickString(20) + ".dat") + tmpPath := filepath.Join(GetCurrentDir(), filepath.Join(GetCurrentDir(), TMP_PATH), "file_" + QuickString(20) + ".dat") f, err := os.OpenFile(tmpPath, os.O_RDWR|os.O_CREATE, os.ModePerm); if err != nil { SendErrorResult(res, err) diff --git a/server/ctrl/search.go b/server/ctrl/search.go new file mode 100644 index 00000000..e1ba577e --- /dev/null +++ b/server/ctrl/search.go @@ -0,0 +1,25 @@ +package ctrl + +import ( + . "github.com/mickael-kerjean/filestash/server/common" + "github.com/mickael-kerjean/filestash/server/model" + "net/http" +) + +func FileSearch(ctx App, res http.ResponseWriter, req *http.Request) { + if Config.Get("features.search.enable").Bool() == false { + SendErrorResult(res, ErrNotAllowed) + return + } + + path, err := pathBuilder(ctx, req.URL.Query().Get("path")) + if err != nil { + path = "/" + } + q := req.URL.Query().Get("q") + if model.CanRead(&ctx) == false { + SendErrorResult(res, ErrPermissionDenied) + return + } + SendSuccessResults(res, model.Search(&ctx, path, q)) +} diff --git a/server/ctrl/session.go b/server/ctrl/session.go index 0ccf183d..19b4ae3e 100644 --- a/server/ctrl/session.go +++ b/server/ctrl/session.go @@ -85,7 +85,7 @@ func SessionAuthenticate(ctx App, res http.ResponseWriter, req *http.Request) { HttpOnly: true, SameSite: http.SameSiteStrictMode, } - http.SetCookie(res, &cookie) + http.SetCookie(res, &cookie) if home == "" { SendSuccessResult(res, nil) diff --git a/server/main.go b/server/main.go index 7dee631d..6a8d59a5 100644 --- a/server/main.go +++ b/server/main.go @@ -77,6 +77,8 @@ func Init(a *App) { files.HandleFunc("/rm", NewMiddlewareChain(FileRm, middlewares, *a)).Methods("GET") files.HandleFunc("/mkdir", NewMiddlewareChain(FileMkdir, middlewares, *a)).Methods("GET") files.HandleFunc("/touch", NewMiddlewareChain(FileTouch, middlewares, *a)).Methods("GET") + middlewares = []Middleware{ ApiHeaders, SessionStart, LoggedInOnly } + files.HandleFunc("/search", NewMiddlewareChain(FileSearch, middlewares, *a)).Methods("GET") // API for exporter middlewares = []Middleware{ ApiHeaders, SecureHeaders, RedirectSharedLoginIfNeeded, SessionStart, LoggedInOnly } diff --git a/server/model/index.go b/server/model/index.go index 65c46df0..827115f1 100644 --- a/server/model/index.go +++ b/server/model/index.go @@ -11,13 +11,11 @@ import ( var DB *sql.DB -const DBCachePath = "data/" - func init() { - cachePath := filepath.Join(GetCurrentDir(), DBCachePath) + cachePath := filepath.Join(GetCurrentDir(), DB_PATH) os.MkdirAll(cachePath, os.ModePerm) var err error - if DB, err = sql.Open("sqlite3", cachePath+"/db.sql?_fk=true"); err != nil { + if DB, err = sql.Open("sqlite3", cachePath+"/share.sql?_fk=true"); err != nil { return } diff --git a/server/model/search.go b/server/model/search.go new file mode 100644 index 00000000..5cbb41e3 --- /dev/null +++ b/server/model/search.go @@ -0,0 +1,531 @@ +package model + +import ( + "container/heap" + "database/sql" + "encoding/base64" + "github.com/mattn/go-sqlite3" + . "github.com/mickael-kerjean/filestash/server/common" + "hash/fnv" + "math/rand" + "path/filepath" + "regexp" + "strconv" + "strings" + "sync" + "time" +) + +const ( + PHASE_EXPLORE = "PHASE_EXPLORE" + PHASE_INDEXING = "PHASE_INDEXING" + PHASE_MAINTAIN = "PHASE_MAINTAIN" +) +var ( + SEARCH_ENABLE func() bool + SEARCH_PROCESS_MAX func() int + SEARCH_PROCESS_PAR func() int + SEARCH_REINDEX func() int + CYCLE_TIME func() int + MAX_INDEXING_FSIZE func() int + INDEXING_EXT func() string +) + +var SProc SearchProcess = SearchProcess{ + idx: make([]SearchIndexer, 0), + n: -1, +} + +func init(){ + SEARCH_ENABLE = func() bool { + return Config.Get("features.search.enable").Schema(func(f *FormElement) *FormElement { + if f == nil { + f = &FormElement{} + } + f.Name = "enable" + f.Type = "enable" + f.Target = []string{"process_max", "process_par", "reindex_time", "cycle_time", "max_size", "indexer_ext"} + f.Description = "Enable/Disable the search feature" + f.Placeholder = "Default: false" + f.Default = false + return f + }).Bool() + } + SEARCH_ENABLE() + SEARCH_PROCESS_MAX = func() int { + return Config.Get("features.search.process_max").Schema(func(f *FormElement) *FormElement { + if f == nil { + f = &FormElement{} + } + f.Id = "process_max" + f.Name = "process_max" + f.Type = "number" + f.Description = "Size of the pool containing the indexers" + f.Placeholder = "Default: 5" + f.Default = 5 + return f + }).Int() + } + SEARCH_PROCESS_MAX() + SEARCH_PROCESS_PAR = func() int { + return 1 + return Config.Get("features.search.process_par").Schema(func(f *FormElement) *FormElement { + if f == nil { + f = &FormElement{} + } + f.Id = "process_par" + f.Name = "process_par" + f.Type = "number" + f.Description = "How many concurrent indexers are running in the same time (requires a restart)" + f.Placeholder = "Default: 2" + f.Default = 2 + return f + }).Int() + } + SEARCH_PROCESS_PAR() + SEARCH_REINDEX = func() int { + return Config.Get("features.search.reindex_time").Schema(func(f *FormElement) *FormElement { + if f == nil { + f = &FormElement{} + } + f.Id = "reindex_time" + f.Name = "reindex_time" + f.Type = "number" + f.Description = "Time in hours after which we consider our index to be stale and needs to be reindexed" + f.Placeholder = "Default: 24h" + f.Default = 24 + return f + }).Int() + } + SEARCH_REINDEX() + CYCLE_TIME = func() int { + return Config.Get("features.search.cycle_time").Schema(func(f *FormElement) *FormElement { + if f == nil { + f = &FormElement{} + } + f.Id = "cycle_time" + f.Name = "cycle_time" + f.Type = "number" + f.Description = "Time the indexer needs to spend for each cycle in seconds (discovery, indexing and maintenance)" + f.Placeholder = "Default: 10s" + f.Default = 10 + return f + }).Int() + } + CYCLE_TIME() + MAX_INDEXING_FSIZE = func() int { + return Config.Get("features.search.max_size").Schema(func(f *FormElement) *FormElement { + if f == nil { + f = &FormElement{} + } + f.Id = "max_size" + f.Name = "max_size" + f.Type = "number" + f.Description = "Maximum size of files the indexer will perform full text search" + f.Placeholder = "Default: 524288000 => 512MB" + f.Default = 524288000 + return f + }).Int() + } + MAX_INDEXING_FSIZE() + INDEXING_EXT = func() string { + return Config.Get("features.search.indexer_ext").Schema(func(f *FormElement) *FormElement { + if f == nil { + f = &FormElement{} + } + f.Id = "indexer_ext" + f.Name = "indexer_ext" + f.Type = "string" + f.Description = "File extension we want to see indexed" + f.Placeholder = "Default: org,txt,docx,pdf,md" + f.Default = "/" + return f + }).String() + } + INDEXING_EXT() + + runner := func() { + for { + if SEARCH_ENABLE() == false { + time.Sleep(60 * time.Second) + continue + } + sidx := SProc.Peek() + if sidx == nil { + time.Sleep(5 * time.Second) + continue + } else if sidx.FoldersUnknown.Len() == 0 { + time.Sleep(5 * time.Second) + continue + } + sidx.mu.Lock() + sidx.Execute() + sidx.mu.Unlock() + } + } + for i:=0; i