diff --git a/server/plugin/plg_search_sqlitefts/crawler/configuration.go b/server/plugin/plg_search_sqlitefts/crawler/configuration.go index 70dbe1de..26b75a2c 100644 --- a/server/plugin/plg_search_sqlitefts/crawler/configuration.go +++ b/server/plugin/plg_search_sqlitefts/crawler/configuration.go @@ -1,12 +1,15 @@ package plg_search_sqlitefts import ( + "strings" + . "github.com/mickael-kerjean/filestash/server/common" ) func init() { Hooks.Register.Onload(func() { SEARCH_ENABLE() + SEARCH_EXCLUSION() SEARCH_PROCESS_MAX() SEARCH_PROCESS_PAR() SEARCH_REINDEX() @@ -16,11 +19,6 @@ func init() { }) } -var INDEXING_EXCLUSION = []string{ - "/node_modules/", "/bower_components/", - "/.cache/", "/.npm/", "/.git/", -} - var SEARCH_ENABLE = func() bool { return Config.Get("features.search.enable").Schema(func(f *FormElement) *FormElement { if f == nil { @@ -30,7 +28,7 @@ var SEARCH_ENABLE = func() bool { f.Type = "enable" f.Target = []string{ "process_max", "process_par", "reindex_time", - "cycle_time", "max_size", "indexer_ext", + "cycle_time", "max_size", "indexer_ext", "folder_exclusion", } f.Description = "Enable/Disable full text search" f.Placeholder = "Default: true" @@ -39,6 +37,26 @@ var SEARCH_ENABLE = func() bool { }).Bool() } +var SEARCH_EXCLUSION = func() []string { + listOfFolders := Config.Get("features.search.folder_exclusion").Schema(func(f *FormElement) *FormElement { + if f == nil { + f = &FormElement{} + } + f.Id = "folder_exclusion" + f.Name = "folder_exclusion" + f.Type = "text" + f.Description = "Exclude folders during the exploration phase" + f.Placeholder = "Default: node_modules,bower_components,.cache,.npm,.git" + f.Default = "node_modules,bower_components,.cache,.npm,.git" + return f + }).String() + out := []string{} + for _, folder := range strings.Split(listOfFolders, ",") { + out = append(out, strings.TrimSpace(folder)) + } + return out +} + var SEARCH_PROCESS_MAX = func() int { return Config.Get("features.search.process_max").Schema(func(f *FormElement) *FormElement { if f == nil { diff --git a/server/plugin/plg_search_sqlitefts/crawler/phase_explore.go b/server/plugin/plg_search_sqlitefts/crawler/phase_explore.go index 3e6f4767..03134639 100644 --- a/server/plugin/plg_search_sqlitefts/crawler/phase_explore.go +++ b/server/plugin/plg_search_sqlitefts/crawler/phase_explore.go @@ -6,6 +6,7 @@ import ( "hash/fnv" "path/filepath" "strconv" + "strings" "time" . "github.com/mickael-kerjean/filestash/server/common" @@ -23,6 +24,7 @@ func (this *Crawler) Discover(tx indexer.Manager) bool { this.CurrentPhase = PHASE_INDEXING return false } + files, err := this.Backend.Ls(doc.Path) if err != nil { this.CurrentPhase = "" @@ -58,9 +60,19 @@ func (this *Crawler) Discover(tx indexer.Manager) bool { } // Insert the newly found data within our index + excluded := SEARCH_EXCLUSION() for i := range files { f := files[i] name := f.Name() + skip := false + for i := 0; i < len(excluded); i++ { + if name == excluded[i] || strings.Contains(doc.Path, excluded[i]) { + skip = true + } + } + if skip { + continue + } if f.IsDir() { var performPush bool = false p := filepath.Join(doc.Path, name) diff --git a/server/plugin/plg_search_sqlitefts/crawler/phase_utils.go b/server/plugin/plg_search_sqlitefts/crawler/phase_utils.go index cad0bfea..92715c33 100644 --- a/server/plugin/plg_search_sqlitefts/crawler/phase_utils.go +++ b/server/plugin/plg_search_sqlitefts/crawler/phase_utils.go @@ -4,7 +4,6 @@ import ( "io/fs" "os" "path/filepath" - "strings" "time" . "github.com/mickael-kerjean/filestash/server/common" @@ -16,11 +15,6 @@ func updateFile(path string, backend IBackend, tx indexer.Manager) error { if err := tx.IndexTimeUpdate(path, time.Now()); err != nil { return err } - for i := 0; i < len(INDEXING_EXCLUSION); i++ { - if strings.Contains(path, INDEXING_EXCLUSION[i]) { - return nil - } - } reader, err := backend.Cat(path) if err != nil { tx.RemoveAll(path) @@ -44,12 +38,6 @@ func updateFolder(path string, backend IBackend, tx indexer.Manager) error { return err } - for i := 0; i < len(INDEXING_EXCLUSION); i++ { - if strings.Contains(path, INDEXING_EXCLUSION[i]) { - return nil - } - } - // Fetch list of folders as in the remote filesystem currFiles, err := backend.Ls(path) if err != nil {