fix (plg_search_sqlitefts): exclusion field

server/plugin/plg_search_sqlitefts/crawler/configuration.go

@@ -1,12 +1,15 @@
 package plg_search_sqlitefts
 
 import (
+	"strings"
+
 	. "github.com/mickael-kerjean/filestash/server/common"
 )
 
 func init() {
 	Hooks.Register.Onload(func() {
 		SEARCH_ENABLE()
+		SEARCH_EXCLUSION()
 		SEARCH_PROCESS_MAX()
 		SEARCH_PROCESS_PAR()
 		SEARCH_REINDEX()
@@ -16,11 +19,6 @@ func init() {
 	})
 }
 
-var INDEXING_EXCLUSION = []string{
-	"/node_modules/", "/bower_components/",
-	"/.cache/", "/.npm/", "/.git/",
-}
-
 var SEARCH_ENABLE = func() bool {
 	return Config.Get("features.search.enable").Schema(func(f *FormElement) *FormElement {
 		if f == nil {
@@ -30,7 +28,7 @@ var SEARCH_ENABLE = func() bool {
 		f.Type = "enable"
 		f.Target = []string{
 			"process_max", "process_par", "reindex_time",
-			"cycle_time", "max_size", "indexer_ext",
+			"cycle_time", "max_size", "indexer_ext", "folder_exclusion",
 		}
 		f.Description = "Enable/Disable full text search"
 		f.Placeholder = "Default: true"
@@ -39,6 +37,26 @@ var SEARCH_ENABLE = func() bool {
 	}).Bool()
 }
 
+var SEARCH_EXCLUSION = func() []string {
+	listOfFolders := Config.Get("features.search.folder_exclusion").Schema(func(f *FormElement) *FormElement {
+		if f == nil {
+			f = &FormElement{}
+		}
+		f.Id = "folder_exclusion"
+		f.Name = "folder_exclusion"
+		f.Type = "text"
+		f.Description = "Exclude folders during the exploration phase"
+		f.Placeholder = "Default: node_modules,bower_components,.cache,.npm,.git"
+		f.Default = "node_modules,bower_components,.cache,.npm,.git"
+		return f
+	}).String()
+	out := []string{}
+	for _, folder := range strings.Split(listOfFolders, ",") {
+		out = append(out, strings.TrimSpace(folder))
+	}
+	return out
+}
+
 var SEARCH_PROCESS_MAX = func() int {
 	return Config.Get("features.search.process_max").Schema(func(f *FormElement) *FormElement {
 		if f == nil {

server/plugin/plg_search_sqlitefts/crawler/phase_explore.go

@@ -6,6 +6,7 @@ import (
 	"hash/fnv"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"time"
 
 	. "github.com/mickael-kerjean/filestash/server/common"
@@ -23,6 +24,7 @@ func (this *Crawler) Discover(tx indexer.Manager) bool {
 		this.CurrentPhase = PHASE_INDEXING
 		return false
 	}
+
 	files, err := this.Backend.Ls(doc.Path)
 	if err != nil {
 		this.CurrentPhase = ""
@@ -58,9 +60,19 @@ func (this *Crawler) Discover(tx indexer.Manager) bool {
 	}
 
 	// Insert the newly found data within our index
+	excluded := SEARCH_EXCLUSION()
 	for i := range files {
 		f := files[i]
 		name := f.Name()
+		skip := false
+		for i := 0; i < len(excluded); i++ {
+			if name == excluded[i] || strings.Contains(doc.Path, excluded[i]) {
+				skip = true
+			}
+		}
+		if skip {
+			continue
+		}
 		if f.IsDir() {
 			var performPush bool = false
 			p := filepath.Join(doc.Path, name)

server/plugin/plg_search_sqlitefts/crawler/phase_utils.go

@@ -4,7 +4,6 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
-	"strings"
 	"time"
 
 	. "github.com/mickael-kerjean/filestash/server/common"
@@ -16,11 +15,6 @@ func updateFile(path string, backend IBackend, tx indexer.Manager) error {
 	if err := tx.IndexTimeUpdate(path, time.Now()); err != nil {
 		return err
 	}
-	for i := 0; i < len(INDEXING_EXCLUSION); i++ {
-		if strings.Contains(path, INDEXING_EXCLUSION[i]) {
-			return nil
-		}
-	}
 	reader, err := backend.Cat(path)
 	if err != nil {
 		tx.RemoveAll(path)
@@ -44,12 +38,6 @@ func updateFolder(path string, backend IBackend, tx indexer.Manager) error {
 		return err
 	}
 
-	for i := 0; i < len(INDEXING_EXCLUSION); i++ {
-		if strings.Contains(path, INDEXING_EXCLUSION[i]) {
-			return nil
-		}
-	}
-
 	// Fetch list of folders as in the remote filesystem
 	currFiles, err := backend.Ls(path)
 	if err != nil {