Fix scan issue when encountering invalid symlinks (#871)

* Implement fixed symwalk algorithm
* Remove dependency

go.mod

@@ -8,7 +8,6 @@ require (
 	github.com/chromedp/cdproto v0.0.0-20200608134039-8a80cdaf865c
 	github.com/chromedp/chromedp v0.5.3
 	github.com/disintegration/imaging v1.6.0
-	github.com/facebookgo/symwalk v0.0.0-20150726040526-42004b9f3222
 	github.com/go-chi/chi v4.0.2+incompatible
 	github.com/gobuffalo/packr/v2 v2.0.2
 	github.com/golang-migrate/migrate/v4 v4.3.1

go.sum

@@ -121,8 +121,6 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
-github.com/facebookgo/symwalk v0.0.0-20150726040526-42004b9f3222 h1:ivxAxcE9py2xLAqpcEwN7sN711aLfEWgh3cY0aha7uY=
-github.com/facebookgo/symwalk v0.0.0-20150726040526-42004b9f3222/go.mod h1:PgrCjL2+FgkITqxQI+erRTONtAv4JkpOzun5ozKW/Jg=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=

pkg/manager/task_scan.go

@@ -11,7 +11,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/facebookgo/symwalk"
 	"github.com/jmoiron/sqlx"
 
 	"github.com/stashapp/stash/pkg/database"
@@ -655,7 +654,7 @@ func walkFilesToScan(s *models.StashConfig, f filepath.WalkFunc) error {
 	excludeVid := config.GetExcludes()
 	excludeImg := config.GetImageExcludes()
 
-	return symwalk.Walk(s.Path, func(path string, info os.FileInfo, err error) error {
+	return utils.SymWalk(s.Path, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			logger.Warnf("error scanning %s: %s", path, err.Error())
 			return nil

pkg/utils/symwalk.go

@@ -0,0 +1,80 @@
+// Modified from github.com/facebookgo/symwalk
+
+// BSD License
+
+// For symwalk software
+
+// Copyright (c) 2015, Facebook, Inc. All rights reserved.
+
+// Redistribution and use in source and binary forms, with or without modification,
+// are permitted provided that the following conditions are met:
+
+//  * Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer.
+
+//  * Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution.
+
+//  * Neither the name Facebook nor the names of its contributors may be used to
+//    endorse or promote products derived from this software without specific
+//    prior written permission.
+
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package utils
+
+import (
+	"os"
+	"path/filepath"
+)
+
+// symwalkFunc calls the provided WalkFn for regular files.
+// However, when it encounters a symbolic link, it resolves the link fully using the
+// filepath.EvalSymlinks function and recursively calls symwalk.Walk on the resolved path.
+// This ensures that unlink filepath.Walk, traversal does not stop at symbolic links.
+//
+// Note that symwalk.Walk does not terminate if there are any non-terminating loops in
+// the file structure.
+func walk(filename string, linkDirname string, walkFn filepath.WalkFunc) error {
+	symWalkFunc := func(path string, info os.FileInfo, err error) error {
+
+		if fname, err := filepath.Rel(filename, path); err == nil {
+			path = filepath.Join(linkDirname, fname)
+		} else {
+			return err
+		}
+
+		if err == nil && info.Mode()&os.ModeSymlink == os.ModeSymlink {
+			finalPath, err := filepath.EvalSymlinks(path)
+			if err != nil {
+				// don't bail out if symlink is invalid
+				return walkFn(path, info, err)
+			}
+			info, err := os.Lstat(finalPath)
+			if err != nil {
+				return walkFn(path, info, err)
+			}
+			if info.IsDir() {
+				return walk(finalPath, path, walkFn)
+			}
+		}
+
+		return walkFn(path, info, err)
+	}
+	return filepath.Walk(filename, symWalkFunc)
+}
+
+// SymWalk extends filepath.Walk to also follow symlinks
+func SymWalk(path string, walkFn filepath.WalkFunc) error {
+	return walk(path, path, walkFn)
+}

vendor/github.com/facebookgo/symwalk/license

@@ -1,30 +0,0 @@
-BSD License
-
-For symwalk software
-
-Copyright (c) 2015, Facebook, Inc. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification,
-are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright notice, this
-   list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above copyright notice,
-   this list of conditions and the following disclaimer in the documentation
-   and/or other materials provided with the distribution.
-
- * Neither the name Facebook nor the names of its contributors may be used to
-   endorse or promote products derived from this software without specific
-   prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/facebookgo/symwalk/patents

@@ -1,33 +0,0 @@
-Additional Grant of Patent Rights Version 2
-
-"Software" means the symwalk software distributed by Facebook, Inc.
-
-Facebook, Inc. ("Facebook") hereby grants to each recipient of the Software
-("you") a perpetual, worldwide, royalty-free, non-exclusive, irrevocable
-(subject to the termination provision below) license under any Necessary
-Claims, to make, have made, use, sell, offer to sell, import, and otherwise
-transfer the Software. For avoidance of doubt, no license is granted under
-Facebook’s rights in any patent claims that are infringed by (i) modifications
-to the Software made by you or any third party or (ii) the Software in
-combination with any software or other technology.
-
-The license granted hereunder will terminate, automatically and without notice,
-if you (or any of your subsidiaries, corporate affiliates or agents) initiate
-directly or indirectly, or take a direct financial interest in, any Patent
-Assertion: (i) against Facebook or any of its subsidiaries or corporate
-affiliates, (ii) against any party if such Patent Assertion arises in whole or
-in part from any software, technology, product or service of Facebook or any of
-its subsidiaries or corporate affiliates, or (iii) against any party relating
-to the Software. Notwithstanding the foregoing, if Facebook or any of its
-subsidiaries or corporate affiliates files a lawsuit alleging patent
-infringement against you in the first instance, and you respond by filing a
-patent infringement counterclaim in that lawsuit against that party that is
-unrelated to the Software, the license granted hereunder will not terminate
-under section (i) of this paragraph due to such counterclaim.
-
-A "Necessary Claim" is a claim of a patent owned by Facebook that is
-necessarily infringed by the Software standing alone.
-
-A "Patent Assertion" is any lawsuit or other action alleging direct, indirect,
-or contributory infringement or inducement to infringe any patent, including a
-cross-claim or counterclaim.

vendor/github.com/facebookgo/symwalk/readme.md

@@ -1,28 +0,0 @@
-symwalk
-=======
-
-``` go
-    import "github.com/facebookgo/symwalk"
-```
-
-Package symwalk provides an implementation of symbolic link aware filepath walk.
-
-Walk calls [filepath.Walk](http://golang.org/pkg/path/filepath/#Walk) by providing it with a special WalkFn that wraps the given WalkFn.
-This function calls the given WalkFn for regular files.
-However, when it encounters a symbolic link, it resolves the link fully using
-[filepath.EvalSymlinks](http://golang.org/pkg/path/filepath/#EvalSymlinks) and recursively calls symwalk.Walk on the resolved path.
-This ensures that unlike filepath.Walk, traversal does not stop at symbolic links.
-
-Using it can be as simple as:
-
-``` go
-    Walk(
-      "/home/me/src",
-      func(path string, info os.FileInfo, err error) error {
-        fmt.Println(path)
-        return nil
-      },
-    )
-```
-
-**CAVEAT**: Note that symwalk.Walk does not terminate if there are any non-terminating loops in the file structure.

vendor/github.com/facebookgo/symwalk/walk.go

@@ -1,50 +0,0 @@
-// Package symwalk provides an implementation of symbolic link aware filepath walk.
-//
-// filepath.Walk does not follow symbolic links.
-// symwalk.Walk calls filepath.Walk by providing it with a special WalkFn called symWalkFunc.
-package symwalk
-
-import (
-	"os"
-	"path/filepath"
-)
-
-// symwalkFunc calls the provided WalkFn for regular files.
-// However, when it encounters a symbolic link, it resolves the link fully using the
-// filepath.EvalSymlinks function and recursively calls symwalk.Walk on the resolved path.
-// This ensures that unlink filepath.Walk, traversal does not stop at symbolic links.
-//
-// Note that symwalk.Walk does not terminate if there are any non-terminating loops in
-// the file structure.
-func walk(filename string, linkDirname string, walkFn filepath.WalkFunc) error {
-	symWalkFunc := func(path string, info os.FileInfo, err error) error {
-
-		if fname, err := filepath.Rel(filename, path); err == nil {
-			path = filepath.Join(linkDirname, fname)
-		} else {
-			return err
-		}
-
-		if err == nil && info.Mode()&os.ModeSymlink == os.ModeSymlink {
-			finalPath, err := filepath.EvalSymlinks(path)
-			if err != nil {
-				return err
-			}
-			info, err := os.Lstat(finalPath)
-			if err != nil {
-				return walkFn(path, info, err)
-			}
-			if info.IsDir() {
-				return walk(finalPath, path, walkFn)
-			}
-		}
-
-		return walkFn(path, info, err)
-	}
-	return filepath.Walk(filename, symWalkFunc)
-}
-
-// Walk extends filepath.Walk to also follow symlinks
-func Walk(path string, walkFn filepath.WalkFunc) error {
-	return walk(path, path, walkFn)
-}

vendor/modules.txt

@@ -98,8 +98,6 @@ github.com/cpuguy83/go-md2man/v2/md2man
 github.com/davecgh/go-spew/spew
 # github.com/disintegration/imaging v1.6.0
 github.com/disintegration/imaging
-# github.com/facebookgo/symwalk v0.0.0-20150726040526-42004b9f3222
-github.com/facebookgo/symwalk
 # github.com/fsnotify/fsnotify v1.4.7
 github.com/fsnotify/fsnotify
 # github.com/go-chi/chi v4.0.2+incompatible