From 872bb70f6e6f9b375b0b3ebb51e950b1ea639fc3 Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Tue, 20 Oct 2020 17:00:23 +1100 Subject: [PATCH] Fix scan issue when encountering invalid symlinks (#871) * Implement fixed symwalk algorithm * Remove dependency --- go.mod | 1 - go.sum | 2 - pkg/manager/task_scan.go | 3 +- pkg/utils/symwalk.go | 80 +++++++++++++++++++ vendor/github.com/facebookgo/symwalk/license | 30 ------- vendor/github.com/facebookgo/symwalk/patents | 33 -------- .../github.com/facebookgo/symwalk/readme.md | 28 ------- vendor/github.com/facebookgo/symwalk/walk.go | 50 ------------ vendor/modules.txt | 2 - 9 files changed, 81 insertions(+), 148 deletions(-) create mode 100644 pkg/utils/symwalk.go delete mode 100644 vendor/github.com/facebookgo/symwalk/license delete mode 100644 vendor/github.com/facebookgo/symwalk/patents delete mode 100644 vendor/github.com/facebookgo/symwalk/readme.md delete mode 100644 vendor/github.com/facebookgo/symwalk/walk.go diff --git a/go.mod b/go.mod index 56e694c6b..08c335f7e 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/chromedp/cdproto v0.0.0-20200608134039-8a80cdaf865c github.com/chromedp/chromedp v0.5.3 github.com/disintegration/imaging v1.6.0 - github.com/facebookgo/symwalk v0.0.0-20150726040526-42004b9f3222 github.com/go-chi/chi v4.0.2+incompatible github.com/gobuffalo/packr/v2 v2.0.2 github.com/golang-migrate/migrate/v4 v4.3.1 diff --git a/go.sum b/go.sum index 237496e4e..7a51679a7 100644 --- a/go.sum +++ b/go.sum @@ -121,8 +121,6 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/facebookgo/symwalk v0.0.0-20150726040526-42004b9f3222 h1:ivxAxcE9py2xLAqpcEwN7sN711aLfEWgh3cY0aha7uY= -github.com/facebookgo/symwalk v0.0.0-20150726040526-42004b9f3222/go.mod h1:PgrCjL2+FgkITqxQI+erRTONtAv4JkpOzun5ozKW/Jg= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= diff --git a/pkg/manager/task_scan.go b/pkg/manager/task_scan.go index 055e280cc..55e78025a 100644 --- a/pkg/manager/task_scan.go +++ b/pkg/manager/task_scan.go @@ -11,7 +11,6 @@ import ( "sync" "time" - "github.com/facebookgo/symwalk" "github.com/jmoiron/sqlx" "github.com/stashapp/stash/pkg/database" @@ -655,7 +654,7 @@ func walkFilesToScan(s *models.StashConfig, f filepath.WalkFunc) error { excludeVid := config.GetExcludes() excludeImg := config.GetImageExcludes() - return symwalk.Walk(s.Path, func(path string, info os.FileInfo, err error) error { + return utils.SymWalk(s.Path, func(path string, info os.FileInfo, err error) error { if err != nil { logger.Warnf("error scanning %s: %s", path, err.Error()) return nil diff --git a/pkg/utils/symwalk.go b/pkg/utils/symwalk.go new file mode 100644 index 000000000..ccb0f02a7 --- /dev/null +++ b/pkg/utils/symwalk.go @@ -0,0 +1,80 @@ +// Modified from github.com/facebookgo/symwalk + +// BSD License + +// For symwalk software + +// Copyright (c) 2015, Facebook, Inc. All rights reserved. + +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: + +// * Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. + +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. + +// * Neither the name Facebook nor the names of its contributors may be used to +// endorse or promote products derived from this software without specific +// prior written permission. + +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package utils + +import ( + "os" + "path/filepath" +) + +// symwalkFunc calls the provided WalkFn for regular files. +// However, when it encounters a symbolic link, it resolves the link fully using the +// filepath.EvalSymlinks function and recursively calls symwalk.Walk on the resolved path. +// This ensures that unlink filepath.Walk, traversal does not stop at symbolic links. +// +// Note that symwalk.Walk does not terminate if there are any non-terminating loops in +// the file structure. +func walk(filename string, linkDirname string, walkFn filepath.WalkFunc) error { + symWalkFunc := func(path string, info os.FileInfo, err error) error { + + if fname, err := filepath.Rel(filename, path); err == nil { + path = filepath.Join(linkDirname, fname) + } else { + return err + } + + if err == nil && info.Mode()&os.ModeSymlink == os.ModeSymlink { + finalPath, err := filepath.EvalSymlinks(path) + if err != nil { + // don't bail out if symlink is invalid + return walkFn(path, info, err) + } + info, err := os.Lstat(finalPath) + if err != nil { + return walkFn(path, info, err) + } + if info.IsDir() { + return walk(finalPath, path, walkFn) + } + } + + return walkFn(path, info, err) + } + return filepath.Walk(filename, symWalkFunc) +} + +// SymWalk extends filepath.Walk to also follow symlinks +func SymWalk(path string, walkFn filepath.WalkFunc) error { + return walk(path, path, walkFn) +} diff --git a/vendor/github.com/facebookgo/symwalk/license b/vendor/github.com/facebookgo/symwalk/license deleted file mode 100644 index 3c59345af..000000000 --- a/vendor/github.com/facebookgo/symwalk/license +++ /dev/null @@ -1,30 +0,0 @@ -BSD License - -For symwalk software - -Copyright (c) 2015, Facebook, Inc. All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - * Neither the name Facebook nor the names of its contributors may be used to - endorse or promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/facebookgo/symwalk/patents b/vendor/github.com/facebookgo/symwalk/patents deleted file mode 100644 index a23385c00..000000000 --- a/vendor/github.com/facebookgo/symwalk/patents +++ /dev/null @@ -1,33 +0,0 @@ -Additional Grant of Patent Rights Version 2 - -"Software" means the symwalk software distributed by Facebook, Inc. - -Facebook, Inc. ("Facebook") hereby grants to each recipient of the Software -("you") a perpetual, worldwide, royalty-free, non-exclusive, irrevocable -(subject to the termination provision below) license under any Necessary -Claims, to make, have made, use, sell, offer to sell, import, and otherwise -transfer the Software. For avoidance of doubt, no license is granted under -Facebook’s rights in any patent claims that are infringed by (i) modifications -to the Software made by you or any third party or (ii) the Software in -combination with any software or other technology. - -The license granted hereunder will terminate, automatically and without notice, -if you (or any of your subsidiaries, corporate affiliates or agents) initiate -directly or indirectly, or take a direct financial interest in, any Patent -Assertion: (i) against Facebook or any of its subsidiaries or corporate -affiliates, (ii) against any party if such Patent Assertion arises in whole or -in part from any software, technology, product or service of Facebook or any of -its subsidiaries or corporate affiliates, or (iii) against any party relating -to the Software. Notwithstanding the foregoing, if Facebook or any of its -subsidiaries or corporate affiliates files a lawsuit alleging patent -infringement against you in the first instance, and you respond by filing a -patent infringement counterclaim in that lawsuit against that party that is -unrelated to the Software, the license granted hereunder will not terminate -under section (i) of this paragraph due to such counterclaim. - -A "Necessary Claim" is a claim of a patent owned by Facebook that is -necessarily infringed by the Software standing alone. - -A "Patent Assertion" is any lawsuit or other action alleging direct, indirect, -or contributory infringement or inducement to infringe any patent, including a -cross-claim or counterclaim. diff --git a/vendor/github.com/facebookgo/symwalk/readme.md b/vendor/github.com/facebookgo/symwalk/readme.md deleted file mode 100644 index e3cb9a79f..000000000 --- a/vendor/github.com/facebookgo/symwalk/readme.md +++ /dev/null @@ -1,28 +0,0 @@ -symwalk -======= - -``` go - import "github.com/facebookgo/symwalk" -``` - -Package symwalk provides an implementation of symbolic link aware filepath walk. - -Walk calls [filepath.Walk](http://golang.org/pkg/path/filepath/#Walk) by providing it with a special WalkFn that wraps the given WalkFn. -This function calls the given WalkFn for regular files. -However, when it encounters a symbolic link, it resolves the link fully using -[filepath.EvalSymlinks](http://golang.org/pkg/path/filepath/#EvalSymlinks) and recursively calls symwalk.Walk on the resolved path. -This ensures that unlike filepath.Walk, traversal does not stop at symbolic links. - -Using it can be as simple as: - -``` go - Walk( - "/home/me/src", - func(path string, info os.FileInfo, err error) error { - fmt.Println(path) - return nil - }, - ) -``` - -**CAVEAT**: Note that symwalk.Walk does not terminate if there are any non-terminating loops in the file structure. diff --git a/vendor/github.com/facebookgo/symwalk/walk.go b/vendor/github.com/facebookgo/symwalk/walk.go deleted file mode 100644 index ff7e22df4..000000000 --- a/vendor/github.com/facebookgo/symwalk/walk.go +++ /dev/null @@ -1,50 +0,0 @@ -// Package symwalk provides an implementation of symbolic link aware filepath walk. -// -// filepath.Walk does not follow symbolic links. -// symwalk.Walk calls filepath.Walk by providing it with a special WalkFn called symWalkFunc. -package symwalk - -import ( - "os" - "path/filepath" -) - -// symwalkFunc calls the provided WalkFn for regular files. -// However, when it encounters a symbolic link, it resolves the link fully using the -// filepath.EvalSymlinks function and recursively calls symwalk.Walk on the resolved path. -// This ensures that unlink filepath.Walk, traversal does not stop at symbolic links. -// -// Note that symwalk.Walk does not terminate if there are any non-terminating loops in -// the file structure. -func walk(filename string, linkDirname string, walkFn filepath.WalkFunc) error { - symWalkFunc := func(path string, info os.FileInfo, err error) error { - - if fname, err := filepath.Rel(filename, path); err == nil { - path = filepath.Join(linkDirname, fname) - } else { - return err - } - - if err == nil && info.Mode()&os.ModeSymlink == os.ModeSymlink { - finalPath, err := filepath.EvalSymlinks(path) - if err != nil { - return err - } - info, err := os.Lstat(finalPath) - if err != nil { - return walkFn(path, info, err) - } - if info.IsDir() { - return walk(finalPath, path, walkFn) - } - } - - return walkFn(path, info, err) - } - return filepath.Walk(filename, symWalkFunc) -} - -// Walk extends filepath.Walk to also follow symlinks -func Walk(path string, walkFn filepath.WalkFunc) error { - return walk(path, path, walkFn) -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 6ec86c574..ea595a4df 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -98,8 +98,6 @@ github.com/cpuguy83/go-md2man/v2/md2man github.com/davecgh/go-spew/spew # github.com/disintegration/imaging v1.6.0 github.com/disintegration/imaging -# github.com/facebookgo/symwalk v0.0.0-20150726040526-42004b9f3222 -github.com/facebookgo/symwalk # github.com/fsnotify/fsnotify v1.4.7 github.com/fsnotify/fsnotify # github.com/go-chi/chi v4.0.2+incompatible