Find scrapers in subdirectories (#554)

This commit is contained in:
WithoutPants 2020-05-19 08:44:33 +10:00 committed by GitHub
parent 5450fe8e9a
commit 05488d59c3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23

View file

@ -2,6 +2,7 @@ package scraper
import (
"errors"
"os"
"path/filepath"
"strconv"
@ -21,7 +22,13 @@ func loadScrapers() ([]scraperConfig, error) {
scrapers = make([]scraperConfig, 0)
logger.Debugf("Reading scraper configs from %s", path)
scraperFiles, err := filepath.Glob(filepath.Join(path, "*.yml"))
scraperFiles := []string{}
err := filepath.Walk(path, func(fp string, f os.FileInfo, err error) error {
if filepath.Ext(fp) == ".yml" {
scraperFiles = append(scraperFiles, fp)
}
return nil
})
if err != nil {
logger.Errorf("Error reading scraper configs: %s", err.Error())