From f9f7b5c4d3aa9bc6fb3981a6f2fd0f04a3a51fc0 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 17 Apr 2019 12:59:23 +1000 Subject: [PATCH] feature (backend): add a mysql backend --- Makefile | 1 + README.md | 6 +- client/pages/adminpage/dashboard.scss | 2 + server/plugin/plg_backend_mysql/index.go | 992 +++++++++++++++++++++++ 4 files changed, 998 insertions(+), 3 deletions(-) create mode 100644 server/plugin/plg_backend_mysql/index.go diff --git a/Makefile b/Makefile index 7892392c..7cf1327f 100644 --- a/Makefile +++ b/Makefile @@ -14,5 +14,6 @@ build_plugins: go build -buildmode=plugin -o ./dist/data/plugin/image.so server/plugin/plg_image_light/index.go go build -buildmode=plugin -o ./dist/data/plugin/backend_dav.so server/plugin/plg_backend_dav/index.go go build -buildmode=plugin -o ./dist/data/plugin/backend_ldap.so server/plugin/plg_backend_ldap/index.go + go build -buildmode=plugin -o ./dist/data/plugin/backend_mysql.so server/plugin/plg_backend_mysql/index.go go build -buildmode=plugin -o dist/data/plugin/backend_backblaze.so server/plugin/plg_backend_backblaze/index.go go build -buildmode=plugin -o dist/data/plugin/security_scanner.so server/plugin/plg_security_scanner/index.go diff --git a/README.md b/README.md index f3ddcfa5..18a2b875 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@

A Dropbox-like file manager that let you manage your data anywhere it is located:
- FTP • SFTP • WebDAV • Git • S3 • LDAP • Minio
- CardDAV • CalDAV • Backblaze B2
- Dropbox • Google Drive + FTP • SFTP • WebDAV • Git • S3 • LDAP • Mysql
+ CardDAV • CalDAV • Backblaze B2 • Minio
+ Dropbox • Google Drive

diff --git a/client/pages/adminpage/dashboard.scss b/client/pages/adminpage/dashboard.scss index 79c9559d..8f1a067e 100644 --- a/client/pages/adminpage/dashboard.scss +++ b/client/pages/adminpage/dashboard.scss @@ -31,6 +31,8 @@ box-shadow: 2px 2px 10px var(--emphasis-primary); margin: 8px; padding: 30px 0; + @media (max-width: 900px){ padding: 25px 0; } + @media (max-width: 750px){ padding: 20px 0; } text-align: center; background: var(--primary); color: white; diff --git a/server/plugin/plg_backend_mysql/index.go b/server/plugin/plg_backend_mysql/index.go new file mode 100644 index 00000000..a9e5c5cf --- /dev/null +++ b/server/plugin/plg_backend_mysql/index.go @@ -0,0 +1,992 @@ +package main + +import ( + "database/sql" + "encoding/json" + "fmt" + _ "github.com/go-sql-driver/mysql" + . "github.com/mickael-kerjean/filestash/server/common" + "io" + "os" + "regexp" + "sort" + "strings" + "strconv" + "time" +) + +type Mysql struct { + params map[string]string + db *sql.DB +} + +func Init(config *Configuration) { + Backend.Register("mysql", Mysql{}) +} + +func (this Mysql) Init(params map[string]string, app *App) (IBackend, error) { + if params["host"] == "" { + params["host"] = "127.0.0.1" + } + if params["port"] == "" { + params["port"] = "3306" + } + + db, err := sql.Open( + "mysql", + fmt.Sprintf( + "%s:%s@tcp(%s:%s)/", + params["username"], + params["password"], + params["host"], + params["port"], + ), + ) + if err != nil { + return nil, err + } + return Mysql{ + params: params, + db: db, + }, nil +} + +func (this Mysql) LoginForm() Form { + return Form{ + Elmnts: []FormElement{ + FormElement{ + Name: "type", + Type: "hidden", + Value: "mysql", + }, + FormElement{ + Name: "host", + Type: "text", + Placeholder: "Host", + }, + FormElement{ + Name: "username", + Type: "text", + Placeholder: "Username", + }, + FormElement{ + Name: "password", + Type: "password", + Placeholder: "Password", + }, + FormElement{ + Name: "port", + Type: "number", + Placeholder: "Port", + }, + }, + } +} + +func (this Mysql) Ls(path string) ([]os.FileInfo, error) { + defer this.db.Close() + location, err := NewDBLocation(path) + if err != nil { + return nil, err + } + files := make([]os.FileInfo, 0) + + if location.db == "" { // first level folder = a list all the available databases + rows, err := this.db.Query("SELECT s.schema_name, t.update_time, t.create_time FROM information_schema.SCHEMATA as s LEFT JOIN ( SELECT table_schema, MAX(update_time) as update_time, MAX(create_time) as create_time FROM information_schema.tables GROUP BY table_schema ) as t ON s.schema_name = t.table_schema ORDER BY schema_name") + if err != nil { + return nil, err + } + for rows.Next() { + var name string + var create string + var rcreate sql.RawBytes + var update string + var rupdate sql.RawBytes + + if err := rows.Scan(&name, &rcreate, &rupdate); err != nil { + return nil, err + } + create = string(rcreate) + update = string(rupdate) + + files = append(files, File{ + FName: name, + FType: "directory", + FTime: func() int64 { + var t time.Time + var err error + if create == "" && update == "" { + return 0 + } else if update == "" { + if t, err = time.Parse("2006-01-02 15:04:05", create); err != nil { + return 0 + } + return t.Unix() + } + if t, err = time.Parse("2006-01-02 15:04:05", update); err != nil { + return 0 + } + return t.Unix() + }(), + }) + } + return files, nil + } else if location.table == "" { // second level folder = a list of all the tables available in a database + rows, err := this.db.Query("SELECT table_name, create_time, update_time FROM information_schema.tables WHERE table_schema = ?", location.db) + if err != nil { + return nil, err + } + for rows.Next() { + var name string + var create string + var rcreate sql.RawBytes + var update string + var rupdate sql.RawBytes + + if err := rows.Scan(&name, &rcreate, &rupdate); err != nil { + return nil, err + } + create = string(rcreate) + update = string(rupdate) + + files = append(files, File{ + FName: name, + FType: "directory", + FTime: func() int64 { + var t time.Time + var err error + if create == "" && update == "" { + return 0 + } else if update == "" { + if t, err = time.Parse("2006-01-02 15:04:05", create); err != nil { + return 0 + } + return t.Unix() + } + if t, err = time.Parse("2006-01-02 15:04:05", update); err != nil { + return 0 + } + return t.Unix() + }(), + }) + } + return files, nil + } else if location.row == "" { // third level folder = a list of all the available rows within the selected table + sqlFields, err := FindQuerySelection(this.db, location) + if err != nil { + return nil, err + } + extractSingleName := func(s QuerySelection) string { + return s.Name + } + extractName := func(s []QuerySelection) []string { + t := make([]string, 0, len(s)) + for i := range s { + t = append(t, extractSingleName(s[i])) + } + return t + } + extractNamePlus := func(s []QuerySelection) []string { + t := make([]string, 0, len(s)) + for i := range s { + t = append(t, "IFNULL(" + extractSingleName(s[i]) + ", '')") + } + return t + } + + rows, err := this.db.Query(fmt.Sprintf( + "SELECT CONCAT(%s) as filename %sFROM %s.%s %s LIMIT 15000", + func() string { + q := strings.Join(extractNamePlus(sqlFields.Select), ", ' - ', ") + if len(sqlFields.Esthetics) != 0 { + q += ", ' - ', " + strings.Join(extractNamePlus(sqlFields.Esthetics), ", ' ', ") + } + return q + }(), + func() string{ + if extractSingleName(sqlFields.Date) != "" { + return ", " + extractSingleName(sqlFields.Date) + " as date " + } + return "" + }(), + location.db, + location.table, + func() string { + if len(sqlFields.Order) != 0 { + return "ORDER BY " + strings.Join(extractName(sqlFields.Order), ", ") + " DESC " + } + return "" + }(), + )); + if err != nil { + return nil, err + } + + for rows.Next() { + var name_raw sql.RawBytes + var date sql.RawBytes + if extractSingleName(sqlFields.Date) == "" { + if err := rows.Scan(&name_raw); err != nil { + return nil, err + } + } else { + if err := rows.Scan(&name_raw, &date); err != nil { + return nil, err + } + } + files = append(files, File{ + FName: string(name_raw)+".form", + FType: "file", + FSize: -1, + FTime: func() int64 { + t, err := time.Parse("2006-01-02", fmt.Sprintf("%s", date)) + if err != nil { + return 0 + } + return t.Unix() + }(), + }) + } + return files, nil + } + return nil, ErrNotValid +} + +func (this Mysql) Cat(path string) (io.ReadCloser, error) { + defer this.db.Close() + location, err := NewDBLocation(path) + if err != nil { + return nil, err + } else if location.db == "" || location.table == "" || location.row == "" { + return nil, ErrNotValid + } + + // STEP 1: Perform the database query + fields, err := FindQuerySelection(this.db, location) + if err != nil { + return nil, err + } + whereSQL, whereParams := sqlWhereClause(fields, location) + query := fmt.Sprintf( + "SELECT * FROM %s.%s WHERE %s", + location.db, + location.table, + whereSQL, + ) + + rows, err := this.db.Query(query, whereParams...) + if err != nil { + return nil, err + } + columnsName, err := rows.Columns() + if err != nil { + return nil, err + } + + // STEP 2: find potential foreign key on given results + // those will be shown as a list of possible choice + columnsChoice, err := FindForeignKeysChoices(this.db, location) + if err != nil { + return nil, err + } + + // STEP 3: Encode the result of the query into a form object + var forms []FormElement = []FormElement{} + + dummy := make([]interface{}, len(columnsName)) + columnPointers := make([]interface{}, len(columnsName)) + for i := range columnsName { + columnPointers[i] = &dummy[i] + } + for rows.Next() { + if err := rows.Scan(columnPointers...); err != nil { + return nil, err + } + break + } + for i := range columnsName { + if pval, ok := columnPointers[i].(*interface{}); ok { + if pval == nil { + continue + } + el := FormElement{ + Name: columnsName[i], + Type: "text", + } + + switch fields.All[columnsName[i]].Type { + case "int": + el.Value = fmt.Sprintf("%d", *pval) + el.Type = "number" + case "integer": + el.Value = fmt.Sprintf("%d", *pval) + el.Type = "number" + case "decimal": + el.Value = fmt.Sprintf("%d", *pval) + el.Type = "number" + case "dec": + el.Value = fmt.Sprintf("%d", *pval) + el.Type = "number" + case "float": + el.Value = fmt.Sprintf("%d", *pval) + el.Type = "number" + case "double": + el.Value = fmt.Sprintf("%d", *pval) + el.Type = "number" + case "tinyint": + el.Value = fmt.Sprintf("%d", *pval) + el.Type = "number" + case "smallint": + el.Value = fmt.Sprintf("%d", *pval) + el.Type = "number" + case "mediumint": + el.Value = fmt.Sprintf("%d", *pval) + el.Type = "number" + case "bigint": + el.Value = fmt.Sprintf("%d", *pval) + el.Type = "number" + case "enum": + el.Type = "select" + reg := regexp.MustCompile(`^'(.*)'$`) + el.Opts = func () []string{ + r := strings.Split(strings.TrimSuffix(strings.TrimPrefix(fields.All[columnsName[i]].RawType, "enum("), ")"), ",") + for i:=0; i 0 { + text := []string{} + for i:=0; i 200 { + return fields, NewError("This table doesn't have any defined keys.", 405) + } + fields.Select = queryCandidates + fields.Esthetics = make([]QuerySelection, 0) + } + } + + // STEP 4: organise our finding into a data structure that's usable + sortQuerySelection := func(s []QuerySelection) func(i, j int) bool { + calculateScore := func(q QuerySelection) int { + score := 0 + if q.Key == "UNI" { + score = 4 + } else if q.Key == "PRI" { + score = 5 + } else { + return 0 + } + if lowerName := strings.ToLower(q.Name); lowerName == "id" || lowerName == "gid" || lowerName == "uid" { + score -= 2 + } + if q.Type == "varchar" || q.Type == "char" { + score += 1 + } else if q.Type == "date" { + score -= 1 + } + return score + } + return func(i, j int) bool { + return calculateScore(s[i]) > calculateScore(s[j]) + } + } + sort.SliceStable(fields.Select, sortQuerySelection(fields.Select)) + sort.SliceStable(fields.Order, sortQuerySelection(fields.Order)) + fields.Date.Name = func() string { + if len(fields.Order) == 0 { + return fields.Date.Name + } + return fields.Order[0].Name + }() + fields.Esthetics = func() []QuerySelection{ // fields whose only value is to make our generated field look good + var size int = 0 + var i int + for i = range fields.Select { + size += fields.Select[i].Size + } + for i = range fields.Esthetics { + s := fields.Esthetics[i].Size + if size + s > 100 { + break + } + size += s + } + if i+1 > len(fields.Esthetics){ + return fields.Esthetics + } + return fields.Esthetics[:i+1] + }() + + return fields, nil +} + +func (this Mysql) Close() error { + return this.db.Close() +} + +func FindForeignKeysChoices(db *sql.DB, location DBLocation) (map[string][]string, error) { + choices := make(map[string][]string, 0) + rows, err := db.Query("SELECT column_name, referenced_table_name, referenced_column_name FROM information_schema.key_column_usage WHERE table_schema = ? AND table_name = ? AND referenced_column_name IS NOT NULL", location.db, location.table) + if err != nil { + return choices, err + } + for rows.Next() { + var column_name string + var referenced_table_schema string + var referenced_column_name string + if err := rows.Scan(&column_name, &referenced_table_schema, &referenced_column_name); err != nil { + return choices, err + } + r, err := db.Query(fmt.Sprintf("SELECT DISTINCT(%s) FROM %s.%s LIMIT 10000", column_name, location.db, location.table)) + if err != nil { + return choices, err + } + var res []string = make([]string, 0) + for r.Next() { + var value string + r.Scan(&value) + res = append(res, value) + } + choices[column_name] = res + } + return choices, nil +} + +func FindWhoIsUsing(db *sql.DB, location DBLocation) ([]DBLocation, error) { + locations := make([]DBLocation, 0) + rows, err := db.Query("SELECT table_schema, table_name, column_name FROM information_schema.key_column_usage WHERE referenced_table_schema = ? AND referenced_table_name = ? AND column_name IS NOT NULL", location.db, location.table) + if err != nil { + return locations, err + } + for rows.Next() { + var table_schema string + var table_name string + var column_name string + if err := rows.Scan(&table_schema, &table_name, &column_name); err != nil { + return locations, err + } + locations = append(locations, DBLocation{ + db: table_schema, + table: table_name, + row: column_name, + }) + } + return locations, nil +} + +func FindWhoOwns(db *sql.DB, location DBLocation) (DBLocation, error) { + var referenced_table_schema string + var referenced_table_name string + var referenced_column_name string + + if err := db.QueryRow( + fmt.Sprintf("SELECT referenced_table_schema, referenced_table_name, referenced_column_name FROM information_schema.key_column_usage WHERE table_schema = ? AND table_name = ? AND column_name = ? AND referenced_column_name IS NOT NULL"), + location.db, + location.table, + location.row, + ).Scan(&referenced_table_schema, &referenced_table_name, &referenced_column_name); err != nil { + return DBLocation{}, err + } + return DBLocation{ referenced_table_schema, referenced_table_name, referenced_column_name }, nil +} + +func FindHowManyOccurenceOfaValue(db *sql.DB, location DBLocation, value interface{}) int { + var count int + if err := db.QueryRow( + fmt.Sprintf("SELECT COUNT(*) FROM %s.%s WHERE %s = ?", location.db, location.table, location.row), + value, + ).Scan(&count); err != nil { + return 0 + } + return count +} + +func generateLink(chroot string, l DBLocation, value interface{}) string { + chrootLocation, err := NewDBLocation(chroot) + if err != nil { + return fmt.Sprintf("'%s'", l.table) + } + + if chrootLocation.db == "" { + return fmt.Sprintf( + "[%s](/files/%s/%s/%s)", + l.table, + l.db, + l.table, + func() string { + if l.row == "" { + return "" + } + return fmt.Sprintf("?q=%s%%3D%s", l.row, value) + }(), + ) + } else if chrootLocation.table == "" { + return fmt.Sprintf( + "[%s](/files/%s/%s)", + l.table, + l.table, + func() string { + if l.row == "" { + return "" + } + return fmt.Sprintf("?q=%s%%3D%s", l.row, value) + }(), + ) + } else { + return fmt.Sprintf("'%s'", l.table) + } +}