diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
index ab6aec0..1851afb 100644
--- a/.github/workflows/check.yml
+++ b/.github/workflows/check.yml
@@ -27,5 +27,5 @@ jobs:
with:
go-version: "1.22.*"
- run: sudo apt-get update && sudo apt-get install libvips-tools
- - run: curl https://rclone.org/install.sh | sudo bash
+ - run: curl https://rclone.org/install.sh | sudo bash -s beta
- run: make test
diff --git a/.golangci.yml b/.golangci.yml
index 2bf0908..c9acf32 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -22,7 +22,6 @@ linters:
- govet
- ineffassign
- misspell
- - nestif
- nilerr
- nilnil
- noctx
diff --git a/Dockerfile b/Dockerfile
index bc40efb..388cfaf 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -12,7 +12,9 @@ RUN make build && ./bin/rview --version
# Download rclone.
-FROM rclone/rclone:1.64 AS rclone-src
+# We use beta image because we need this fix - https://github.com/rclone/rclone/issues/7335.
+# TODO: update on next release.
+FROM rclone/rclone:beta AS rclone-src
RUN rclone --version
diff --git a/README.md b/README.md
index 4d7c873..f3c96ae 100644
--- a/README.md
+++ b/README.md
@@ -12,10 +12,11 @@
- [Limitations](#limitations)
- [Demo](#demo)
- [Run](#run)
+ - [Advanced](#advanced)
- [Configuration](#configuration)
- [Development](#development)
- - [API](#api)
- - [Metrics](#metrics)
+ - [API](#api)
+ - [Metrics](#metrics)
- [Thanks](#thanks)
## Features
@@ -38,6 +39,7 @@
significantly improve response time.
## Demo
+
Check out the live demo [here](https://rview.0x5f3759df.stream), credentials for Basic Auth: `rview:rview`.
## Run
@@ -78,26 +80,32 @@ Check out the live demo [here](https://rview.0x5f3759df.stream), credentials for
5. Go to http://localhost:8080.
+### Advanced
+
+You can run **Rview** with an existing Rclone instance and without access to the internet.
+Read more [here](./docs/Advanced%20Setup.md).
+
## Configuration
-| Flag | Default Value | Description |
-| -------------------------------- | -------------------------- | -------------------------------------------- |
-| `--debug-log-level` | `false` | display debug log messages |
-| `--dir` | `./var` | directory for app data (thumbnails and etc.) |
-| `--port` | `8080` | server port |
-| `--rclone-port` | `8181` | port of a rclone instance |
-| `--rclone-target` | no default value, required | rclone target |
-| `--read-static-files-from-disk` | `false` | read static files directly from disk |
-| `--thumbnails` | `true` | generate image thumbnails |
-| `--thumbnails-max-age-days` | `365` | max age of thumbnails, days |
-| `--thumbnails-max-total-size-mb` | `500` | max total size of thumbnails, MiB |
-| `--thumbnails-workers-count` | number of logical CPUs | number of workers for thumbnail generation |
+| Flag | Default | Description |
+| -------------------------------- | ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `--rclone-target` | none, **required** | Rclone target |
+| `--rclone-url` | none, optional | Url of an existing rclone instance. If url is not specified,
a local rclone instance will be launched with the default
config file. Url should include credentials for
Basic Auth, e.g., http://user:pass@rclone:80 |
+| `--debug-log-level` | `false` | Display debug log messages |
+| `--dir` | `./var` | Directory for app data |
+| `--port` | `8080` | Server port |
+| `--read-static-files-from-disk` | `false` | Read static files directly from disk |
+| `--thumbnails` | `true` | Generate image thumbnails |
+| `--thumbnails-max-age-days` | `365` | Max age of thumbnails, days |
+| `--thumbnails-max-total-size-mb` | `500` | Max total size of thumbnails, MiB |
+| `--thumbnails-workers-count` | number of logical CPUs | Number of workers for thumbnail generation |
+| `--version` | | Print version and exit |
## Development
First, you have to install the following dependencies:
-1. [rclone](https://github.com/rclone/rclone) - instructions can be found [here](https://rclone.org/install/).
+1. [Rclone](https://github.com/rclone/rclone) - instructions can be found [here](https://rclone.org/install/).
2. [libvips](https://github.com/libvips/libvips) - you can install it with this command:
```bash
@@ -141,3 +149,4 @@ Special thanks to these open-source projects:
- [Rclone](https://github.com/rclone/rclone) - rsync for cloud storage.
- [Material Icon Theme](https://github.com/PKief/vscode-material-icon-theme) - Material Design icons for VS Code.
- [Feather](https://github.com/feathericons/feather) - Simply beautiful open source icons.
+- [libvips](https://github.com/libvips/libvips) - A fast image processing library with low memory needs.
diff --git a/cmd/rview.go b/cmd/rview.go
index a34656a..aa66938 100644
--- a/cmd/rview.go
+++ b/cmd/rview.go
@@ -71,7 +71,7 @@ func (r *Rview) Prepare() (err error) {
}
// Rclone Instance
- r.rcloneInstance, err = rclone.NewRclone(r.cfg.RclonePort, r.cfg.RcloneTarget, r.cfg.RcloneDirCacheTime)
+ r.rcloneInstance, err = rclone.NewRclone(r.cfg.Rclone)
if err != nil {
return fmt.Errorf("couldn't prepare rclone: %w", err)
}
diff --git a/docs/Advanced Setup.md b/docs/Advanced Setup.md
new file mode 100644
index 0000000..9810547
--- /dev/null
+++ b/docs/Advanced Setup.md
@@ -0,0 +1,113 @@
+# Advanced Setup for Paranoiacs
+
+You don't have to trust **Rview** - it can work isolated from the internet with access only
+to the small subset of read-only Rclone commands. Unfortunately, Rclone itself doesn't support
+fine-grained permissions. So, we have to use other tools - for example, Nginx.
+
+First, create `entrypoint.sh` with the following content and replace all the variables.
+Don't forget to run `chmod +x entrypoint.sh`.
+
+```sh
+#!/bin/sh
+
+set -e
+
+# Start Nginx
+
+apk update && apk add nginx
+
+cat < /etc/nginx/http.d/rclone.conf
+server {
+ listen 8080;
+ listen [::]:8080;
+
+ # Serve dirs and files with no access to other remotes.
+ location /[]/ {
+ proxy_pass http://localhost:5572;
+ }
+ # This command is required to build the search index. Read more: https://rclone.org/rc/#operations-list
+ location /operations/list {
+ proxy_pass http://localhost:5572;
+ }
+ location / {
+ return 403;
+ }
+}
+EOT
+
+nginx
+
+# Start Rclone. Don't forget to change values for '--rc-user' and '--rc-pass'!
+
+rclone rcd \
+ --rc-addr localhost:5572 \
+ --rc-user user \
+ --rc-pass pass \
+ --rc-template /config/rclone/rclone.gotmpl \
+ --rc-serve
+```
+
+Second, create `docker-compose.yml`:
+
+```yaml
+version: "2"
+
+services:
+ rview:
+ image: ghcr.io/shoshinnikita/rview:main
+ container_name: rview
+ volumes:
+ - ./var:/srv/var # mount app data directory to cache image thumbnails
+ ports:
+ - "127.0.0.1:8080:8080"
+ networks:
+ - rview # connect rclone and rview
+ command: [
+ "--rclone-url", "http://user:pass@rclone:8080", # don't forget to change username and password
+ "--rclone-target", ""
+ ]
+
+ rclone:
+ # We use beta image because we need this fix - https://github.com/rclone/rclone/issues/7335.
+ image: rclone/rclone:beta
+ container_name: rclone
+ volumes:
+ - ./entrypoint.sh:/entrypoint.sh # mount the script you created in the first step
+ - ./rclone.gotmpl:/config/rclone/rclone.gotmpl # template can be found in 'static' dir
+ - ~/.config/rclone/rclone.conf:/config/rclone/rclone.conf:ro # mount your Rclone config file
+ networks:
+ - internet # rclone must have access to the internet
+ - rview # connect rclone and rview
+ entrypoint: "/entrypoint.sh"
+
+networks:
+ # Network with access to the internet.
+ internet:
+ # Internal network without access to the internet.
+ rview:
+ internal: true
+```
+
+Finally, you can run `docker compose up` and go to http://localhost:8080.
+
+Optionally, you can check that everything works as expected:
+
+1. Check `rclone` container
+ ```sh
+ docker exec -it rclone sh
+
+ ping 1.1.1.1 # you should see pings
+ exit
+ ```
+2. Check `rview` container
+ ```sh
+ docker exec -it rview sh
+
+ ping 1.1.1.1 # no pings
+ ping rclone # you should see pings
+ wget -O - -q "http://rclone:5572" # connection refused, no direct access to rclone
+ wget -O - -q "http://rclone:8080/[]/" # 401, auth is on
+ wget -O - -q "http://user:pass@rclone:8080/[]/" # 200, ok
+ wget -O - -q "http://user:pass@rclone:8080/[/bin]/" # 403, no access to other remotes
+ wget -O - -q --post-data "" "http://user:pass@rclone:8080/operations/list?fs=/bin&remote=" # 200, ok
+ ```
diff --git a/rclone/rclone.go b/rclone/rclone.go
index 2d6fe7b..749333d 100644
--- a/rclone/rclone.go
+++ b/rclone/rclone.go
@@ -4,6 +4,8 @@ import (
"bufio"
"bytes"
"context"
+ "crypto/rand"
+ "encoding/hex"
"encoding/json"
"errors"
"fmt"
@@ -32,62 +34,106 @@ type Rclone struct {
stoppedByShutdown atomic.Bool
stoppedCh chan struct{}
- httpClient *http.Client
+ httpClient *http.Client
+ // rcloneURL with username and password for basic auth.
rcloneURL *url.URL
rcloneTarget string
}
-func NewRclone(rclonePort int, rcloneTarget string, dirCacheTime time.Duration) (*Rclone, error) {
- // Check if rclone is installed.
- _, err := exec.LookPath("rclone")
- if err != nil {
- return nil, err
- }
+func NewRclone(cfg rview.RcloneConfig) (_ *Rclone, err error) {
+ var (
+ cmd *exec.Cmd
+ stopCmd func()
+ rcloneURL *url.URL
+ )
+ if cfg.URL != "" {
+ // Use an existing rclone instance.
+ stopCmd = func() {}
+ rcloneURL, err = url.Parse(cfg.URL)
+ if err != nil {
+ return nil, fmt.Errorf("couldn't parse rclone url %q: %w", cfg.URL, err)
+ }
- f, err := os.CreateTemp("", "rview-rclone-template-*")
- if err != nil {
- return nil, fmt.Errorf("couldn't create temp file for rclone template: %w", err)
- }
- _, err = f.WriteString(static.RcloneTemplate)
- if err != nil {
- return nil, fmt.Errorf("couldn't write rclone template file: %w", err)
- }
- if err := f.Close(); err != nil {
- return nil, fmt.Errorf("couldn't close rclone template file: %w", err)
- }
+ } else {
+ // Have to run a new rclone instance.
+ user := "rview"
+ pass, err := newRandomString(10)
+ if err != nil {
+ return nil, fmt.Errorf("couldn't generate password for rclone rc: %w", err)
+ }
- ctx, cancel := context.WithCancel(context.Background())
+ // Check if rclone is installed.
+ _, err = exec.LookPath("rclone")
+ if err != nil {
+ return nil, err
+ }
- //nolint:gosec
- return &Rclone{
- cmd: exec.CommandContext(ctx,
+ f, err := os.CreateTemp("", "rview-rclone-template-*")
+ if err != nil {
+ return nil, fmt.Errorf("couldn't create temp file for rclone template: %w", err)
+ }
+ _, err = f.WriteString(static.RcloneTemplate)
+ if err != nil {
+ return nil, fmt.Errorf("couldn't write rclone template file: %w", err)
+ }
+ if err := f.Close(); err != nil {
+ return nil, fmt.Errorf("couldn't close rclone template file: %w", err)
+ }
+
+ host := "localhost:" + strconv.Itoa(cfg.Port)
+
+ ctx, cancel := context.WithCancel(context.Background())
+
+ cmd = exec.CommandContext(ctx,
"rclone",
- "serve",
- "http",
- "--addr", ":"+strconv.Itoa(rclonePort),
- "--template", f.Name(),
- "--dir-cache-time", dirCacheTime.String(),
- rcloneTarget,
- ),
- stopCmd: cancel,
+ "rcd",
+ "--rc-user", user,
+ "--rc-pass", pass,
+ "--rc-serve",
+ "--rc-addr", host,
+ "--rc-template", f.Name(),
+ "--rc-web-gui-no-open-browser",
+ )
+ stopCmd = cancel
+ rcloneURL = &url.URL{
+ Scheme: "http",
+ Host: host,
+ User: url.UserPassword(user, pass),
+ }
+ }
+
+ return &Rclone{
+ cmd: cmd,
+ stopCmd: stopCmd,
stoppedCh: make(chan struct{}),
//
httpClient: &http.Client{
Timeout: 2 * time.Minute,
},
- rcloneURL: &url.URL{
- Scheme: "http",
- Host: "localhost:" + strconv.Itoa(rclonePort),
- },
- rcloneTarget: rcloneTarget,
+ rcloneURL: rcloneURL,
+ rcloneTarget: cfg.Target,
}, nil
}
+func newRandomString(size int) (string, error) {
+ data := make([]byte, size/2)
+ _, err := rand.Read(data)
+ if err != nil {
+ return "", err
+ }
+ return hex.EncodeToString(data), nil
+}
+
func (r *Rclone) Start() error {
defer func() {
close(r.stoppedCh)
}()
+ if r.cmd == nil {
+ // We use an existing rclone instance.
+ return nil
+ }
+
stdout, err := r.cmd.StdoutPipe()
if err != nil {
return fmt.Errorf("couldn't get rclone stdout: %w", err)
@@ -155,9 +201,9 @@ func (r *Rclone) Shutdown(ctx context.Context) error {
}
func (r *Rclone) GetFile(ctx context.Context, id rview.FileID) (io.ReadCloser, http.Header, error) {
- rcloneURL := r.rcloneURL.JoinPath(id.GetPath())
+ rcloneURL := r.rcloneURL.JoinPath("["+r.rcloneTarget+"]", id.GetPath())
- body, headers, err := r.makRequest(ctx, rcloneURL)
+ body, headers, err := r.makeRequest(ctx, "GET", rcloneURL, nil)
if err != nil {
return nil, nil, err
}
@@ -190,12 +236,12 @@ func (r *Rclone) GetDirInfo(ctx context.Context, path string, sort, order string
rlog.Debugf("rclone info for %q was loaded in %s", path, dur)
}()
- rcloneURL := r.rcloneURL.JoinPath(path)
+ rcloneURL := r.rcloneURL.JoinPath("["+r.rcloneTarget+"]", path)
rcloneURL.RawQuery = url.Values{
"sort": []string{sort},
"order": []string{order},
}.Encode()
- body, _, err := r.makRequest(ctx, rcloneURL)
+ body, _, err := r.makeRequest(ctx, "GET", rcloneURL, nil)
if err != nil {
return nil, err
}
@@ -216,14 +262,25 @@ func (r *Rclone) GetDirInfo(ctx context.Context, path string, sort, order string
rcloneInfo.Entries[i].URL = html.UnescapeString(rcloneInfo.Entries[i].URL)
}
+ // Rclone can't accurately report dir size. So, just reset it (and replicate behavior of 'rclone serve').
+ for i := range rcloneInfo.Entries {
+ if rcloneInfo.Entries[i].IsDir {
+ rcloneInfo.Entries[i].Size = 0
+ }
+ }
+
return &rcloneInfo, nil
}
-func (r *Rclone) makRequest(ctx context.Context, url *url.URL) (io.ReadCloser, http.Header, error) {
- req, err := http.NewRequestWithContext(ctx, "GET", url.String(), nil)
+func (r *Rclone) makeRequest(ctx context.Context, method string, url *url.URL, body io.Reader) (io.ReadCloser, http.Header, error) {
+ req, err := http.NewRequestWithContext(ctx, method, url.String(), body)
if err != nil {
return nil, nil, fmt.Errorf("couldn't prepare request: %w", err)
}
+ if body != nil {
+ req.Header.Set("Content-Type", "application/json")
+ }
+
resp, err := r.httpClient.Do(req)
if err != nil {
return nil, nil, fmt.Errorf("request failed: %w", err)
@@ -245,34 +302,40 @@ func (r *Rclone) makRequest(ctx context.Context, url *url.URL) (io.ReadCloser, h
return resp.Body, resp.Header, nil
}
-func (r *Rclone) GetAllFiles(ctx context.Context) (res []string, err error) {
- //nolint:gosec
- cmd := exec.CommandContext(ctx,
- "rclone",
- "lsf",
- "-R",
- r.rcloneTarget,
- )
-
- data, err := cmd.Output()
+func (r *Rclone) GetAllFiles(ctx context.Context) (dirs, files []string, err error) {
+ url := r.rcloneURL.JoinPath("operations/list")
+ reqBody, _ := json.Marshal(map[string]any{ // error is always nil
+ "fs": r.rcloneTarget,
+ "remote": "",
+ "opt": map[string]any{
+ "noModTime": true,
+ "noMimeType": true,
+ "recurse": true,
+ },
+ })
+ body, _, err := r.makeRequest(ctx, "POST", url, bytes.NewReader(reqBody))
if err != nil {
- var exitErr *exec.ExitError
- if errors.As(err, &exitErr) {
- stderr := string(exitErr.Stderr)
- if len(stderr) > 50 {
- stderr = stderr[:50] + "..."
- }
- err = fmt.Errorf("%s, stderr: %q", exitErr.ProcessState.String(), stderr)
- }
- return nil, fmt.Errorf("command error: %w", err)
+ return nil, nil, fmt.Errorf("request failed: %w", err)
}
+ defer body.Close()
- s := bufio.NewScanner(bytes.NewReader(data))
- for s.Scan() {
- res = append(res, s.Text())
+ var resp struct {
+ List []struct {
+ Path string
+ IsDir bool
+ } `json:"list"`
+ }
+ err = json.NewDecoder(body).Decode(&resp)
+ if err != nil {
+ return nil, nil, fmt.Errorf("couldn't decode rclone response: %w", err)
}
- if err := s.Err(); err != nil {
- return nil, fmt.Errorf("couldn't scan rclone output: %w", err)
+
+ for _, v := range resp.List {
+ if v.IsDir {
+ dirs = append(dirs, v.Path)
+ } else {
+ files = append(files, v.Path)
+ }
}
- return res, nil
+ return dirs, files, nil
}
diff --git a/rview/config.go b/rview/config.go
index 8cc070b..786994b 100644
--- a/rview/config.go
+++ b/rview/config.go
@@ -18,22 +18,17 @@ type Config struct {
ServerPort int
Dir string
- RcloneTarget string
- RclonePort int
-
Thumbnails bool
ThumbnailsMaxAgeInDays int
ThumbnailsMaxTotalSizeInMB int
ThumbnailsWorkersCount int
+ Rclone RcloneConfig
+
// Debug options
DebugLogLevel bool
ReadStaticFilesFromDisk bool
-
- // Internal
-
- RcloneDirCacheTime time.Duration
}
type BuildInfo struct {
@@ -41,6 +36,12 @@ type BuildInfo struct {
CommitTime string
}
+type RcloneConfig struct {
+ URL string
+ Target string
+ Port int
+}
+
type flagParams struct {
// p is a pointer to a value.
p any
@@ -51,49 +52,54 @@ type flagParams struct {
func (cfg *Config) getFlagParams() map[string]flagParams {
return map[string]flagParams{
"port": {
- p: &cfg.ServerPort, defaultValue: 8080, desc: "server port",
+ p: &cfg.ServerPort, defaultValue: 8080, desc: "Server port",
},
"dir": {
- p: &cfg.Dir, defaultValue: "./var", desc: "directory for app data",
+ p: &cfg.Dir, defaultValue: "./var", desc: "Directory for app data (thumbnails and etc.)",
},
//
- "rclone-port": {
- p: &cfg.RclonePort, defaultValue: 8181, desc: "port of a rclone instance",
+ "rclone-url": {
+ p: &cfg.Rclone.URL, defaultValue: "", desc: "" +
+ "Url of an existing rclone instance, optional. If url is not specified,\n" +
+ "a local rclone instance will be launched with the default config file.\n" +
+ "Url should include credentials for Basic Auth, e.g., http://user:pass@rclone:80",
},
"rclone-target": {
- p: &cfg.RcloneTarget, defaultValue: "", desc: "rclone target",
+ p: &cfg.Rclone.Target, defaultValue: "", desc: "Rclone target, required",
},
//
"thumbnails": {
- p: &cfg.Thumbnails, defaultValue: true, desc: "generate image thumbnails",
+ p: &cfg.Thumbnails, defaultValue: true, desc: "Generate image thumbnails",
},
"thumbnails-max-age-days": {
- p: &cfg.ThumbnailsMaxAgeInDays, defaultValue: 365, desc: "max age of thumbnails, days",
+ p: &cfg.ThumbnailsMaxAgeInDays, defaultValue: 365, desc: "Max age of thumbnails, days",
},
"thumbnails-max-total-size-mb": {
- p: &cfg.ThumbnailsMaxTotalSizeInMB, defaultValue: 500, desc: "max total size of thumbnails, MiB",
+ p: &cfg.ThumbnailsMaxTotalSizeInMB, defaultValue: 500, desc: "Max total size of thumbnails, MiB",
},
"thumbnails-workers-count": {
- p: &cfg.ThumbnailsWorkersCount, defaultValue: runtime.NumCPU(), desc: "number of workers for thumbnail generation",
+ p: &cfg.ThumbnailsWorkersCount, defaultValue: runtime.NumCPU(), desc: "Number of workers for thumbnail generation",
},
//
"debug-log-level": {
- p: &cfg.DebugLogLevel, defaultValue: false, desc: "display debug log messages",
+ p: &cfg.DebugLogLevel, defaultValue: false, desc: "Display debug log messages",
},
"read-static-files-from-disk": {
- p: &cfg.ReadStaticFilesFromDisk, defaultValue: false, desc: "read static files directly from disk",
+ p: &cfg.ReadStaticFilesFromDisk, defaultValue: false, desc: "Read static files directly from disk",
},
}
}
func ParseConfig() (Config, error) {
cfg := Config{
- BuildInfo: readBuildInfo(),
- RcloneDirCacheTime: time.Minute,
+ BuildInfo: readBuildInfo(),
+ Rclone: RcloneConfig{
+ Port: 8181,
+ },
}
var printVersion bool
- flag.BoolVar(&printVersion, "version", false, "print version and exit")
+ flag.BoolVar(&printVersion, "version", false, "Print version and exit")
flags := cfg.getFlagParams()
for name, params := range flags {
@@ -119,7 +125,7 @@ func ParseConfig() (Config, error) {
if cfg.ServerPort == 0 {
return cfg, errors.New("server port must be > 0")
}
- if cfg.RcloneTarget == "" {
+ if cfg.Rclone.Target == "" {
return cfg, errors.New("rclone target can't be empty")
}
if cfg.Dir == "" {
diff --git a/search/service.go b/search/service.go
index 371baa3..d0ff6d4 100644
--- a/search/service.go
+++ b/search/service.go
@@ -31,7 +31,7 @@ type Service struct {
}
type Rclone interface {
- GetAllFiles(ctx context.Context) ([]string, error)
+ GetAllFiles(ctx context.Context) (dirs, files []string, err error)
}
type builtIndexes struct {
@@ -78,10 +78,20 @@ func (s *Service) Start() (err error) {
rlog.Infof("couldn't load search indexes from cache, prepare new ones: %s", err)
- if err := s.RefreshIndexes(context.Background()); err != nil {
- return fmt.Errorf("couldn't prepare search indexes: %w", err)
+ // The first few requests can fail with error "connection refused" because
+ // rclone is still starting.
+ for i := 0; true; i++ {
+ err = s.RefreshIndexes(context.Background())
+ if err == nil {
+ return nil
+ }
+ if i == 5 {
+ return fmt.Errorf("couldn't prepare search indexes: %w", err)
+ }
+
+ time.Sleep(50 * time.Millisecond)
}
- return nil
+ panic("unreachable")
}
func (s *Service) loadIndexesFromCache() (res *builtIndexes, err error) {
@@ -194,20 +204,16 @@ func (s *Service) RefreshIndexes(ctx context.Context) (finalErr error) {
rlog.Infof("search indexes were successfully refreshed in %s, entries count: %d", dur, entriesCount)
}()
- allFilenames, err := s.rclone.GetAllFiles(ctx)
+ dirs, filenames, err := s.rclone.GetAllFiles(ctx)
if err != nil {
return fmt.Errorf("couldn't get all files from rclone: %w", err)
}
- entriesCount = len(allFilenames)
-
- var dirs, filenames []string
- for _, f := range allFilenames {
- if strings.HasSuffix(f, "/") {
- dirs = append(dirs, f)
- } else {
- filenames = append(filenames, f)
+ for i := range dirs {
+ if !strings.HasSuffix(dirs[i], "/") {
+ dirs[i] += "/"
}
}
+ entriesCount = len(dirs) + len(filenames)
indexes := &builtIndexes{
Dirs: newPrefixIndex(dirs, s.minPrefixLen, s.maxPrefixLen),
diff --git a/search/service_test.go b/search/service_test.go
index e7f149c..09e4c0f 100644
--- a/search/service_test.go
+++ b/search/service_test.go
@@ -14,11 +14,12 @@ func TestService_RefreshIndexes(t *testing.T) {
ctx := context.Background()
rclone := &rcloneStub{
- GetAllFilesFn: func(context.Context) ([]string, error) {
- return []string{
+ GetAllFilesFn: func(context.Context) (dirs, files []string, err error) {
+ files = []string{
"hello world.go",
"arts/games/1.jpeg",
- }, nil
+ }
+ return dirs, files, nil
},
}
s := NewService(rclone, cache.NewInMemoryCache())
@@ -34,11 +35,12 @@ func TestService_RefreshIndexes(t *testing.T) {
r.Empty(dirs)
r.NotEmpty(files)
- rclone.GetAllFilesFn = func(context.Context) ([]string, error) {
- return []string{
+ rclone.GetAllFilesFn = func(context.Context) (dirs, files []string, err error) {
+ files = []string{
"hello world.go",
"qwerty.txt",
- }, nil
+ }
+ return dirs, files, nil
}
err = s.RefreshIndexes(ctx)
@@ -51,10 +53,10 @@ func TestService_RefreshIndexes(t *testing.T) {
}
type rcloneStub struct {
- GetAllFilesFn func(context.Context) ([]string, error)
+ GetAllFilesFn func(context.Context) (dirs, files []string, err error)
}
-func (s rcloneStub) GetAllFiles(ctx context.Context) ([]string, error) {
+func (s rcloneStub) GetAllFiles(ctx context.Context) (dirs, files []string, err error) {
return s.GetAllFilesFn(ctx)
}
@@ -106,7 +108,7 @@ func ExampleSearch() {
}
rclone := &rcloneStub{
- GetAllFilesFn: func(context.Context) ([]string, error) { return files, nil },
+ GetAllFilesFn: func(context.Context) (_, _ []string, err error) { return nil, files, nil },
}
s := NewService(rclone, cache.NewInMemoryCache())
assertNoError(s.Start())
diff --git a/tests/api_test.go b/tests/api_test.go
index 9a61c20..2a7c731 100644
--- a/tests/api_test.go
+++ b/tests/api_test.go
@@ -47,23 +47,26 @@ func startTestRview() {
ServerPort: mustGetFreePort(),
Dir: tempDir,
//
- RcloneTarget: "./testdata",
- RclonePort: mustGetFreePort(),
+ Rclone: rview.RcloneConfig{
+ Target: "./testdata",
+ Port: mustGetFreePort(),
+ },
//
Thumbnails: true,
ThumbnailsWorkersCount: 1,
//
DebugLogLevel: true,
- //
- RcloneDirCacheTime: 0,
}
rviewAPIAddr = fmt.Sprintf("http://localhost:%d", cfg.ServerPort)
- rcloneAddr := fmt.Sprintf("http://localhost:%d", cfg.RclonePort)
+ rcloneAddr := fmt.Sprintf("http://localhost:%d", cfg.Rclone.Port)
testRview = cmd.NewRview(cfg)
if err := testRview.Prepare(); err != nil {
panic(fmt.Errorf("couldn't prepare rview: %w", err))
}
+
+ time.Sleep(100 * time.Millisecond)
+
testRviewDone = testRview.Start(func() {
panic(fmt.Errorf("rview error"))
})