Skip to content

Commit

Permalink
fix UI and FTP importer
Browse files Browse the repository at this point in the history
  • Loading branch information
@poolpOrg
poolpOrg committed Nov 11, 2024
1 parent bca92be commit 0ab03ef
Show file tree
Hide file tree
Showing 7 changed files with 171 additions and 99 deletions.
1 change: 1 addition & 0 deletions cmd/plakar/plakar.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,7 @@ func entryPoint() int {
fmt.Fprintf(os.Stderr, "%s: %s\n", flag.CommandLine.Name(), err)
return retval
}
return retval
}

// special case, server skips passphrase as it only serves storage layer
Expand Down
24 changes: 19 additions & 5 deletions objects/objects.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,16 +163,30 @@ func (f FileInfo) Sys() any {
}

func FileInfoFromStat(stat os.FileInfo) FileInfo {
Ldev := uint64(0)
Lino := uint64(0)
Luid := uint64(0)
Lgid := uint64(0)
Lnlink := uint16(0)

if _, ok := stat.Sys().(*syscall.Stat_t); ok {
Ldev = uint64(stat.Sys().(*syscall.Stat_t).Dev)
Lino = uint64(stat.Sys().(*syscall.Stat_t).Ino)
Luid = uint64(stat.Sys().(*syscall.Stat_t).Uid)
Lgid = uint64(stat.Sys().(*syscall.Stat_t).Gid)
Lnlink = uint16(stat.Sys().(*syscall.Stat_t).Nlink)
}

return FileInfo{
Lname: stat.Name(),
Lsize: stat.Size(),
Lmode: stat.Mode(),
LmodTime: stat.ModTime(),
Ldev: uint64(stat.Sys().(*syscall.Stat_t).Dev),
Lino: uint64(stat.Sys().(*syscall.Stat_t).Ino),
Luid: uint64(stat.Sys().(*syscall.Stat_t).Uid),
Lgid: uint64(stat.Sys().(*syscall.Stat_t).Gid),
Lnlink: uint16(stat.Sys().(*syscall.Stat_t).Nlink),
Ldev: Ldev,
Lino: Lino,
Luid: Luid,
Lgid: Lgid,
Lnlink: Lnlink,
}
}

Expand Down
17 changes: 8 additions & 9 deletions snapshot/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -296,15 +296,15 @@ func (snap *Snapshot) updateImporterStatistics(record importer.ScanResult) {
}
}

func (snap *Snapshot) importerJob(sc *scanCache, scanDir string, options *PushOptions, maxConcurrency chan bool) (chan importer.ScanRecord, func(), error) {
imp, err := importer.NewImporter(scanDir)
if err != nil {
return nil, nil, err
}
func (snap *Snapshot) importerJob(imp *importer.Importer, sc *scanCache, scanDir string, options *PushOptions, maxConcurrency chan bool) (chan importer.ScanRecord, error) {
//imp, err := importer.NewImporter(scanDir)
//if err != nil {
// return nil, nil, err
// }

scanner, err := imp.Scan()
if err != nil {
return nil, nil, err
return nil, err
}

mu := sync.Mutex{}
Expand Down Expand Up @@ -359,7 +359,7 @@ func (snap *Snapshot) importerJob(sc *scanCache, scanDir string, options *PushOp
snap.statistics.ImporterDuration = time.Since(snap.statistics.ImporterStart)
}()

return filesChannel, func() { imp.Close() }, nil
return filesChannel, nil
}

func (snap *Snapshot) Backup(scanDir string, options *PushOptions) error {
Expand Down Expand Up @@ -404,11 +404,10 @@ func (snap *Snapshot) Backup(scanDir string, options *PushOptions) error {
maxConcurrency := make(chan bool, options.MaxConcurrency)

/* importer */
filesChannel, closeImporter, err := snap.importerJob(sc, scanDir, options, maxConcurrency)
filesChannel, err := snap.importerJob(imp, sc, scanDir, options, maxConcurrency)
if err != nil {
return err
}
defer closeImporter()

/* scanner */
scannerWg := sync.WaitGroup{}
Expand Down
200 changes: 129 additions & 71 deletions snapshot/importer/ftp/ftp.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@ import (
"log"
"net/url"
"os"
"path"
"path/filepath"
"strings"
"sync"
"time"

"github.com/PlakarKorp/plakar/objects"
Expand All @@ -49,18 +49,11 @@ func connectToFTP(host, username, password string) (*goftp.Client, error) {
Password: password,
Timeout: 10 * time.Second,
}
client, err := goftp.DialConfig(config, host)
if err != nil {
return nil, err
}
return client, nil
return goftp.DialConfig(config, host)
}

func NewFTPImporter(location string) (importer.ImporterBackend, error) {

//username := ""
//password := ""

parsed, err := url.Parse(location)
if err != nil {
return nil, err
Expand All @@ -80,90 +73,155 @@ func NewFTPImporter(location string) (importer.ImporterBackend, error) {
}, nil
}

func (p *FTPImporter) Scan() (<-chan importer.ScanResult, error) {
client, err := connectToFTP(p.host, "", "")
if err != nil {
fmt.Println(err)
return nil, err
}
p.client = client
c := make(chan importer.ScanResult)
func (p *FTPImporter) ftpWalker_worker(jobs <-chan string, results chan<- importer.ScanResult, wg *sync.WaitGroup) {
defer wg.Done()

go func() {
defer close(c)

dirInfo := objects.NewFileInfo(
"/", 0, 0700|os.ModeDir, time.Now(), 0, 0, 0, 0, 1,
)
c <- importer.ScanRecord{Pathname: "/", Stat: dirInfo}

err := p.createParentNodes(p.rootDir, c)
for path := range jobs {
info, err := p.client.Stat(path)
if err != nil {
log.Printf("Error creating parent nodes: %v", err)
results <- importer.ScanError{Pathname: path, Err: err}
continue
}

err = p.scanDirectory(p.rootDir, c)
if err != nil {
log.Printf("Error scanning FTP server: %v", err)
// Use fs.DirEntry.Type() to avoid another stat call when possible
var recordType importer.RecordType
switch mode := info.Mode(); {
case mode.IsRegular():
recordType = importer.RecordTypeFile
case mode.IsDir():
recordType = importer.RecordTypeDirectory
case mode&os.ModeSymlink != 0:
recordType = importer.RecordTypeSymlink
case mode&os.ModeDevice != 0:
recordType = importer.RecordTypeDevice
case mode&os.ModeNamedPipe != 0:
recordType = importer.RecordTypePipe
case mode&os.ModeSocket != 0:
recordType = importer.RecordTypeSocket
default:
recordType = importer.RecordTypeFile // Default to file if type is unknown
}
}()
fileinfo := objects.FileInfoFromStat(info)

if fileinfo.Mode().IsDir() {
entries, err := p.client.ReadDir(path)
if err != nil {
results <- importer.ScanError{Pathname: path, Err: err}
continue
}

return c, nil
var children []objects.FileInfo
prefix := p.rootDir
for _, child := range entries {
fullpath := filepath.Join(path, child.Name())
if !child.IsDir() {
if !strings.HasPrefix(fullpath, prefix) {
continue
}
} else {
if len(fullpath) < len(prefix) {
if !strings.HasPrefix(prefix, fullpath) {
continue
}
} else {
if !strings.HasPrefix(fullpath, prefix) {
continue
}
}
}
children = append(children, objects.FileInfoFromStat(child))
}
results <- importer.ScanRecord{Type: recordType, Pathname: filepath.ToSlash(path), Stat: fileinfo, Children: children}
} else {
results <- importer.ScanRecord{Type: recordType, Pathname: filepath.ToSlash(path), Stat: fileinfo}
}

// Handle symlinks separately
if fileinfo.Mode()&os.ModeSymlink != 0 {
originFile, err := os.Readlink(path)
if err != nil {
results <- importer.ScanError{Pathname: path, Err: err}
continue
}
results <- importer.ScanRecord{Type: recordType, Pathname: filepath.ToSlash(path), Target: originFile, Stat: fileinfo}
}
}
}

func (p *FTPImporter) createParentNodes(dirPath string, c chan importer.ScanResult) error {
// Split the root directory into individual parts and create each directory node
parts := strings.Split(dirPath, "/")
func (p *FTPImporter) ftpWalker_addPrefixDirectories(jobs chan<- string, results chan<- importer.ScanResult) {
directory := filepath.Clean(p.rootDir)
atoms := strings.Split(directory, string(os.PathSeparator))

for i := 0; i < len(atoms); i++ {
path := filepath.Join(atoms[0 : i+1]...)

// Reconstruct the path step-by-step, skipping empty parts
currentPath := "/"
for _, part := range parts {
if part == "" {
if !strings.HasPrefix(path, "/") {
path = "/" + path
}

if _, err := p.client.Stat(path); err != nil {
results <- importer.ScanError{Pathname: path, Err: err}
continue
}
currentPath = path.Join(currentPath, part)

// Create a directory node for the current path
dirInfo := objects.NewFileInfo(
part,
0,
0700|os.ModeDir,
time.Now(),
0,
0,
0,
0,
1,
)
c <- importer.ScanRecord{Type: importer.RecordTypeDirectory, Pathname: currentPath, Stat: dirInfo}
}

return nil
jobs <- path
}
}

func (p *FTPImporter) scanDirectory(dirPath string, c chan importer.ScanResult) error {
entries, err := p.client.ReadDir(dirPath)
func (p *FTPImporter) walkDir(root string, results chan<- string, wg *sync.WaitGroup) {
defer wg.Done()

entries, err := p.client.ReadDir(root)
if err != nil {
return err
log.Printf("Error reading directory %s: %v", root, err)
return
}

for _, entry := range entries {
entryPath := filepath.Join(dirPath, entry.Name())
fileInfo := objects.NewFileInfo(entry.Name(), entry.Size(), entry.Mode(), entry.ModTime(), 0, 0, 0, 0, 1)
entryPath := filepath.Join(root, entry.Name())

// Send the current entry to the results channel
results <- entryPath

// If the entry is a directory, traverse it recursively
if entry.IsDir() {
// Directory: Scan it recursively
c <- importer.ScanRecord{Type: importer.RecordTypeDirectory, Pathname: entryPath, Stat: fileInfo}
err := p.scanDirectory(entryPath, c)
if err != nil {
return err
}
} else {
// File: Send file information
c <- importer.ScanRecord{Type: importer.RecordTypeFile, Pathname: entryPath, Stat: fileInfo}
wg.Add(1)
go p.walkDir(entryPath, results, wg)
}
}
return nil
}

func (p *FTPImporter) Scan() (<-chan importer.ScanResult, error) {
client, err := connectToFTP(p.host, "", "")
if err != nil {
fmt.Println(err)
return nil, err
}
p.client = client

results := make(chan importer.ScanResult, 1000) // Larger buffer for results
jobs := make(chan string, 1000) // Buffered channel to feed paths to workers
var wg sync.WaitGroup
numWorkers := 256

for w := 1; w <= numWorkers; w++ {
wg.Add(1)
go p.ftpWalker_worker(jobs, results, &wg)
}

go func() {
defer close(jobs)
p.ftpWalker_addPrefixDirectories(jobs, results)
wg.Add(1)
p.walkDir(p.rootDir, jobs, &wg)
}()

go func() {
wg.Wait()
close(results)
}()

return results, nil
}

func (p *FTPImporter) NewReader(pathname string) (io.ReadCloser, error) {
Expand Down
14 changes: 7 additions & 7 deletions ui/v1/browse.tmpl
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{{template "base" .}}

{{define "title"}}{{.Snapshot.Header.IndexID | IDtoHex}}{{end}}
{{define "title"}}{{.Snapshot.Header.SnapshotID | IDtoHex}}{{end}}

{{define "main"}}
{{$Snapshot:=.Snapshot}}
Expand All @@ -14,14 +14,14 @@
{{$SymlinksResolve:=.SymlinksResolve}}
{{$Others:=.Others}}

<h1><a href="/snapshot/{{$Snapshot.Header.IndexID | IDtoHex}}:/">{{slice $Snapshot.Header.IndexID 0 4 | ShortIDtoHex}}</a></h1>
<h1><a href="/snapshot/{{$Snapshot.Header.SnapshotID | IDtoHex}}:/">{{slice $Snapshot.Header.SnapshotID 0 4 | ShortIDtoHex}}</a></h1>
<b>{{$Snapshot.Header.Username }}@{{$Snapshot.Header.Hostname }}</b>,
<i>{{$Snapshot.Header.CreationTime }}</i>

<nav style="--bs-breadcrumb-divider: '/';" aria-label="breadcrumb">
<ol class="breadcrumb">
{{range $atom := $Navigation }}
<li class="breadcrumb-item" aria-current="page"><a href="/snapshot/{{$Snapshot.Header.IndexID | IDtoHex}}:{{index $NavigationLinks $atom}}/">{{$atom}}</a></li>
<li class="breadcrumb-item" aria-current="page"><a href="/snapshot/{{$Snapshot.Header.SnapshotID | IDtoHex}}:{{index $NavigationLinks $atom}}/">{{$atom}}</a></li>
{{end}}
</ol>
</nav>
Expand All @@ -36,7 +36,7 @@
<tbody>
{{range $directory := .Scanned}}
<tr>
<td><a href="/snapshot/{{$Snapshot.Header.IndexID | IDtoHex}}:{{$directory}}/">{{$directory}}</a></td>
<td><a href="/snapshot/{{$Snapshot.Header.SnapshotID | IDtoHex}}:{{$directory}}/">{{$directory}}</a></td>
</tr>
{{end}}
</tbody>
Expand All @@ -62,7 +62,7 @@
<tbody>
{{range $fi := .Directories}}
<tr>
<td><a href="/snapshot/{{$Snapshot.Header.IndexID | IDtoHex}}:{{$Path}}/{{$fi.Name}}/">{{$fi.Name}}</a></td>
<td><a href="/snapshot/{{$Snapshot.Header.SnapshotID | IDtoHex}}:{{$Path}}/{{$fi.Name}}/">{{$fi.Name}}</a></td>
<td>{{.Mode }}</td>
<td>{{.Uid }}</td>
<td>{{.Gid }}</td>
Expand Down Expand Up @@ -91,13 +91,13 @@
<tbody>
{{range $fi := .Files}}
<tr>
<td><a href="/snapshot/{{$Snapshot.Header.IndexID | IDtoHex}}:{{$Path}}/{{$fi.Name}}">{{$fi.Name}}</a></td>
<td><a href="/snapshot/{{$Snapshot.Header.SnapshotID | IDtoHex}}:{{$Path}}/{{$fi.Name}}">{{$fi.Name}}</a></td>
<td>{{.Mode }}</td>
<td>{{.Uid }}</td>
<td>{{.Gid }}</td>
<td>{{.ModTime }}</td>
<td>{{.HumanSize }}</td>
<td><a href="/raw/{{$Snapshot.Header.IndexID | IDtoHex}}:{{$Path}}/{{$fi.Name}}?download=true">
<td><a href="/raw/{{$Snapshot.Header.SnapshotID | IDtoHex}}:{{$Path}}/{{$fi.Name}}?download=true">
<button type="button" class="btn btn-primary btn-sm">
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-download" viewBox="0 0 16 16">
<path d="M.5 9.9a.5.5 0 0 1 .5.5v2.5a1 1 0 0 0 1 1h12a1 1 0 0 0 1-1v-2.5a.5.5 0 0 1 1 0v2.5a2 2 0 0 1-2 2H2a2 2 0 0 1-2-2v-2.5a.5.5 0 0 1 .5-.5z"/>
Expand Down
Loading

0 comments on commit 0ab03ef

Please sign in to comment.