Implement Hubic storage system - fixes rclone#200

Author: ncw

docs/content/about.md

@@ -20,6 +20,7 @@ Rclone is a command line program to sync files and directories to and from
   * Google Cloud Storage
   * Amazon Cloud Drive
   * Microsoft One Drive
+  * Hubic
   * The local filesystem
 
 Features

docs/content/hubic.md

@@ -0,0 +1,98 @@
+---
+title: "Hubic"
+description: "Rclone docs for Hubic"
+date: "2015-11-08"
+---
+
+<i class="fa fa-space-shuttle"></i> Hubic
+-----------------------------------------
+
+Paths are specified as `remote:path`
+
+Paths are specified as `remote:container` (or `remote:` for the `lsd`
+command.)  You may put subdirectories in too, eg `remote:container/path/to/dir`.
+
+The initial setup for Hubic involves getting a token from Hubic which
+you need to do in your browser.  `rclone config` walks you through it.
+
+Here is an example of how to make a remote called `remote`.  First run:
+
+     rclone config
+
+This will guide you through an interactive setup process:
+
+```
+n) New remote
+d) Delete remote
+q) Quit config
+e/n/d/q> n
+name> remote
+What type of source is it?
+Choose a number from below
+ 1) amazon cloud drive
+ 2) drive
+ 3) dropbox
+ 4) google cloud storage
+ 5) local
+ 6) onedrive
+ 7) hubic
+ 8) s3
+ 9) swift
+type> 7
+Hubic App Client Id - leave blank normally.
+client_id> 
+Hubic App Client Secret - leave blank normally.
+client_secret> 
+Remote config
+If your browser doesn't open automatically go to the following link: http://localhost:53682/auth
+Log in and authorize rclone for access
+Waiting for code...
+Got code
+--------------------
+[remote]
+client_id = 
+client_secret = 
+token = {"access_token":"XXXXXX"}
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+```
+
+Note that rclone runs a webserver on your local machine to collect the
+token as returned from Hubic. This only runs from the moment it opens
+your browser to the moment you get back the verification code.  This
+is on `http://127.0.0.1:53682/` and this it may require you to unblock
+it temporarily if you are running a host firewall.
+
+Once configured you can then use `rclone` like this,
+
+List containers in the top level of your Hubic
+
+    rclone lsd remote:
+
+List all the files in your Hubic
+
+    rclone ls remote:
+
+To copy a local directory to an Hubic directory called backup
+
+    rclone copy /home/source remote:backup
+
+### Modified time ###
+
+The modified time is stored as metadata on the object as
+`X-Object-Meta-Mtime` as floating point since the epoch accurate to 1
+ns.
+
+This is a defacto standard (used in the official python-swiftclient
+amongst others) for storing the modification time for an object.
+
+Note that Hubic wraps the Swift backend, so most of the properties of
+are the same.
+
+### Limitations ###
+
+Code to refresh the OpenStack token isn't done yet which may cause
+problems with very long transfers.

docs/content/overview.md

@@ -24,6 +24,7 @@ Here is an overview of the major features of each cloud storage system.
 | Google Cloud Storage   | Yes     | Yes     | No             | No              |
 | Amazon Cloud Drive     | Yes     | No      | Yes            | No              |
 | Microsoft One Drive    | No      | Yes     | Yes            | No              |
+| Hubic                  | Yes     | Yes     | No             | No              |
 | The local filesystem   | Yes     | Yes     | Depends        | No              |
 
 ### MD5SUM ###

docs/layouts/chrome/navbar.html

@@ -38,6 +38,7 @@
                     <li><a href="/googlecloudstorage/"><i class="fa fa-google"></i> Google Cloud Storage</a></li>
                     <li><a href="/amazonclouddrive/"><i class="fa fa-amazon"></i> Amazon Cloud Drive</a></li>
                     <li><a href="/onedrive/"><i class="fa fa-windows"></i> Microsoft One Drive</a></li>
+                    <li><a href="/hubic/"><i class="fa fa-space-shuttle"></i> Hubic</a></li>
                     <li><a href="/local/"><i class="fa fa-file"></i> Local</a></li>
                   </ul>
                 </li>

fs/operations_test.go

@@ -24,6 +24,7 @@ import (
 	_ "github.com/ncw/rclone/drive"
 	_ "github.com/ncw/rclone/dropbox"
 	_ "github.com/ncw/rclone/googlecloudstorage"
+	_ "github.com/ncw/rclone/hubic"
 	_ "github.com/ncw/rclone/local"
 	_ "github.com/ncw/rclone/onedrive"
 	_ "github.com/ncw/rclone/s3"

fs/test_all.sh

@@ -10,6 +10,7 @@ TestGoogleCloudStorage:
 TestDropbox:
 TestAmazonCloudDrive:
 TestOneDrive:
+TestHubic:
 "
 
 function test_remote {

fstest/fstests/fstests.go

@@ -435,7 +435,14 @@ func TestObjectString(t *testing.T) {
 func TestObjectFs(t *testing.T) {
 	skipIfNotOk(t)
 	obj := findObject(t, file1.Path)
-	if obj.Fs() != remote {
+	equal := obj.Fs() == remote
+	if !equal {
+		// Check to see if this wraps something else
+		if unwrap, ok := remote.(fs.UnWrapper); ok {
+			equal = obj.Fs() == unwrap.UnWrap()
+		}
+	}
+	if !equal {
 		t.Errorf("Fs is wrong %v != %v", obj.Fs(), remote)
 	}
 }
@@ -558,7 +565,13 @@ func TestLimitedFs(t *testing.T) {
 	fstest.CheckListing(t, fileRemote, []fstest.Item{file2Copy})
 	_, ok := fileRemote.(*fs.Limited)
 	if !ok {
-		t.Errorf("%v is not a fs.Limited", fileRemote)
+		// Check to see if this wraps a Limited FS
+		if unwrap, hasUnWrap := fileRemote.(fs.UnWrapper); hasUnWrap {
+			_, ok = unwrap.UnWrap().(*fs.Limited)
+		}
+		if !ok {
+			t.Errorf("%v is not a fs.Limited", fileRemote)
+		}
 	}
 }
 

fstest/fstests/gen_tests.go

@@ -132,5 +132,6 @@ func main() {
 	generateTestProgram(t, fns, "Dropbox")
 	generateTestProgram(t, fns, "AmazonCloudDrive")
 	generateTestProgram(t, fns, "OneDrive")
+	generateTestProgram(t, fns, "Hubic")
 	log.Printf("Done")
 }

hubic/auth.go

@@ -0,0 +1,54 @@
+package hubic
+
+import (
+	"net/http"
+
+	"github.com/ncw/swift"
+)
+
+// auth is an authenticator for swift
+type auth struct {
+	f *Fs
+}
+
+// newAuth creates a swift authenticator
+func newAuth(f *Fs) *auth {
+	return &auth{
+		f: f,
+	}
+}
+
+// Request constructs a http.Request for authentication
+//
+// returns nil for not needed
+func (a *auth) Request(*swift.Connection) (*http.Request, error) {
+	err := a.f.getCredentials()
+	if err != nil {
+		return nil, err
+	}
+	return nil, nil
+}
+
+// Response parses the result of an http request
+func (a *auth) Response(resp *http.Response) error {
+	return nil
+}
+
+// The public storage URL - set Internal to true to read
+// internal/service net URL
+func (a *auth) StorageUrl(Internal bool) string {
+	return a.f.credentials.Endpoint
+}
+
+// The access token
+func (a *auth) Token() string {
+	return a.f.credentials.Token
+}
+
+// The CDN url if available
+func (a *auth) CdnUrl() string {
+	return ""
+}
+
+// Check the interfaces are satisfied
+var _ swift.Authenticator = (*auth)(nil)