replacing gcloud command with storage client to get GCS object size (#…

…2532) * refactored code * fixed lint * replacing gcloud command with storage client * correcting gcsobjectpath for storage client * reducing code repetition --------- Co-authored-by: Vipin Yadav <[email protected]>
GoogleCloudPlatform · Sep 25, 2024 · fc7f74d · fc7f74d
1 parent 35ce836
commit fc7f74d
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 20 deletions.
diff --git a/tools/integration_tests/gzip/gzip_test.go b/tools/integration_tests/gzip/gzip_test.go
@@ -57,9 +57,10 @@ const (
 var (
 	gcsObjectsToBeDeletedEventually []string
 	storageClient                   *storage.Client
+	ctx                             context.Context
 )
 
-func setup_testdata(m *testing.M, ctx context.Context) error {
+func setup_testdata(m *testing.M) error {
 	fmds := []struct {
 		filename                    string
 		filesize                    int
@@ -168,7 +169,7 @@ func setup_testdata(m *testing.M, ctx context.Context) error {
 	return nil
 }
 
-func destroy_testdata(m *testing.M, ctx context.Context, storageClient *storage.Client) error {
+func destroy_testdata(m *testing.M, storageClient *storage.Client) error {
 	for _, gcsObjectPath := range gcsObjectsToBeDeletedEventually {
 		err := client.DeleteObjectOnGCS(ctx, storageClient, gcsObjectPath)
 		if err != nil {
@@ -194,7 +195,7 @@ func TestMain(m *testing.M) {
 	setup.ParseSetUpFlags()
 
 	var err error
-	ctx := context.Background()
+	ctx = context.Background()
 	if storageClient, err = client.CreateStorageClient(ctx); err != nil {
 		log.Fatalf("Error creating storage client: %v\n", err)
 	}
@@ -218,13 +219,13 @@ func TestMain(m *testing.M) {
 		log.Fatal("Please pass the name of bucket mounted at mountedDirectory to --testBucket flag.")
 	}
 
-	err = setup_testdata(m, ctx)
+	err = setup_testdata(m)
 	if err != nil {
 		log.Fatalf("Failed to setup test data: %v", err)
 	}
 
 	defer func() {
-		err := destroy_testdata(m, ctx, storageClient)
+		err := destroy_testdata(m, storageClient)
 		if err != nil {
 			log.Printf("Failed to destoy gzip test data: %v", err)
 		}

diff --git a/tools/integration_tests/gzip/read_gzip_test.go b/tools/integration_tests/gzip/read_gzip_test.go
@@ -17,15 +17,14 @@ package gzip_test
 
 import (
 	"bytes"
-	"context"
 	"fmt"
 	"io"
 	"io/fs"
 	"os"
 	"path"
 	"testing"
 
-	client2 "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client"
+	"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client"
 	"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations"
 	"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup"
 )
@@ -37,8 +36,8 @@ import (
 // GCS object.
 func verifyFileSizeAndFullFileRead(t *testing.T, filename string) {
 	mountedFilePath := path.Join(setup.MntDir(), TestBucketPrefixPath, filename)
-	gcsObjectPath := path.Join(setup.TestBucket(), TestBucketPrefixPath, filename)
-	gcsObjectSize, err := operations.GetGcsObjectSize(gcsObjectPath)
+	gcsObjectPath := path.Join(TestBucketPrefixPath, filename)
+	gcsObjectSize, err := client.GetGcsObjectSize(ctx, storageClient, gcsObjectPath)
 	if err != nil {
 		t.Fatalf("Failed to get size of gcs object %s: %v\n", gcsObjectPath, err)
 	}
@@ -70,9 +69,9 @@ func verifyFileSizeAndFullFileRead(t *testing.T, filename string) {
 // and its ranged read returns the same size as the requested read size.
 func verifyRangedRead(t *testing.T, filename string) {
 	mountedFilePath := path.Join(setup.MntDir(), TestBucketPrefixPath, filename)
+	gcsObjectPath := path.Join(TestBucketPrefixPath, filename)
+	gcsObjectSize, err := client.GetGcsObjectSize(ctx, storageClient, gcsObjectPath)
 
-	gcsObjectPath := path.Join(setup.TestBucket(), TestBucketPrefixPath, filename)
-	gcsObjectSize, err := operations.GetGcsObjectSize(gcsObjectPath)
 	if err != nil {
 		t.Fatalf("Failed to get size of gcs object %s: %v\n", gcsObjectPath, err)
 	}
@@ -119,11 +118,10 @@ func verifyRangedRead(t *testing.T, filename string) {
 // possible as they both always read back objects with content-encoding: gzip as
 // uncompressed/decompressed irrespective of any argument passed.
 func downloadGzipGcsObjectAsCompressed(t *testing.T, bucketName, objPathInBucket string) (tempfile string, err error) {
-	gcsObjectPath := path.Join(setup.TestBucket(), objPathInBucket)
-	gcsObjectSize, err := operations.GetGcsObjectSize(gcsObjectPath)
+	gcsObjectSize, err := client.GetGcsObjectSize(ctx, storageClient, objPathInBucket)
 
 	if err != nil {
-		return "", fmt.Errorf("failed to get size of gcs object %s: %w", gcsObjectPath, err)
+		return "", fmt.Errorf("failed to get size of gcs object %s: %w", objPathInBucket, err)
 	}
 
 	content, err := createContentOfSize(1)
@@ -142,8 +140,7 @@ func downloadGzipGcsObjectAsCompressed(t *testing.T, bucketName, objPathInBucket
 		}
 	}()
 
-	ctx := context.Background()
-	client, err := client2.CreateStorageClient(ctx)
+	client, err := client.CreateStorageClient(ctx)
 	if err != nil || client == nil {
 		return "", fmt.Errorf("failed to create storage client: %w", err)
 	}
@@ -172,7 +169,7 @@ func downloadGzipGcsObjectAsCompressed(t *testing.T, bucketName, objPathInBucket
 	defer r.Close()
 
 	gcsObjectData, err := io.ReadAll(r)
-	if len(gcsObjectData) < gcsObjectSize || err != nil {
+	if len(gcsObjectData) < int(gcsObjectSize) || err != nil {
 		return "", fmt.Errorf("failed to read object %s from bucket %s (expected read-size: %d, actual read-size: %d): %w", objPathInBucket, bktName, gcsObjectSize, len(gcsObjectData), err)
 	}
 

diff --git a/tools/integration_tests/gzip/write_gzip_test.go b/tools/integration_tests/gzip/write_gzip_test.go
@@ -19,6 +19,7 @@ import (
 	"path"
 	"testing"
 
+	"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client"
 	"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations"
 	"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup"
 )
@@ -33,8 +34,8 @@ const overwrittenFileSize = 1000
 // GCS object.
 func verifyFullFileOverwrite(t *testing.T, filename string) {
 	mountedFilePath := path.Join(setup.MntDir(), TestBucketPrefixPath, filename)
-	gcsObjectPath := path.Join(setup.TestBucket(), TestBucketPrefixPath, filename)
-	gcsObjectSize, err := operations.GetGcsObjectSize(gcsObjectPath)
+	gcsObjectPath := path.Join(TestBucketPrefixPath, filename)
+	gcsObjectSize, err := client.GetGcsObjectSize(ctx, storageClient, gcsObjectPath)
 	if err != nil {
 		t.Fatalf("Failed to get size of gcs object %s: %v\n", gcsObjectPath, err)
 	}
@@ -70,7 +71,7 @@ func verifyFullFileOverwrite(t *testing.T, filename string) {
 		t.Fatalf("Failed to copy/overwrite temp file %s to existing gzip object/file %s: %v", tempfile, mountedFilePath, err)
 	}
 
-	gcsObjectSize, err = operations.GetGcsObjectSize(gcsObjectPath)
+	gcsObjectSize, err = client.GetGcsObjectSize(ctx, storageClient, gcsObjectPath)
 	if err != nil {
 		t.Fatalf("Failed to get size of gcs object %s: %v\n", gcsObjectPath, err)
 	}

diff --git a/tools/integration_tests/util/client/storage_client.go b/tools/integration_tests/util/client/storage_client.go
@@ -267,3 +267,12 @@ func UploadGcsObject(ctx context.Context, client *storage.Client, localPath, buc
 	}
 	return nil
 }
+
+// Get the object size of the GCS object.
+func GetGcsObjectSize(ctx context.Context, client *storage.Client, object string) (int64, error) {
+	attrs, err := StatObject(ctx, client, object)
+	if err != nil {
+		return -1, err
+	}
+	return attrs.Size, nil
+}