diff --git a/tools/integration_tests/explicit_dir/explicit_dir_test.go b/tools/integration_tests/explicit_dir/explicit_dir_test.go index 85b0b729ac..24891ebafa 100644 --- a/tools/integration_tests/explicit_dir/explicit_dir_test.go +++ b/tools/integration_tests/explicit_dir/explicit_dir_test.go @@ -20,6 +20,7 @@ import ( "log" "os" "testing" + "time" "cloud.google.com/go/storage" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" @@ -29,6 +30,11 @@ import ( const DirForExplicitDirTests = "dirForExplicitDirTests" +var ( + ctx context.Context + storageClient *storage.Client +) + func TestMain(m *testing.M) { setup.ParseSetUpFlags() var storageClient *storage.Client @@ -49,6 +55,15 @@ func TestMain(m *testing.M) { return } + // Create storage client before running tests. + ctx = context.Background() + closeStorageClient := client.CreateStorageClientWithTimeOut(&ctx, &storageClient, time.Minute*15) + defer func() { + err := closeStorageClient() + if err != nil { + log.Fatalf("closeStorageClient failed: %v", err) + } + }() flags := [][]string{{"--implicit-dirs=false"}} if !testing.Short() { diff --git a/tools/integration_tests/explicit_dir/list_test.go b/tools/integration_tests/explicit_dir/list_test.go index 4421ef1ba4..31d843c6fb 100644 --- a/tools/integration_tests/explicit_dir/list_test.go +++ b/tools/integration_tests/explicit_dir/list_test.go @@ -38,7 +38,7 @@ func TestListOnlyExplicitObjectsFromBucket(t *testing.T) { // testBucket/dirForExplicitDirTests/explicitDirectory/fileInExplicitDir1 -- File // testBucket/dirForExplicitDirTests/explicitDirectory/fileInExplicitDir2 -- File - implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(DirForExplicitDirTests) + implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(ctx, DirForExplicitDirTests, storageClient, t) implicit_and_explicit_dir_setup.CreateExplicitDirectoryStructure(DirForExplicitDirTests, t) err := filepath.WalkDir(testDir, func(path string, dir fs.DirEntry, err error) error { diff --git a/tools/integration_tests/implicit_dir/delete_test.go b/tools/integration_tests/implicit_dir/delete_test.go index d403fdfedf..c1f078425b 100644 --- a/tools/integration_tests/implicit_dir/delete_test.go +++ b/tools/integration_tests/implicit_dir/delete_test.go @@ -31,7 +31,7 @@ import ( func TestDeleteNonEmptyImplicitDir(t *testing.T) { testDirName := "testDeleteNonEmptyImplicitDir" testDirPath := setupTestDir(testDirName) - implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(path.Join(DirForImplicitDirTests, testDirName)) + implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(ctx, path.Join(DirForImplicitDirTests, testDirName), storageClient, t) dirPath := path.Join(testDirPath, implicit_and_explicit_dir_setup.ImplicitDirectory) @@ -46,7 +46,7 @@ func TestDeleteNonEmptyImplicitDir(t *testing.T) { func TestDeleteNonEmptyImplicitSubDir(t *testing.T) { testDirName := "testDeleteNonEmptyImplicitSubDir" testDirPath := setupTestDir(testDirName) - implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(path.Join(DirForImplicitDirTests, testDirName)) + implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(ctx, path.Join(DirForImplicitDirTests, testDirName), storageClient, t) subDirPath := path.Join(testDirPath, implicit_and_explicit_dir_setup.ImplicitDirectory, implicit_and_explicit_dir_setup.ImplicitSubDirectory) @@ -63,7 +63,7 @@ func TestDeleteNonEmptyImplicitSubDir(t *testing.T) { func TestDeleteImplicitDirWithExplicitSubDir(t *testing.T) { testDirName := "testDeleteImplicitDirWithExplicitSubDir" testDirPath := setupTestDir(testDirName) - implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(path.Join(DirForImplicitDirTests, testDirName)) + implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(ctx, path.Join(DirForImplicitDirTests, testDirName), storageClient, t) explicitDirPath := path.Join(testDirPath, implicit_and_explicit_dir_setup.ImplicitDirectory, ExplicitDirInImplicitDir) @@ -84,7 +84,7 @@ func TestDeleteImplicitDirWithExplicitSubDir(t *testing.T) { func TestDeleteImplicitDirWithImplicitSubDirContainingExplicitDir(t *testing.T) { testDirName := "testDeleteImplicitDirWithImplicitSubDirContainingExplicitDir" testDirPath := setupTestDir(testDirName) - implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(path.Join(DirForImplicitDirTests, testDirName)) + implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(ctx, path.Join(DirForImplicitDirTests, testDirName), storageClient, t) explicitDirPath := path.Join(testDirPath, implicit_and_explicit_dir_setup.ImplicitDirectory, implicit_and_explicit_dir_setup.ImplicitSubDirectory, ExplicitDirInImplicitSubDir) operations.CreateDirectoryWithNFiles(NumberOfFilesInExplicitDirInImplicitSubDir, explicitDirPath, PrefixFileInExplicitDirInImplicitSubDir, t) @@ -106,7 +106,7 @@ func TestDeleteImplicitDirWithImplicitSubDirContainingExplicitDir(t *testing.T) func TestDeleteImplicitDirInExplicitDir(t *testing.T) { testDirName := "testDeleteImplicitDirInExplicitDir" testDirPath := setupTestDir(testDirName) - implicit_and_explicit_dir_setup.CreateImplicitDirectoryInExplicitDirectoryStructure(path.Join(DirForImplicitDirTests, testDirName), t) + implicit_and_explicit_dir_setup.CreateImplicitDirectoryInExplicitDirectoryStructure(ctx, path.Join(DirForImplicitDirTests, testDirName), storageClient, t) dirPath := path.Join(testDirPath, implicit_and_explicit_dir_setup.ExplicitDirectory, implicit_and_explicit_dir_setup.ImplicitDirectory) @@ -125,7 +125,7 @@ func TestDeleteImplicitDirInExplicitDir(t *testing.T) { func TestDeleteExplicitDirContainingImplicitSubDir(t *testing.T) { testDirName := "testDeleteExplicitDirContainingImplicitSubDir" testDirPath := setupTestDir(testDirName) - implicit_and_explicit_dir_setup.CreateImplicitDirectoryInExplicitDirectoryStructure(path.Join(DirForImplicitDirTests, testDirName), t) + implicit_and_explicit_dir_setup.CreateImplicitDirectoryInExplicitDirectoryStructure(ctx, path.Join(DirForImplicitDirTests, testDirName), storageClient, t) dirPath := path.Join(testDirPath, implicit_and_explicit_dir_setup.ExplicitDirectory) diff --git a/tools/integration_tests/implicit_dir/list_test.go b/tools/integration_tests/implicit_dir/list_test.go index a9addd94fe..5abf972627 100644 --- a/tools/integration_tests/implicit_dir/list_test.go +++ b/tools/integration_tests/implicit_dir/list_test.go @@ -38,9 +38,12 @@ func TestListImplicitObjectsFromBucket(t *testing.T) { // testBucket/dirForImplicitDirTests/testDir/explicitFile -- File // testBucket/dirForImplicitDirTests/testDir/explicitDirectory/fileInExplicitDir1 -- File // testBucket/dirForImplicitDirTests/testDir/explicitDirectory/fileInExplicitDir2 -- File + // testBucket/dirForImplicitDirTests/testDir/implicitDirectory//fileInUnsupportedImplicitDir1 -- File + // testBucket//fileInUnsupportedPathInRootDirectory -- File - implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(path.Join(DirForImplicitDirTests, testDirName)) + implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(ctx, path.Join(DirForImplicitDirTests, testDirName), storageClient, t) implicit_and_explicit_dir_setup.CreateExplicitDirectoryStructure(path.Join(DirForImplicitDirTests, testDirName), t) + implicit_and_explicit_dir_setup.CreateUnsupportedImplicitDirectoryStructure(ctx, path.Join(DirForImplicitDirTests, testDirName), storageClient, t) err := filepath.WalkDir(testDirPath, func(path string, dir fs.DirEntry, err error) error { if err != nil { @@ -62,7 +65,7 @@ func TestListImplicitObjectsFromBucket(t *testing.T) { if path == testDirPath { // numberOfObjects - 3 if len(objs) != implicit_and_explicit_dir_setup.NumberOfTotalObjects { - t.Errorf("Incorrect number of objects in the bucket.") + t.Fatalf("Incorrect number of objects in the bucket: actual=%v, expected=%v.", len(objs), implicit_and_explicit_dir_setup.NumberOfTotalObjects) } // testBucket/dirForImplicitDirTests/testDir/explicitDir -- Dir @@ -84,7 +87,7 @@ func TestListImplicitObjectsFromBucket(t *testing.T) { if dir.IsDir() && dir.Name() == implicit_and_explicit_dir_setup.ExplicitDirectory { // numberOfObjects - 2 if len(objs) != implicit_and_explicit_dir_setup.NumberOfFilesInExplicitDirectory { - t.Errorf("Incorrect number of objects in the explicitDirectory.") + t.Fatalf("Incorrect number of objects in the explicitDirectory: actual=%v,expected:%v", len(objs), implicit_and_explicit_dir_setup.NumberOfFilesInExplicitDirectory) } // testBucket/dirForImplicitDirTests/testDir/explicitDir/fileInExplicitDir1 -- File @@ -103,7 +106,7 @@ func TestListImplicitObjectsFromBucket(t *testing.T) { if dir.IsDir() && dir.Name() == implicit_and_explicit_dir_setup.ImplicitDirectory { // numberOfObjects - 2 if len(objs) != implicit_and_explicit_dir_setup.NumberOfFilesInImplicitDirectory { - t.Errorf("Incorrect number of objects in the implicitDirectory.") + t.Fatalf("Incorrect number of objects in the implicitDirectory: actual=%v,expected=%v", len(objs), implicit_and_explicit_dir_setup.NumberOfFilesInImplicitDirectory) } // testBucket/dirForImplicitDirTests/testDir/implicitDir/fileInImplicitDir1 -- File diff --git a/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup/implicit_and_explicit_dir_setup.go b/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup/implicit_and_explicit_dir_setup.go index d20879c4da..f631ba003b 100644 --- a/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup/implicit_and_explicit_dir_setup.go +++ b/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup/implicit_and_explicit_dir_setup.go @@ -15,11 +15,14 @@ package implicit_and_explicit_dir_setup import ( + "context" "log" "os" "path" "testing" + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/persistent_mounting" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" @@ -41,6 +44,8 @@ const FirstFileInExplicitDirectory = "fileInExplicitDir1" const SecondFileInExplicitDirectory = "fileInExplicitDir2" const FileInImplicitDirectory = "fileInImplicitDir1" const FileInImplicitSubDirectory = "fileInImplicitDir2" +const FileInUnsupportedImplicitDirectory1 = "fileInUnsupportedImplicitDir1" +const FileInUnsupportedPathInRootDirectory = "fileInUnsupportedPathInRootDirectory" func RunTestsForImplicitDirAndExplicitDir(flags [][]string, m *testing.M) int { setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() @@ -74,7 +79,27 @@ func RemoveAndCheckIfDirIsDeleted(dirPath string, dirName string, t *testing.T) } } -func CreateImplicitDirectoryStructure(testDir string) { +type objectCreationMetadata struct{ content, completeObjectName string } + +func createObjectsOnGcs(ctx context.Context, objects []objectCreationMetadata, sgClient *storage.Client, t *testing.T) { + for _, object := range objects { + if err := client.CreateObjectOnGCS(ctx, sgClient, object.completeObjectName, object.content); err != nil { + t.Fatalf("Failed to create object %s: %v", object.completeObjectName, err) + } + } +} + +func createObjectsInImplicitDir(ctx context.Context, completeTestDirName string, storageClient *storage.Client, t *testing.T) { + implicitDirName := path.Join(completeTestDirName, ImplicitDirectory) + createObjectsOnGcs(ctx, + []objectCreationMetadata{ + {"This is from directory fileInImplicitDir1 file implicitDirectory", path.Join(implicitDirName, FileInImplicitDirectory)}, + {"This is from directory implicitDirectory/implicitSubDirectory file fileInImplicitDir2", path.Join(implicitDirName, ImplicitSubDirectory, FileInImplicitSubDirectory)}, + }, + storageClient, t) +} + +func CreateImplicitDirectoryStructure(ctx context.Context, testDir string, storageClient *storage.Client, t *testing.T) { // Implicit Directory Structure // testBucket/testDir/implicitDirectory -- Dir // testBucket/testDir/implicitDirectory/fileInImplicitDir1 -- File @@ -82,7 +107,26 @@ func CreateImplicitDirectoryStructure(testDir string) { // testBucket/testDir/implicitDirectory/implicitSubDirectory/fileInImplicitDir2 -- File // Create implicit directory in bucket for testing. - setup.RunScriptForTestData("../util/setup/implicit_and_explicit_dir_setup/testdata/create_objects.sh", path.Join(setup.TestBucket(), testDir)) + createObjectsInImplicitDir(ctx, testDir, storageClient, t) +} + +func CreateUnsupportedImplicitDirectoryStructure(ctx context.Context, testDir string, storageClient *storage.Client, t *testing.T) { + // Unsupported Implicit Directory Structure + // testBucket/testDir/implicitDirectory -- Dir + // testBucket/testDir/implicitDirectory//fileInUnsupportedImplicitDir1 -- File + // testBucket//FileInUnsupportedPathInRootDirectory -- File + + completeGcsTestDirName := path.Join(testDir, ImplicitDirectory) + createObjectsOnGcs(ctx, + []objectCreationMetadata{ + { + "This is testBucket/testDir//fileInUnsupportedImplicitDir1", completeGcsTestDirName + "//" + FileInUnsupportedImplicitDirectory1, + }, + { + "This is testBucket//fileInUnsupportedPathInRootDirectory", "/" + FileInUnsupportedPathInRootDirectory, + }, + }, + storageClient, t) } func CreateExplicitDirectoryStructure(testDir string, t *testing.T) { @@ -104,7 +148,7 @@ func CreateExplicitDirectoryStructure(testDir string, t *testing.T) { defer operations.CloseFile(file) } -func CreateImplicitDirectoryInExplicitDirectoryStructure(testDir string, t *testing.T) { +func CreateImplicitDirectoryInExplicitDirectoryStructure(ctx context.Context, testDir string, storageClient *storage.Client, t *testing.T) { // testBucket/testDir/explicitDirectory -- Dir // testBucket/testDir/explictFile -- File // testBucket/testDir/explicitDirectory/fileInExplicitDir1 -- File @@ -115,6 +159,5 @@ func CreateImplicitDirectoryInExplicitDirectoryStructure(testDir string, t *test // testBucket/testDir/explicitDirectory/implicitDirectory/implicitSubDirectory/fileInImplicitDir2 -- File CreateExplicitDirectoryStructure(testDir, t) - dirPathInBucket := path.Join(setup.TestBucket(), testDir, ExplicitDirectory) - setup.RunScriptForTestData("../util/setup/implicit_and_explicit_dir_setup/testdata/create_objects.sh", dirPathInBucket) + createObjectsInImplicitDir(ctx, path.Join(testDir, ExplicitDirectory), storageClient, t) } diff --git a/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup/testdata/create_objects.sh b/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup/testdata/create_objects.sh deleted file mode 100755 index 82ba8cabbb..0000000000 --- a/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup/testdata/create_objects.sh +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# The directory name is generated with a random component to avoid collisions as we are running implicit_dir_test and explicit_dir test parallelly. -temp_dir=$(mktemp -d) -cd "$temp_dir" -# Here $1 refers to the testBucket argument -echo "This is from directory fileInImplicitDir1 file implicitDirectory" > fileInImplicitDir1 -# bucket/implicitDirectory/fileInImplicitDir1 -gcloud storage cp fileInImplicitDir1 gs://$1/implicitDirectory/ -echo "This is from directory implicitDirectory/implicitSubDirectory file fileInImplicitDir2" > fileInImplicitDir2 -# bucket/implicitDirectory/implicitSubDirectory/fileInImplicitDir2 -gcloud storage cp fileInImplicitDir2 gs://$1/implicitDirectory/implicitSubDirectory/ -cd .. -rm -rf "$temp_dir"