Skip to content

Commit

Permalink
[testing] Add e2e tests for unsupported directory names
Browse files Browse the repository at this point in the history 
This adds tests for unsupported directories (
directories which are either named '', or '.'
or '..', or have '//', '/./', '/../' in their
total path in their bucket.

update documentation in semantics.md

[debug] initialize log-file in fs_test

[debug] add logs in bucket_handle list function

update list implementation in fake bucket_handle

improve util.go

fix lint error in integration test

[debug] add logfile in internal/fs/implicit_dir

add walkdir test in internal/fs/implicit_dir

[debug] remove earlier added internal/fs/implicit_dir test

fix bucket_handle unit tests

Remove tests for ., .., \0
Optimized test code.

Restrict unsupported directory check to '/'

[debug-fix] revert unsupported dir checks in fake-bucket

Improve error-check in integration test

add logs in internal/fs/implicit_dir

[debug] add debug logs in fake bucket

[debug] update debug logs in internal/fs/implicit_dir

reenable fix for unsupportd dirs in fake/bucket

Move the check for unsupported directory in fake/bucket

[debug] log changes in fake/bucket

add test for unsupported directories in fake/bucket

reenable more paths in internal/fs/implicit_dir walk-dir test

fix for unsupported dirs in fake/bucket

[debug] log changes in fake/bucket

remove commented out logic of unsupported directories in fake/bucket

[debug] all logs from fake/bucket

fix internal/fs/implicit_dir walkdir test

add more test paths to internal/fs/implicit-dir walkdir test

[debug] remove debug logs from internal/fs/implicit-dir walkdir test

added comments in fake/bucket fix

beefed up the unit test for unsupported names in fake/bucket

Revert "[debug] remove earlier added internal/fs/implicit_dir test"

This reverts commit f512982.

Revert "[debug] all logs from fake/bucket"

This reverts commit 0fcb4e3.

Reapply "[debug] all logs from fake/bucket"

This reverts commit 67621f8.

fix linter errors

merge isunsupported* functions into one

code cleanup

1. Remove all logs, comments
2. Remove all commented out code
3. Remove all unnecessary changes

temp

fix case for unsupported objects

add test for listing with MaxResults 1 in bucket_handle_test

cleanup

add more test cases in internal/fs/implicit_dir_test

address self-review comments

add more unsupported paths in fs/implicit_dir_tests

address self-review comments

add more details to semantics doc

Remove integration test changes from this PR

address some review comments

fix test broken by last commit

Add TestListObjectMethodWithOneMaxResult

Test in bucket_handle_test.go for MaxResults=1 case.

switch implementation

disable other test files in internal/fs

disable older preexisting tests in internal/fs/implicit_dirs_test.go

fix failing tests

code cleanup

Revert "disable older preexisting tests in internal/fs/implicit_dirs_test.go"

This reverts commit bd2dedc.

Revert "disable other test files in internal/fs"

This reverts commit cf00084.

fix broken test

address self-review comments

move RemoveUnsupportedObjectsFromListing to util package

add a code comment

fix comment and shorten code

remove code meant for follow-up branches

fix linter error

remove unnecessarily added test

fix typo

improve composite tests

rerun presubmit

Optimize a unit test

fix comment

Add test for unsupported objects in dir_test.go

address a unit test

update a unit test

update in a unit test

rerun presubmit

Revert "Remove integration test changes from this PR"

This reverts commit caa5342.

Create unsupported name test objects with client

Switching from earlier creating them with gcloud command

remove unused constant

remove  dependence on create_objects.sh

optimize runtime, fix comments

address a review comment

add fix build failure

address a review comment

address a review comment

Pass package-context in all createObject calls

fix bucket-handle unit tests

fix dir_test units
  • Loading branch information
@gargnitingoogle
gargnitingoogle committed Oct 8, 2024
1 parent f866623 commit d2c4305
Show file tree
Hide file tree
Showing 6 changed files with 77 additions and 42 deletions.
15 changes: 15 additions & 0 deletions tools/integration_tests/explicit_dir/explicit_dir_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"log"
"os"
"testing"
"time"

"cloud.google.com/go/storage"
"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client"
Expand All @@ -29,6 +30,11 @@ import (

const DirForExplicitDirTests = "dirForExplicitDirTests"

var (
ctx context.Context
storageClient *storage.Client
)

func TestMain(m *testing.M) {
setup.ParseSetUpFlags()
var storageClient *storage.Client
Expand All @@ -49,6 +55,15 @@ func TestMain(m *testing.M) {
return
}

// Create storage client before running tests.
ctx = context.Background()
closeStorageClient := client.CreateStorageClientWithTimeOut(&ctx, &storageClient, time.Minute*15)
defer func() {
err := closeStorageClient()
if err != nil {
log.Fatalf("closeStorageClient failed: %v", err)
}
}()
flags := [][]string{{"--implicit-dirs=false"}}

if !testing.Short() {
Expand Down
2 changes: 1 addition & 1 deletion tools/integration_tests/explicit_dir/list_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ func TestListOnlyExplicitObjectsFromBucket(t *testing.T) {
// testBucket/dirForExplicitDirTests/explicitDirectory/fileInExplicitDir1 -- File
// testBucket/dirForExplicitDirTests/explicitDirectory/fileInExplicitDir2 -- File

implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(DirForExplicitDirTests)
implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(ctx, DirForExplicitDirTests, storageClient, t)
implicit_and_explicit_dir_setup.CreateExplicitDirectoryStructure(DirForExplicitDirTests, t)

err := filepath.WalkDir(testDir, func(path string, dir fs.DirEntry, err error) error {
Expand Down
12 changes: 6 additions & 6 deletions tools/integration_tests/implicit_dir/delete_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ import (
// testBucket/dirForImplicitDirTests/implicitDirectory/implicitSubDirectory/fileInImplicitDir2 -- File
func TestDeleteNonEmptyImplicitDir(t *testing.T) {
testDir := setup.SetupTestDirectory(DirForImplicitDirTests)
implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(DirForImplicitDirTests)
implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(ctx, DirForImplicitDirTests, storageClient, t)

dirPath := path.Join(testDir, implicit_and_explicit_dir_setup.ImplicitDirectory)

Expand All @@ -45,7 +45,7 @@ func TestDeleteNonEmptyImplicitDir(t *testing.T) {
// testBucket/dirForImplicitDirTests/implicitDirectory/implicitSubDirectory/fileInImplicitDir2 -- File
func TestDeleteNonEmptyImplicitSubDir(t *testing.T) {
testDir := setup.SetupTestDirectory(DirForImplicitDirTests)
implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(DirForImplicitDirTests)
implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(ctx, DirForImplicitDirTests, storageClient, t)

subDirPath := path.Join(testDir, implicit_and_explicit_dir_setup.ImplicitDirectory, implicit_and_explicit_dir_setup.ImplicitSubDirectory)

Expand All @@ -61,7 +61,7 @@ func TestDeleteNonEmptyImplicitSubDir(t *testing.T) {
// testBucket/dirForImplicitDirTests/implicitDirectory/implicitSubDirectory/fileInImplicitDir2 -- File
func TestDeleteImplicitDirWithExplicitSubDir(t *testing.T) {
testDir := setup.SetupTestDirectory(DirForImplicitDirTests)
implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(DirForImplicitDirTests)
implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(ctx, DirForImplicitDirTests, storageClient, t)

explicitDirPath := path.Join(testDir, implicit_and_explicit_dir_setup.ImplicitDirectory, ExplicitDirInImplicitDir)

Expand All @@ -81,7 +81,7 @@ func TestDeleteImplicitDirWithExplicitSubDir(t *testing.T) {
// testBucket/dirForImplicitDirTests/implicitDirectory/implicitSubDirectory/explicitDirInImplicitDir/fileInExplicitDirInImplicitDir -- File
func TestDeleteImplicitDirWithImplicitSubDirContainingExplicitDir(t *testing.T) {
testDir := setup.SetupTestDirectory(DirForImplicitDirTests)
implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(DirForImplicitDirTests)
implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(ctx, DirForImplicitDirTests, storageClient, t)
explicitDirPath := path.Join(testDir, implicit_and_explicit_dir_setup.ImplicitDirectory, implicit_and_explicit_dir_setup.ImplicitSubDirectory, ExplicitDirInImplicitSubDir)

operations.CreateDirectoryWithNFiles(NumberOfFilesInExplicitDirInImplicitSubDir, explicitDirPath, PrefixFileInExplicitDirInImplicitSubDir, t)
Expand All @@ -102,7 +102,7 @@ func TestDeleteImplicitDirWithImplicitSubDirContainingExplicitDir(t *testing.T)
// testBucket/dirForImplicitDirTests/explicitDirectory/implicitDirectory/implicitSubDirectory/fileInImplicitDir2 -- File
func TestDeleteImplicitDirInExplicitDir(t *testing.T) {
testDir := setup.SetupTestDirectory(DirForImplicitDirTests)
implicit_and_explicit_dir_setup.CreateImplicitDirectoryInExplicitDirectoryStructure(DirForImplicitDirTests, t)
implicit_and_explicit_dir_setup.CreateImplicitDirectoryInExplicitDirectoryStructure(ctx, DirForImplicitDirTests, storageClient, t)

dirPath := path.Join(testDir, implicit_and_explicit_dir_setup.ExplicitDirectory, implicit_and_explicit_dir_setup.ImplicitDirectory)

Expand All @@ -120,7 +120,7 @@ func TestDeleteImplicitDirInExplicitDir(t *testing.T) {
// testBucket/dirForImplicitDirTests/explicitDirectory/implicitDirectory/implicitSubDirectory/fileInImplicitDir2 -- File
func TestDeleteExplicitDirContainingImplicitSubDir(t *testing.T) {
testDir := setup.SetupTestDirectory(DirForImplicitDirTests)
implicit_and_explicit_dir_setup.CreateImplicitDirectoryInExplicitDirectoryStructure(DirForImplicitDirTests, t)
implicit_and_explicit_dir_setup.CreateImplicitDirectoryInExplicitDirectoryStructure(ctx, DirForImplicitDirTests, storageClient, t)

dirPath := path.Join(testDir, implicit_and_explicit_dir_setup.ExplicitDirectory)

Expand Down
11 changes: 7 additions & 4 deletions tools/integration_tests/implicit_dir/list_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,12 @@ func TestListImplicitObjectsFromBucket(t *testing.T) {
// testBucket/dirForImplicitDirTests/explicitFile -- File
// testBucket/dirForImplicitDirTests/explicitDirectory/fileInExplicitDir1 -- File
// testBucket/dirForImplicitDirTests/explicitDirectory/fileInExplicitDir2 -- File
// testBucket/dirForImplicitDirTests/implicitDirectory//fileInUnsupportedImplicitDir1 -- File
// testBucket//fileInUnsupportedPathInRootDirectory -- File

implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(DirForImplicitDirTests)
implicit_and_explicit_dir_setup.CreateImplicitDirectoryStructure(ctx, DirForImplicitDirTests, storageClient, t)
implicit_and_explicit_dir_setup.CreateExplicitDirectoryStructure(DirForImplicitDirTests, t)
implicit_and_explicit_dir_setup.CreateUnsupportedImplicitDirectoryStructure(ctx, DirForImplicitDirTests, storageClient, t)

err := filepath.WalkDir(testDir, func(path string, dir fs.DirEntry, err error) error {
if err != nil {
Expand All @@ -61,7 +64,7 @@ func TestListImplicitObjectsFromBucket(t *testing.T) {
if path == testDir {
// numberOfObjects - 3
if len(objs) != implicit_and_explicit_dir_setup.NumberOfTotalObjects {
t.Errorf("Incorrect number of objects in the bucket.")
t.Fatalf("Incorrect number of objects in the bucket: actual=%v, expected=%v.", len(objs), implicit_and_explicit_dir_setup.NumberOfTotalObjects)
}

// testBucket/dirForImplicitDirTests/explicitDir -- Dir
Expand All @@ -83,7 +86,7 @@ func TestListImplicitObjectsFromBucket(t *testing.T) {
if dir.IsDir() && dir.Name() == implicit_and_explicit_dir_setup.ExplicitDirectory {
// numberOfObjects - 2
if len(objs) != implicit_and_explicit_dir_setup.NumberOfFilesInExplicitDirectory {
t.Errorf("Incorrect number of objects in the explicitDirectory.")
t.Fatalf("Incorrect number of objects in the explicitDirectory: actual=%v,expected:%v", len(objs), implicit_and_explicit_dir_setup.NumberOfFilesInExplicitDirectory)
}

// testBucket/dirForImplicitDirTests/explicitDir/fileInExplicitDir1 -- File
Expand All @@ -102,7 +105,7 @@ func TestListImplicitObjectsFromBucket(t *testing.T) {
if dir.IsDir() && dir.Name() == implicit_and_explicit_dir_setup.ImplicitDirectory {
// numberOfObjects - 2
if len(objs) != implicit_and_explicit_dir_setup.NumberOfFilesInImplicitDirectory {
t.Errorf("Incorrect number of objects in the implicitDirectory.")
t.Fatalf("Incorrect number of objects in the implicitDirectory: actual=%v,expected=%v", len(objs), implicit_and_explicit_dir_setup.NumberOfFilesInImplicitDirectory)
}

// testBucket/dirForImplicitDirTests/implicitDir/fileInImplicitDir1 -- File
Expand Down
53 changes: 48 additions & 5 deletions ...ation_tests/util/setup/implicit_and_explicit_dir_setup/implicit_and_explicit_dir_setup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,14 @@
package implicit_and_explicit_dir_setup

import (
"context"
"log"
"os"
"path"
"testing"

"cloud.google.com/go/storage"
"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client"
"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/persistent_mounting"
"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting"
"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations"
Expand All @@ -41,6 +44,8 @@ const FirstFileInExplicitDirectory = "fileInExplicitDir1"
const SecondFileInExplicitDirectory = "fileInExplicitDir2"
const FileInImplicitDirectory = "fileInImplicitDir1"
const FileInImplicitSubDirectory = "fileInImplicitDir2"
const FileInUnsupportedImplicitDirectory1 = "fileInUnsupportedImplicitDir1"
const FileInUnsupportedPathInRootDirectory = "fileInUnsupportedPathInRootDirectory"

func RunTestsForImplicitDirAndExplicitDir(flags [][]string, m *testing.M) int {
setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet()
Expand Down Expand Up @@ -74,15 +79,54 @@ func RemoveAndCheckIfDirIsDeleted(dirPath string, dirName string, t *testing.T)
}
}

func CreateImplicitDirectoryStructure(testDir string) {
type objectCreationMetadata struct{ content, completeObjectName string }

func createObjectsOnGcs(ctx context.Context, objects []objectCreationMetadata, sgClient *storage.Client, t *testing.T) {
for _, object := range objects {
if err := client.CreateObjectOnGCS(ctx, sgClient, object.completeObjectName, object.content); err != nil {
t.Fatalf("Failed to create object %s: %v", object.completeObjectName, err)
}
}
}

func createObjectsInImplicitDir(ctx context.Context, completeTestDirName string, storageClient *storage.Client, t *testing.T) {
implicitDirName := path.Join(completeTestDirName, ImplicitDirectory)
createObjectsOnGcs(ctx,
[]objectCreationMetadata{
{"This is from directory fileInImplicitDir1 file implicitDirectory", path.Join(implicitDirName, FileInImplicitDirectory)},
{"This is from directory implicitDirectory/implicitSubDirectory file fileInImplicitDir2", path.Join(implicitDirName, ImplicitSubDirectory, FileInImplicitSubDirectory)},
},
storageClient, t)
}

func CreateImplicitDirectoryStructure(ctx context.Context, testDir string, storageClient *storage.Client, t *testing.T) {
// Implicit Directory Structure
// testBucket/testDir/implicitDirectory -- Dir
// testBucket/testDir/implicitDirectory/fileInImplicitDir1 -- File
// testBucket/testDir/implicitDirectory/implicitSubDirectory -- Dir
// testBucket/testDir/implicitDirectory/implicitSubDirectory/fileInImplicitDir2 -- File

// Create implicit directory in bucket for testing.
setup.RunScriptForTestData("../util/setup/implicit_and_explicit_dir_setup/testdata/create_objects.sh", path.Join(setup.TestBucket(), testDir))
createObjectsInImplicitDir(ctx, testDir, storageClient, t)
}

func CreateUnsupportedImplicitDirectoryStructure(ctx context.Context, testDir string, storageClient *storage.Client, t *testing.T) {
// Unsupported Implicit Directory Structure
// testBucket/testDir/implicitDirectory -- Dir
// testBucket/testDir/implicitDirectory//fileInUnsupportedImplicitDir1 -- File
// testBucket//FileInUnsupportedPathInRootDirectory -- File

completeGcsTestDirName := path.Join(testDir, ImplicitDirectory)
createObjectsOnGcs(ctx,
[]objectCreationMetadata{
{
"This is testBucket/testDir//fileInUnsupportedImplicitDir1", completeGcsTestDirName + "//" + FileInUnsupportedImplicitDirectory1,
},
{
"This is testBucket//fileInUnsupportedPathInRootDirectory", "/" + FileInUnsupportedPathInRootDirectory,
},
},
storageClient, t)
}

func CreateExplicitDirectoryStructure(testDir string, t *testing.T) {
Expand All @@ -104,7 +148,7 @@ func CreateExplicitDirectoryStructure(testDir string, t *testing.T) {
defer operations.CloseFile(file)
}

func CreateImplicitDirectoryInExplicitDirectoryStructure(testDir string, t *testing.T) {
func CreateImplicitDirectoryInExplicitDirectoryStructure(ctx context.Context, testDir string, storageClient *storage.Client, t *testing.T) {
// testBucket/testDir/explicitDirectory -- Dir
// testBucket/testDir/explictFile -- File
// testBucket/testDir/explicitDirectory/fileInExplicitDir1 -- File
Expand All @@ -115,6 +159,5 @@ func CreateImplicitDirectoryInExplicitDirectoryStructure(testDir string, t *test
// testBucket/testDir/explicitDirectory/implicitDirectory/implicitSubDirectory/fileInImplicitDir2 -- File

CreateExplicitDirectoryStructure(testDir, t)
dirPathInBucket := path.Join(setup.TestBucket(), testDir, ExplicitDirectory)
setup.RunScriptForTestData("../util/setup/implicit_and_explicit_dir_setup/testdata/create_objects.sh", dirPathInBucket)
createObjectsInImplicitDir(ctx, path.Join(testDir, ExplicitDirectory), storageClient, t)
}
26 changes: 0 additions & 26 deletions ...s/integration_tests/util/setup/implicit_and_explicit_dir_setup/testdata/create_objects.sh
View file

This file was deleted.

0 comments on commit d2c4305

Please sign in to comment.