Skip to content

Commit

Permalink
Invoke-DbaDbDataGenerator - Test for MaskingType Random to support sp…
Browse files Browse the repository at this point in the history 
…ecial masking types (#8390)
  • Loading branch information
@andreasjordan
andreasjordan authored Jun 14, 2022
1 parent e169c89 commit 2c90d9e
Show file tree
Hide file tree
Showing 4 changed files with 73 additions and 63 deletions.
44 changes: 44 additions & 0 deletions bin/datamasking/columntypes.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
[
{
"TypeName": "Address",
"MaskingType": "Address",
"SubType": "StreetAddress",
"Synonym": [
"Address",
"Location",
Expand All @@ -10,20 +12,26 @@
},
{
"TypeName": "Bic",
"MaskingType": "Finance",
"SubType": "Bic",
"Synonym": [
"Bic",
"BicAddress"
]
},
{
"TypeName": "Bitcoin",
"MaskingType": "Finance",
"SubType": "BitcoinAddress",
"Synonym": [
"Bitcoin",
"BitcoinAddress"
]
},
{
"TypeName": "Email",
"MaskingType": "Internet",
"SubType": "Email",
"Synonym": [
"Email",
"E-mail",
Expand All @@ -32,94 +40,122 @@
},
{
"TypeName": "Ethereum",
"MaskingType": "Finance",
"SubType": "EthereumAddress",
"Synonym": [
"Ethereum",
"EthereumAddress"
]
},
{
"TypeName": "Creditcard",
"MaskingType": "Finance",
"SubType": "CreditcardNumber",
"Synonym": [
"Creditcard",
"CreditcardNumber"
]
},
{
"TypeName": "CreditcardCVV",
"MaskingType": "Finance",
"SubType": "CreditCardCvv",
"Synonym": [
"Cvv",
"CreditcardCvv"
]
},
{
"TypeName": "City",
"MaskingType": "Address",
"SubType": "City",
"Synonym": [
"City"
]
},
{
"TypeName": "Company",
"MaskingType": "Company",
"SubType": "CompanyName",
"Synonym": [
"Company",
"CompanyName"
]
},
{
"TypeName": "Country",
"MaskingType": "Address",
"SubType": "Country",
"Synonym": [
"Country"
]
},
{
"TypeName": "CountryCode",
"MaskingType": "Address",
"SubType": "CountryCode",
"Synonym": [
"CountryCode"
]
},
{
"TypeName": "Firstname",
"MaskingType": "Name",
"SubType": "Firstname",
"Synonym": [
"Firstname",
"Forename"
]
},
{
"TypeName": "Fullname",
"MaskingType": "Name",
"SubType": "FullName",
"Synonym": [
"Fullname",
"TypeName"
]
},
{
"TypeName": "Iban",
"MaskingType": "Finance",
"SubType": "Iban",
"Synonym": [
"Iban",
"IbanNumber"
]
},
{
"TypeName": "Lastname",
"MaskingType": "Name",
"SubType": "Lastname",
"Synonym": [
"Lastname",
"Surname"
]
},
{
"TypeName": "Latitude",
"MaskingType": "Address",
"SubType": "Latitude",
"Synonym": [
"Latitude",
"Lat"
]
},
{
"TypeName": "Longitude",
"MaskingType": "Address",
"SubType": "Longitude",
"Synonym": [
"Longitude",
"Lon"
]
},
{
"TypeName": "Phone",
"MaskingType": "Phone",
"SubType": "PhoneNumber",
"Synonym": [
"Fax",
"FaxNumber",
Expand All @@ -131,19 +167,25 @@
},
{
"TypeName": "State",
"MaskingType": "Address",
"SubType": "State",
"Synonym": [
"State"
]
},
{
"TypeName": "StateAbbr",
"MaskingType": "Address",
"SubType": "StateAbbr",
"Synonym": [
"StateAbbreviation",
"StateCode"
]
},
{
"TypeName": "Username",
"MaskingType": "Internet",
"SubType": "UserName",
"Synonym": [
"Login",
"LoginId",
Expand All @@ -153,6 +195,8 @@
},
{
"TypeName": "Zipcode",
"MaskingType": "Address",
"SubType": "Zipcode",
"Synonym": [
"Zipcode",
"Zip",
Expand Down
2 changes: 1 addition & 1 deletion functions/Get-DbaRandomizedValue.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ function Get-DbaRandomizedValue {
Subtype to use.
.PARAMETER Min
Minimum value used to generate certain lengths of values. Default is 0
Minimum value used to generate certain lengths of values. Default is 1
.PARAMETER Max
Maximum value used to generate certain lengths of values. Default is 255
Expand Down
30 changes: 20 additions & 10 deletions functions/Invoke-DbaDbDataGenerator.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,14 +109,14 @@ function Invoke-DbaDbDataGenerator {

# Create the faker objects
try {
$faker = New-Object Bogus.Faker($Locale)
$script:faker = New-Object Bogus.Faker($Locale)
} catch {
Stop-Function -Message "Could not load randomizer class" -Continue
}

$supportedDataTypes = 'bigint', 'bit', 'bool', 'char', 'date', 'datetime', 'datetime2', 'decimal', 'int', 'float', 'guid', 'money', 'numeric', 'nchar', 'ntext', 'nvarchar', 'real', 'smalldatetime', 'smallint', 'text', 'time', 'tinyint', 'uniqueidentifier', 'userdefineddatatype', 'varchar'
$supportedFakerMaskingTypes = ($faker | Get-Member -MemberType Property | Select-Object Name -ExpandProperty Name)
$supportedFakerSubTypes = ($faker | Get-Member -MemberType Property) | ForEach-Object { ($faker.$($_.Name)) | Get-Member -MemberType Method | Where-Object { $_.Name -notlike 'To*' -and $_.Name -notlike 'Get*' -and $_.Name -notlike 'Trim*' -and $_.Name -notin 'Add', 'Equals', 'CompareTo', 'Clone', 'Contains', 'CopyTo', 'EndsWith', 'IndexOf', 'IndexOfAny', 'Insert', 'IsNormalized', 'LastIndexOf', 'LastIndexOfAny', 'Normalize', 'PadLeft', 'PadRight', 'Remove', 'Replace', 'Split', 'StartsWith', 'Substring', 'Letter', 'Lines', 'Paragraph', 'Paragraphs', 'Sentence', 'Sentences' } | Select-Object name -ExpandProperty Name }
$supportedFakerMaskingTypes = ($script:faker | Get-Member -MemberType Property | Select-Object Name -ExpandProperty Name)
$supportedFakerSubTypes = ($script:faker | Get-Member -MemberType Property) | ForEach-Object { ($script:faker.$($_.Name)) | Get-Member -MemberType Method | Where-Object { $_.Name -notlike 'To*' -and $_.Name -notlike 'Get*' -and $_.Name -notlike 'Trim*' -and $_.Name -notin 'Add', 'Equals', 'CompareTo', 'Clone', 'Contains', 'CopyTo', 'EndsWith', 'IndexOf', 'IndexOfAny', 'Insert', 'IsNormalized', 'LastIndexOf', 'LastIndexOfAny', 'Normalize', 'PadLeft', 'PadRight', 'Remove', 'Replace', 'Split', 'StartsWith', 'Substring', 'Letter', 'Lines', 'Paragraph', 'Paragraphs', 'Sentence', 'Sentences' } | Select-Object name -ExpandProperty Name }
$supportedFakerSubTypes += "Date"
#$foreignKeyQuery = Get-Content -Path "$script:PSModuleRoot\bin\datageneration\ForeignKeyHierarchy.sql"
}
Expand Down Expand Up @@ -217,7 +217,7 @@ function Invoke-DbaDbDataGenerator {
if ($PSBoundParameters.MaxValue -and $columnMaskInfo.SubType -eq 'String' -and $columnMaskInfo.MaxValue -gt $MaxValue) {
$columnMaskInfo.MaxValue = $MaxValue
}
if ($columnMaskInfo.ColumnType -in $supportedDataTypes) {
if ($columnMaskInfo.ColumnType -in $supportedDataTypes -and $columnMaskInfo.MaskingType -eq 'Random' -and $columnMaskInfo.SubType -in 'Bool', 'Number', 'Float', 'Byte', 'String') {
$newValue = Get-DbaRandomizedValue -DataType $columnMaskInfo.ColumnType -Locale $Locale -Min $columnMaskInfo.MinValue -Max $columnMaskInfo.MaxValue
} else {
$newValue = Get-DbaRandomizedValue -RandomizerType $columnMaskInfo.MaskingType -RandomizerSubtype $columnMaskInfo.SubType -Locale $Locale -Min $columnMaskInfo.MinValue -Max $columnMaskInfo.MaxValue
Expand Down Expand Up @@ -249,14 +249,14 @@ function Invoke-DbaDbDataGenerator {
if ($PSBoundParameters.MaxValue -and $columnMaskInfo.SubType -eq 'String' -and $columnMaskInfo.MaxValue -gt $MaxValue) {
$columnMaskInfo.MaxValue = $MaxValue
}
if ($columnMaskInfo.ColumnType -in $supportedDataTypes) {
if ($columnMaskInfo.ColumnType -in $supportedDataTypes -and $columnMaskInfo.MaskingType -eq 'Random' -and $columnMaskInfo.SubType -in 'Bool', 'Number', 'Float', 'Byte', 'String') {
$newValue = Get-DbaRandomizedValue -DataType $columnMaskInfo.ColumnType -Locale $Locale -Min $columnMaskInfo.MinValue -Max $columnMaskInfo.MaxValue
} else {
$newValue = Get-DbaRandomizedValue -RandomizerType $columnMaskInfo.MaskingType -RandomizerSubtype $columnMaskInfo.SubType -Locale $Locale -Min $columnMaskInfo.MinValue -Max $columnMaskInfo.MaxValue
}

} catch {
Stop-Function -Message "Failure" -Target $faker -Continue -ErrorRecord $_
Stop-Function -Message "Failure" -Target $script:faker -Continue -ErrorRecord $_
}

# Check if the value is already present as a property
Expand Down Expand Up @@ -300,7 +300,7 @@ function Invoke-DbaDbDataGenerator {
Write-ProgressHelper -StepNumber ($stepcounter++) -TotalSteps $tables.Tables.Count -Activity "Generating data" -Message "Inserting $($tableobject.Rows) rows in $($tableobject.Schema).$($tableobject.Name) in $($db.Name) on $instance"

if ($tableobject.TruncateTable) {
$query += "TRUNCATE TABLE [$($tableobject.Schema)].[$($tableobject.Name)];`n"
$query = "TRUNCATE TABLE [$($tableobject.Schema)].[$($tableobject.Name)];"

try {
$null = Invoke-DbaQuery -SqlInstance $SqlInstance -SqlCredential $SqlCredential -Database $db.Name -Query $query
Expand All @@ -318,10 +318,20 @@ function Invoke-DbaDbDataGenerator {

try {
$identityValues = Invoke-DbaQuery -SqlInstance $SqlInstance -SqlCredential $SqlCredential -Database $db.Name -Query $query
# https://docs.microsoft.com/en-us/sql/t-sql/functions/ident-current-transact-sql says:
# When the IDENT_CURRENT value is NULL (because the table has never contained rows or has been truncated), the IDENT_CURRENT function returns the seed value.
# So if we get a 1 back, we count the rows so that the first row added to an empty table gets the number 1.
if ($identityValues.CurrentIdentity -eq 1) {
$query = "SELECT COUNT(*) FROM [$($tableobject.Schema)].[$($tableobject.Name)];"
$rowcount = Invoke-DbaQuery -SqlInstance $SqlInstance -SqlCredential $SqlCredential -Database $db.Name -Query $query -As SingleValue
if ($rowcount -eq 0) {
$identityValues.CurrentIdentity = 0
}
}
} catch {
Write-Message -Level VeryVerbose -Message "$query"
$errormessage = $_.Exception.Message.ToString()
Stop-Function -Message "Error setting identity values from $($tableobject.Schema).$($tableobject.Name): $errormessage" -Target $query -Continue -ErrorRecord $_
Stop-Function -Message "Error getting identity values from $($tableobject.Schema).$($tableobject.Name): $errormessage" -Target $query -Continue -ErrorRecord $_
}

$insertQuery += "SET IDENTITY_INSERT [$($tableobject.Schema)].[$($tableobject.Name)] ON;`n"
Expand Down Expand Up @@ -388,14 +398,14 @@ function Invoke-DbaDbDataGenerator {
if ($PSBoundParameters.MaxValue -and $columnobject.SubType -eq 'String' -and $columnobject.MaxValue -gt $MaxValue) {
$columnobject.MaxValue = $MaxValue
}
if ($columnobject.ColumnType -in $supportedDataTypes) {
if ($columnobject.ColumnType -in $supportedDataTypes -and $columnobject.MaskingType -eq 'Random' -and $columnobject.SubType -in 'Bool', 'Number', 'Float', 'Byte', 'String') {
$columnValue = Get-DbaRandomizedValue -DataType $columnobject.ColumnType -CharacterString $charstring -Locale $Locale -Min $columnobject.MinValue -Max $columnobject.MaxValue
} else {
$columnValue = Get-DbaRandomizedValue -RandomizerType $columnobject.MaskingType -RandomizerSubtype $columnobject.SubType -CharacterString $charstring -Locale $Locale -Min $columnobject.MinValue -Max $columnobject.MaxValue
}

} catch {
Stop-Function -Message "Failure" -Target $faker -Continue -ErrorRecord $_
Stop-Function -Message "Failure" -Target $script:faker -Continue -ErrorRecord $_
}

}
Expand Down
Loading

0 comments on commit 2c90d9e

Please sign in to comment.