diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b15562a26..cc24cbf1d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -134,7 +134,12 @@ ### Experimental -* None. +* Adds a new `--report-coverage` command line option for the `lint` and `analyze` subcommands, and an + equivalent `report_coverage` configuration file setting. Coverage is measured against enabled + rules (reflecting disablement of enabled rules via `swiftlint:disable`), and also against all linter + or analyzer rules (reflecting the actual usage of Swiftlint versus its potential). + [Martin Redington](https://github.com/mildm8nnered) + [#5907](https://github.com/realm/SwiftLint/issues/5907) ### Enhancements diff --git a/README.md b/README.md index 73ec013550..90c646dd0e 100644 --- a/README.md +++ b/README.md @@ -736,6 +736,9 @@ write_baseline: Baseline.json # If true, SwiftLint will check for updates after linting or analyzing. check_for_updates: true +# If true, SwiftLint will report coverage statistics after linting or analyzing. +report_coverage: true + # configurable rules can be customized from this configuration file # binary rules can set their severity level force_cast: warning # implicitly diff --git a/Source/SwiftLintCore/Extensions/SwiftLintFile+Cache.swift b/Source/SwiftLintCore/Extensions/SwiftLintFile+Cache.swift index 61a5d63302..1cb5edf889 100644 --- a/Source/SwiftLintCore/Extensions/SwiftLintFile+Cache.swift +++ b/Source/SwiftLintCore/Extensions/SwiftLintFile+Cache.swift @@ -42,6 +42,9 @@ private let commandsCache = Cache { file -> [Command] in return CommandVisitor(locationConverter: file.locationConverter) .walk(file: file, handler: \.commands) } +private let regionsCache = Cache { file -> [Region] in + file.regions() +} private let syntaxMapCache = Cache { file in responseCache.get(file).map { SwiftLintSyntaxMap(value: SyntaxMap(sourceKitResponse: $0)) } } @@ -168,6 +171,8 @@ extension SwiftLintFile { public var invalidCommands: [Command] { commandsCache.get(self).filter { !$0.isValid } } + public var regions: [Region] { regionsCache.get(self) } + public var syntaxTokensByLines: [[SwiftLintSyntaxToken]] { guard let syntaxTokensByLines = syntaxTokensByLinesCache.get(self) else { if let handler = assertHandler { @@ -204,6 +209,7 @@ extension SwiftLintFile { foldedSyntaxTreeCache.invalidate(self) locationConverterCache.invalidate(self) commandsCache.invalidate(self) + regionsCache.invalidate(self) linesWithTokensCache.invalidate(self) } @@ -219,6 +225,7 @@ extension SwiftLintFile { foldedSyntaxTreeCache.clear() locationConverterCache.clear() commandsCache.clear() + regionsCache.clear() linesWithTokensCache.clear() } } diff --git a/Source/SwiftLintCore/Extensions/SwiftLintFile+Regex.swift b/Source/SwiftLintCore/Extensions/SwiftLintFile+Regex.swift index c8fe43d36d..5fe7a60345 100644 --- a/Source/SwiftLintCore/Extensions/SwiftLintFile+Regex.swift +++ b/Source/SwiftLintCore/Extensions/SwiftLintFile+Regex.swift @@ -268,7 +268,7 @@ extension SwiftLintFile { } public func ruleEnabled(violatingRanges: [NSRange], for rule: some Rule) -> [NSRange] { - let fileRegions = regions() + let fileRegions = regions if fileRegions.isEmpty { return violatingRanges } return violatingRanges.filter { range in let region = fileRegions.first { diff --git a/Source/SwiftLintCore/Protocols/SwiftSyntaxCorrectableRule.swift b/Source/SwiftLintCore/Protocols/SwiftSyntaxCorrectableRule.swift index f977d9b050..5211fc17e7 100644 --- a/Source/SwiftLintCore/Protocols/SwiftSyntaxCorrectableRule.swift +++ b/Source/SwiftLintCore/Protocols/SwiftSyntaxCorrectableRule.swift @@ -47,7 +47,7 @@ public extension SwiftSyntaxCorrectableRule { } let locationConverter = file.locationConverter - let disabledRegions = file.regions() + let disabledRegions = file.regions .filter { $0.areRulesDisabled(ruleIDs: Self.description.allIdentifiers) } .compactMap { $0.toSourceRange(locationConverter: locationConverter) } @@ -91,7 +91,7 @@ open class ViolationsSyntaxRewriter: SyntaxRew public lazy var locationConverter = file.locationConverter /// The regions in the traversed file that are disabled by a command. public lazy var disabledRegions = { - file.regions() + file.regions .filter { $0.areRulesDisabled(ruleIDs: Configuration.Parent.description.allIdentifiers) } .compactMap { $0.toSourceRange(locationConverter: locationConverter) } }() diff --git a/Source/SwiftLintFramework/Configuration/Configuration+Merging.swift b/Source/SwiftLintFramework/Configuration/Configuration+Merging.swift index 82a11e342d..6880c8176b 100644 --- a/Source/SwiftLintFramework/Configuration/Configuration+Merging.swift +++ b/Source/SwiftLintFramework/Configuration/Configuration+Merging.swift @@ -22,6 +22,7 @@ extension Configuration { excludedPaths: mergedIncludedAndExcluded.excludedPaths, indentation: childConfiguration.indentation, warningThreshold: mergedWarningTreshold(with: childConfiguration), + reportCoverage: childConfiguration.reportCoverage, reporter: reporter, cachePath: cachePath, allowZeroLintableFiles: childConfiguration.allowZeroLintableFiles, diff --git a/Source/SwiftLintFramework/Configuration/Configuration+Parsing.swift b/Source/SwiftLintFramework/Configuration/Configuration+Parsing.swift index c8326c0776..ce5bdde589 100644 --- a/Source/SwiftLintFramework/Configuration/Configuration+Parsing.swift +++ b/Source/SwiftLintFramework/Configuration/Configuration+Parsing.swift @@ -7,6 +7,7 @@ extension Configuration { case excluded = "excluded" case included = "included" case optInRules = "opt_in_rules" + case reportCoverage = "report_coverage" case reporter = "reporter" case swiftlintVersion = "swiftlint_version" case warningThreshold = "warning_threshold" @@ -99,6 +100,7 @@ extension Configuration { excludedPaths: defaultStringArray(dict[Key.excluded.rawValue]), indentation: Self.getIndentationLogIfInvalid(from: dict), warningThreshold: dict[Key.warningThreshold.rawValue] as? Int, + reportCoverage: dict[Key.reportCoverage.rawValue] as? Bool ?? false, reporter: dict[Key.reporter.rawValue] as? String ?? XcodeReporter.identifier, cachePath: cachePath ?? dict[Key.cachePath.rawValue] as? String, pinnedVersion: dict[Key.swiftlintVersion.rawValue].map { ($0 as? String) ?? String(describing: $0) }, diff --git a/Source/SwiftLintFramework/Configuration/Configuration.swift b/Source/SwiftLintFramework/Configuration/Configuration.swift index bf5d6d9bc1..3a3a375aa6 100644 --- a/Source/SwiftLintFramework/Configuration/Configuration.swift +++ b/Source/SwiftLintFramework/Configuration/Configuration.swift @@ -26,6 +26,9 @@ public struct Configuration { /// The threshold for the number of warnings to tolerate before treating the lint as having failed. public let warningThreshold: Int? + /// Report coverage statistics when linting or analyzing. + public let reportCoverage: Bool + /// The identifier for the `Reporter` to use to report style violations. public let reporter: String? @@ -47,7 +50,7 @@ public struct Configuration { /// The path to write a baseline to. public let writeBaseline: String? - /// Check for updates. + /// Check for updates after linting or analyzing.` public let checkForUpdates: Bool /// This value is `true` iff the `--config` parameter was used to specify (a) configuration file(s) @@ -82,6 +85,7 @@ public struct Configuration { excludedPaths: [String], indentation: IndentationStyle, warningThreshold: Int?, + reportCoverage: Bool, reporter: String?, cachePath: String?, allowZeroLintableFiles: Bool, @@ -97,6 +101,7 @@ public struct Configuration { self.excludedPaths = excludedPaths self.indentation = indentation self.warningThreshold = warningThreshold + self.reportCoverage = reportCoverage self.reporter = reporter self.cachePath = cachePath self.allowZeroLintableFiles = allowZeroLintableFiles @@ -117,6 +122,7 @@ public struct Configuration { excludedPaths = configuration.excludedPaths indentation = configuration.indentation warningThreshold = configuration.warningThreshold + reportCoverage = configuration.reportCoverage reporter = configuration.reporter basedOnCustomConfigurationFiles = configuration.basedOnCustomConfigurationFiles cachePath = configuration.cachePath @@ -143,6 +149,7 @@ public struct Configuration { /// - parameter indentation: The style to use when indenting Swift source code. /// - parameter warningThreshold: The threshold for the number of warnings to tolerate before treating the /// lint as having failed. + /// - parameter reportCoverage: Report coverage data after linting or analyzing. /// - parameter reporter: The identifier for the `Reporter` to use to report style violations. /// - parameter cachePath: The location of the persisted cache to use with this configuration. /// - parameter pinnedVersion: The SwiftLint version defined in this configuration. @@ -152,7 +159,7 @@ public struct Configuration { /// - parameter lenient: Treat errors as warnings. /// - parameter baseline: The path to read a baseline from. /// - parameter writeBaseline: The path to write a baseline to. - /// - parameter checkForUpdates: Check for updates to SwiftLint. + /// - parameter checkForUpdates: Check for updates to SwiftLint after linting or analyzing. package init( rulesMode: RulesMode = .defaultConfiguration(disabled: [], optIn: []), allRulesWrapped: [ConfigurationRuleWrapper]? = nil, @@ -162,6 +169,7 @@ public struct Configuration { excludedPaths: [String] = [], indentation: IndentationStyle = .default, warningThreshold: Int? = nil, + reportCoverage: Bool = false, reporter: String? = nil, cachePath: String? = nil, pinnedVersion: String? = nil, @@ -193,6 +201,7 @@ public struct Configuration { excludedPaths: excludedPaths, indentation: indentation, warningThreshold: warningThreshold, + reportCoverage: reportCoverage, reporter: reporter, cachePath: cachePath, allowZeroLintableFiles: allowZeroLintableFiles, @@ -310,6 +319,7 @@ extension Configuration: Hashable { hasher.combine(excludedPaths) hasher.combine(indentation) hasher.combine(warningThreshold) + hasher.combine(reportCoverage) hasher.combine(reporter) hasher.combine(allowZeroLintableFiles) hasher.combine(strict) @@ -328,6 +338,7 @@ extension Configuration: Hashable { lhs.excludedPaths == rhs.excludedPaths && lhs.indentation == rhs.indentation && lhs.warningThreshold == rhs.warningThreshold && + lhs.reportCoverage == rhs.reportCoverage && lhs.reporter == rhs.reporter && lhs.basedOnCustomConfigurationFiles == rhs.basedOnCustomConfigurationFiles && lhs.cachePath == rhs.cachePath && diff --git a/Source/SwiftLintFramework/Coverage.swift b/Source/SwiftLintFramework/Coverage.swift new file mode 100644 index 0000000000..7be7c1c597 --- /dev/null +++ b/Source/SwiftLintFramework/Coverage.swift @@ -0,0 +1,208 @@ +import Foundation + +/// Collects and reports coverage statistics. +/// +/// Coverage is defined as the sum of the number of rules applied to each line of input, divided by the product of the +/// number of input lines and the total number of rules. +/// +/// If all rules are applied to every input line, then coverage would be `1` (or 100%). If half the rules are applied +/// to all input lines, or if all the rules are applied to half of the input lines, then coverage would be `0.5`, and +/// if no rules are enabled, or there is no input, then coverage would be zero. +/// +/// No distinction is made between actual lines of Swift source, versus blank lines or comments, as SwiftLint may +/// apply rules to those as well. Coverage is only calculated over input files, so if you exclude files in your +/// configuration, they will be ignored. Empty input files, or files containing a single empty line will be ignored, as +/// SwiftLint ignores these files automatically. +/// +/// "All rules" can be defined either as *all enabled rules*, **or** *all available rules, enabled or not*, resulting +/// in two different coverage metrics: +/// +/// * "Enabled rules coverage" measures how frequently enabled rules are being disabled by `swiftlint:disable` +/// commands. +/// * "All rules coverage" measures how many of all possible rules are actually being applied. +/// +/// Typically, enabled rules coverage will be close to `1`, as `swiftlint:disable` is used sparingly. All rules +/// coverage will generally be much lower, as some rules are contradictory, and many rules are optin. With no opt-in +/// rules enabled, all rules coverage will be about `0.4`, rising to `0.8` or more if many opt-in rules are enabled. +/// +/// When calculating all rules coverage `swiftlint:disable` commands are still accounted for, but only for enabled +/// rules. +/// +/// Coverage figures will be different for linting and analyzing as these use different sets of rules. +/// +/// The number of enabled rules is determined on a per-file basis, so child and local configurations will be accounted +/// for. +/// +/// Custom rules, if enabled and defined, will be counted as first class rules for both enabled and all rules coverage. +/// If not enabled, `custom_rules` will be counted as a single rule, even if a configuration exists for it. +/// +/// When calculating enabled rules coverage, the custom rules in the configuration for each input file (e.g. including +/// child configurations) will be taken into account. When calculating all rules coverage, only the main configurations +/// custom rules settings will be used. +/// +struct Coverage { + struct Coverage: Equatable { + var numberOfLinesOfCode = 0 + var observedCoverage = 0 + var maximumCoverage = 0 + + mutating func add(_ coverage: Self) { + numberOfLinesOfCode += coverage.numberOfLinesOfCode + observedCoverage += coverage.observedCoverage + maximumCoverage += coverage.maximumCoverage + } + + func coverage(denominator: Int? = nil) -> Double { + let denominator = denominator ?? maximumCoverage + return denominator == 0 ? 0.0 : (Double(observedCoverage) / Double(denominator)) + } + } + + private let totalNumberOfRules: Int + var coverage = Self.Coverage() + + var enabledRulesCoverage: Double { + coverage.coverage() + } + var allRulesCoverage: Double { + coverage.coverage(denominator: coverage.numberOfLinesOfCode * totalNumberOfRules) + } + var report: String { + """ + Enabled rules coverage: \(enabledRulesCoverage.rounded(toNearestPlaces: 3)) + All rules coverage: \(allRulesCoverage.rounded(toNearestPlaces: 3)) + """ + } + + init(mode: LintOrAnalyzeMode, configuration: Configuration) { + self.totalNumberOfRules = configuration.totalNumberOfRules(for: mode) + } + + mutating func addCoverage(for linter: CollectedLinter) { + coverage.add(linter.file.coverage(for: linter.rules)) + } +} + +private extension SwiftLintFile { + func coverage(for rules: [any Rule]) -> Coverage.Coverage { + guard !isEmpty else { + return Coverage.Coverage() + } + let numberOfLinesInFile = lines.count + let ruleIdentifiers = Set(rules.ruleIdentifiers) + let maximumCoverage = numberOfLinesInFile * rules.numberOfRulesIncludingCustomRules + var observedCoverage = maximumCoverage + for region in regions { + observedCoverage -= region.reducesCoverageBy( + numberOfLinesInFile: numberOfLinesInFile, + rules: rules, + ruleIdentifiers: ruleIdentifiers + ) + } + + return Coverage.Coverage( + numberOfLinesOfCode: numberOfLinesInFile, + observedCoverage: observedCoverage, + maximumCoverage: maximumCoverage + ) + } +} + +private extension Region { + func reducesCoverageBy(numberOfLinesInFile: Int, rules: [any Rule], ruleIdentifiers: Set) -> Int { + guard disabledRuleIdentifiers.isNotEmpty else { + return 0 + } + + let numberOfLinesInRegion = numberOfLines(numberOfLinesInFile: numberOfLinesInFile) + if disabledRuleIdentifiers.contains(.all) { + return numberOfLinesInRegion * rules.numberOfRulesIncludingCustomRules + } + + let disabledRuleIdentifiers = Set(disabledRuleIdentifiers.map { $0.stringRepresentation }) + let numberOfDisabledRules: Int = if disabledRuleIdentifiers.contains(CustomRules.identifier) { + disabledRuleIdentifiers.subtracting( + Set(rules.customRuleIdentifiers + [CustomRules.identifier]) + ).numberOfDisabledRules(from: ruleIdentifiers, rules: rules) + rules.customRuleIdentifiers.count + } else { + disabledRuleIdentifiers.numberOfDisabledRules(from: ruleIdentifiers, rules: rules) + } + return numberOfLinesInRegion * numberOfDisabledRules + } + + private func numberOfLines(numberOfLinesInFile: Int) -> Int { + end.line == .max ? + numberOfLinesInFile - (start.line ?? numberOfLinesInFile) : + max((end.line ?? 0) - (start.line ?? 0), 1) + } +} + +private extension Set { + func numberOfDisabledRules(from ruleIdentifiers: Self, rules: [any Rule]) -> Int { + var remainingDisabledIdentifiers = intersection(ruleIdentifiers) + + // Check whether there is more than one match, or more ruleIdentifiers than there are rules + // We do not need to worry about `custom_rules` being used to disable all custom rules + // as that is taken care of by the caller. + guard remainingDisabledIdentifiers.count > 1, ruleIdentifiers.count > rules.count else { + return remainingDisabledIdentifiers.count + } + // Have all possible identifiers been specified? + guard remainingDisabledIdentifiers.count < ruleIdentifiers.count else { + return rules.count + } + // We need to handle aliases and custom rules specially. + var numberOfDisabledRules = 0 + for rule in rules { + let allRuleIdentifiers = type(of: rule).description.allIdentifiers + [rule].customRuleIdentifiers + if !remainingDisabledIdentifiers.isDisjoint(with: allRuleIdentifiers) { + if rule is CustomRules { + numberOfDisabledRules += remainingDisabledIdentifiers.intersection(allRuleIdentifiers).count + } else { + numberOfDisabledRules += 1 + } + remainingDisabledIdentifiers.subtract(allRuleIdentifiers) + + // If there is only one identifier left, it must match one rule. + if remainingDisabledIdentifiers.count == 1 { + return numberOfDisabledRules + 1 + } + if remainingDisabledIdentifiers.isEmpty { + return numberOfDisabledRules + } + } + } + return numberOfDisabledRules + } +} + +private extension Configuration { + func totalNumberOfRules(for mode: LintOrAnalyzeMode) -> Int { + RuleRegistry.shared.totalNumberOfRules(for: mode) + max(rules.customRuleIdentifiers.count - 1, 0) + } +} + +private extension RuleRegistry { + func totalNumberOfRules(for mode: LintOrAnalyzeMode) -> Int { + list.list.filter({ ($1 is any AnalyzerRule.Type) == (mode == .analyze) }).count + } +} + +private extension [any Rule] { + var ruleIdentifiers: [String] { + Set(flatMap { type(of: $0).description.allIdentifiers }) + customRuleIdentifiers + } + var customRuleIdentifiers: [String] { + (first { $0 is CustomRules } as? CustomRules)?.customRuleIdentifiers ?? [] + } + var numberOfRulesIncludingCustomRules: Int { + count + Swift.max(customRuleIdentifiers.count - 1, 0) + } +} + +private extension Double { + func rounded(toNearestPlaces places: Int) -> Double { + let divisor = pow(10.0, Double(places)) + return (self * divisor).rounded() / divisor + } +} diff --git a/Source/SwiftLintFramework/LintOrAnalyzeCommand.swift b/Source/SwiftLintFramework/LintOrAnalyzeCommand.swift index 3f2bcdfbcc..2bb4fa44df 100644 --- a/Source/SwiftLintFramework/LintOrAnalyzeCommand.swift +++ b/Source/SwiftLintFramework/LintOrAnalyzeCommand.swift @@ -40,6 +40,7 @@ package struct LintOrAnalyzeOptions { let useScriptInputFiles: Bool let useScriptInputFileLists: Bool let benchmark: Bool + let reportCoverage: Bool let reporter: String? let baseline: String? let writeBaseline: String? @@ -68,6 +69,7 @@ package struct LintOrAnalyzeOptions { useScriptInputFiles: Bool, useScriptInputFileLists: Bool, benchmark: Bool, + reportCoverage: Bool, reporter: String?, baseline: String?, writeBaseline: String?, @@ -95,6 +97,7 @@ package struct LintOrAnalyzeOptions { self.useScriptInputFiles = useScriptInputFiles self.useScriptInputFileLists = useScriptInputFileLists self.benchmark = benchmark + self.reportCoverage = reportCoverage self.reporter = reporter self.baseline = baseline self.writeBaseline = writeBaseline @@ -148,6 +151,9 @@ package struct LintOrAnalyzeCommand { let numberOfSeriousViolations = try Signposts.record(name: "LintOrAnalyzeCommand.PostProcessViolations") { try postProcessViolations(files: files, builder: builder) } + if let coverage = builder.coverage { + queuedPrint(coverage.report) + } if options.checkForUpdates || builder.configuration.checkForUpdates { await UpdateChecker.checkForUpdates() } @@ -186,6 +192,11 @@ package struct LintOrAnalyzeCommand { violations: linter.styleViolations(using: builder.storage) ) } + + visitorMutationQueue.sync { + builder.coverage?.addCoverage(for: linter) + } + let filteredViolations = baseline?.filter(currentViolations) ?? currentViolations visitorMutationQueue.sync { builder.unfilteredViolations += currentViolations @@ -360,6 +371,9 @@ package struct LintOrAnalyzeCommand { private class LintOrAnalyzeResultBuilder { var fileBenchmark = Benchmark(name: "files") var ruleBenchmark = Benchmark(name: "rules") + + var coverage: Coverage? + /// All detected violations, unfiltered by the baseline, if any. var unfilteredViolations = [StyleViolation]() /// The violations to be reported, possibly filtered by a baseline, plus any threshold violations. @@ -390,6 +404,10 @@ private class LintOrAnalyzeResultBuilder { Issue.fileNotWritable(path: outFile).print() } } + + if configuration.reportCoverage || options.reportCoverage { + coverage = Coverage(mode: options.mode, configuration: configuration) + } } func report(violations: [StyleViolation], realtimeCondition: Bool) { diff --git a/Source/SwiftLintFramework/Models/Linter.swift b/Source/SwiftLintFramework/Models/Linter.swift index a307597d69..92ae814382 100644 --- a/Source/SwiftLintFramework/Models/Linter.swift +++ b/Source/SwiftLintFramework/Models/Linter.swift @@ -259,7 +259,7 @@ public struct Linter { public struct CollectedLinter { /// The file to lint with this linter. public let file: SwiftLintFile - private let rules: [any Rule] + let rules: [any Rule] private let cache: LinterCache? private let configuration: Configuration private let compilerArguments: [String] @@ -302,7 +302,7 @@ public struct CollectedLinter { return cached } - let regions = file.regions() + let regions = file.regions let superfluousDisableCommandRule = rules.first(where: { $0 is SuperfluousDisableCommandRule }) as? SuperfluousDisableCommandRule @@ -439,7 +439,7 @@ public struct CollectedLinter { } } -private extension SwiftLintFile { +extension SwiftLintFile { var isEmpty: Bool { contents.isEmpty || contents == "\n" } diff --git a/Source/swiftlint/Commands/Analyze.swift b/Source/swiftlint/Commands/Analyze.swift index 5ae0f3ff3c..30823287a3 100644 --- a/Source/swiftlint/Commands/Analyze.swift +++ b/Source/swiftlint/Commands/Analyze.swift @@ -31,6 +31,7 @@ extension SwiftLint { useScriptInputFiles: common.useScriptInputFiles, useScriptInputFileLists: common.useScriptInputFileLists, benchmark: common.benchmark, + reportCoverage: common.reportCoverage, reporter: common.reporter, baseline: common.baseline, writeBaseline: common.writeBaseline, diff --git a/Source/swiftlint/Commands/Lint.swift b/Source/swiftlint/Commands/Lint.swift index eb267c7148..5c59592681 100644 --- a/Source/swiftlint/Commands/Lint.swift +++ b/Source/swiftlint/Commands/Lint.swift @@ -43,6 +43,7 @@ extension SwiftLint { useScriptInputFiles: common.useScriptInputFiles, useScriptInputFileLists: common.useScriptInputFileLists, benchmark: common.benchmark, + reportCoverage: common.reportCoverage, reporter: common.reporter, baseline: common.baseline, writeBaseline: common.writeBaseline, diff --git a/Source/swiftlint/Common/LintOrAnalyzeArguments.swift b/Source/swiftlint/Common/LintOrAnalyzeArguments.swift index e956dd13ba..95e7dcaea8 100644 --- a/Source/swiftlint/Common/LintOrAnalyzeArguments.swift +++ b/Source/swiftlint/Common/LintOrAnalyzeArguments.swift @@ -39,6 +39,8 @@ struct LintOrAnalyzeArguments: ParsableArguments { var forceExclude = false @Flag(help: "Save benchmarks to `benchmark_files.txt` and `benchmark_rules.txt`.") var benchmark = false + @Flag(help: "Print coverage statistics.") + var reportCoverage = false @Option(help: "The reporter used to log errors and warnings.") var reporter: String? @Option(help: "The path to a baseline file, which will be used to filter out detected violations.") diff --git a/Tests/FrameworkTests/CoverageTests.swift b/Tests/FrameworkTests/CoverageTests.swift new file mode 100644 index 0000000000..0d833c5a3a --- /dev/null +++ b/Tests/FrameworkTests/CoverageTests.swift @@ -0,0 +1,299 @@ +@testable import SwiftLintBuiltInRules +@testable import SwiftLintCore +@testable import SwiftLintFramework +import XCTest + +final class CoverageTests: SwiftLintTestCase { + private static let ruleIdentifiers: [String] = [ + ArrayInitRule.identifier, + BlockBasedKVORule.identifier, + ClosingBraceRule.identifier, + DirectReturnRule.identifier, + ] + + private static let customRuleIdentifier1 = "custom1" + private static let customRuleIdentifier2 = "custom2" + private static let customRulesConfiguration = [ + customRuleIdentifier1: ["regex": "pattern"], + customRuleIdentifier2: ["regex": "something"], + ] + + func testEmptySourceCoverage() throws { + try testCoverage(source: "") + } + + func testBlankLineSourceCoverage() throws { + try testCoverage(source: "\n") + try testCoverage(source: "\n\n", observedCoverage: 8, maximumCoverage: 8) + } + + func testNoRulesCoverage() throws { + try testCoverage(for: [], source: "\n") + } + + func testNoDisabledCommandCoverage() throws { + let source = """ + func foo() -> Int { + return 0 + } + """ + + try testCoverage(source: source, observedCoverage: 12, maximumCoverage: 12) + } + + func testDisableAllCoverage() throws { + // The `disable` command line will still be linted, so coverage will not be zero. + try testCoverage( + withDisabledRuleIdentifiers: [RuleIdentifier.all.stringRepresentation], + observedCoverage: 4, + maximumCoverage: 40 + ) + } + + func testCoverageWithRegions() throws { + let enabledRuleRegionSource = """ + func foo() -> Int { + // swiftlint:disable:next direct_return + return 0 + } + + // These blank lines keep the linecount consistent + """ + + let expectedObservedCoverage = 23 + let expectedMaximumCoverage = 24 + + try testCoverage( + source: enabledRuleRegionSource, + observedCoverage: expectedObservedCoverage, + maximumCoverage: expectedMaximumCoverage + ) + + let irrelevantRegionsSource = enabledRuleRegionSource.replacingLastLine( + with: "// swiftlint:disable:previous expiring_todo" + ) + + try testCoverage( + source: irrelevantRegionsSource, + observedCoverage: expectedObservedCoverage, + maximumCoverage: expectedMaximumCoverage + ) + + let overlappingRegionSource = """ + func foo() -> Int { + // swiftlint:disable:next direct_return + return 0 // swiftlint:disable:this direct_return + } // swiftlint:disable:previous direct_return + + // These blank lines keep the linecount consistent + """ + + try testCoverage( + source: overlappingRegionSource, + observedCoverage: expectedObservedCoverage, + maximumCoverage: expectedMaximumCoverage + ) + } + + func testFinalEnable() throws { + let source = """ + // swiftlint:disable direct_return + + // swiftlint:enable direct_return + """ + try testCoverage(source: source, observedCoverage: 10, maximumCoverage: 12) + } + + func testNestedAndOverlappingRegions() throws { + let enabledRuleIdentifiers = Self.ruleIdentifiers + [CustomRules.identifier] + + let source = """ + // swiftlint:disable \(Self.customRuleIdentifier1) + // swiftlint:disable array_init + // swiftlint:disable \(Self.customRuleIdentifier2) direct_return + + // swiftlint:enable array_init direct_return + // swiftlint:enable \(Self.customRuleIdentifier2) + // + // swiftlint:enable \(Self.customRuleIdentifier1) + """ + + try testCoverage( + for: enabledRuleIdentifiers, + customRules: Self.customRulesConfiguration, + source: source, + observedCoverage: 33, + maximumCoverage: 48 + ) + } + + func testCoverageWithCustomRules() throws { + let enabledRuleIdentifiers: [String] = [CustomRules.identifier, ArrayInitRule.identifier] + + try testCoverage( + withDisabledRuleIdentifiers: [RuleIdentifier.all.stringRepresentation], + for: enabledRuleIdentifiers, + customRules: Self.customRulesConfiguration, + observedCoverage: 3, + maximumCoverage: 30 + ) + + let customRulesIdentifier = CustomRules.identifier + let firstCustomRuleIdentifier = Self.customRuleIdentifier1 + let secondCustomRuleIdentifier = Self.customRuleIdentifier2 + + let disabledRuleIdentifiers = [ + [customRulesIdentifier], + [customRulesIdentifier, firstCustomRuleIdentifier], + [customRulesIdentifier, secondCustomRuleIdentifier], + [customRulesIdentifier, firstCustomRuleIdentifier, secondCustomRuleIdentifier], + [firstCustomRuleIdentifier, secondCustomRuleIdentifier], + ] + + try disabledRuleIdentifiers.forEach { + try testCoverage( + withDisabledRuleIdentifiers: $0, + for: enabledRuleIdentifiers, + customRules: Self.customRulesConfiguration, + observedCoverage: 12, + maximumCoverage: 30 + ) + } + + try [firstCustomRuleIdentifier, secondCustomRuleIdentifier].forEach { + try testCoverage( + withDisabledRuleIdentifiers: [$0], + for: enabledRuleIdentifiers, + customRules: Self.customRulesConfiguration, + observedCoverage: 21, + maximumCoverage: 30 + ) + } + } + + func testFinalEnableWithCustomRules() throws { + let source = """ + // swiftlint:disable \(Self.customRuleIdentifier1) + + // swiftlint:enable \(Self.customRuleIdentifier1) + """ + let enabledRuleIdentifiers = Self.ruleIdentifiers + [CustomRules.identifier] + + try testCoverage( + for: enabledRuleIdentifiers, + customRules: Self.customRulesConfiguration, + source: source, + observedCoverage: 16, + maximumCoverage: 18 + ) + + let customRulesConfiguration: [String: [String: String]] = { + [Self.customRuleIdentifier1: Self.customRulesConfiguration[Self.customRuleIdentifier1]!] + }() + + try testCoverage( + for: enabledRuleIdentifiers, + customRules: customRulesConfiguration, + source: source, + observedCoverage: 13, + maximumCoverage: 15 + ) + } + + func testRuleAliasesCoverage() throws { + let enabledRuleIdentifiers = Array(Self.ruleIdentifiers.dropLast() + [ShorthandOptionalBindingRule.identifier]) + let disabledRuleIdentifiers = ShorthandOptionalBindingRule.description.allIdentifiers + XCTAssertGreaterThan(disabledRuleIdentifiers.count, 1) + let maximumCoverage = 40 + try testCoverage( + withDisabledRuleIdentifiers: disabledRuleIdentifiers, + for: enabledRuleIdentifiers, + observedCoverage: 31, + maximumCoverage: maximumCoverage + ) + + try testCoverage( + withDisabledRuleIdentifiers: Array(Set(enabledRuleIdentifiers + disabledRuleIdentifiers)), + for: enabledRuleIdentifiers, + observedCoverage: 4, + maximumCoverage: maximumCoverage + ) + } + + func testCoverageReport() throws { + let source = """ + func foo() -> Int { + return 0 // swiftlint:disable:this direct_return + } + """ + + let coverage = try coverage(file: SwiftLintFile(contents: source)) + let expectedReport = """ + Enabled rules coverage: 0.917 + """ + let report = coverage.report.components(separatedBy: "\n").dropLast().joined(separator: "\n") + XCTAssertEqual(report, expectedReport) + } + + // MARK: - Private + private func testCoverage( + for enabledRuleIdentifiers: [String] = CoverageTests.ruleIdentifiers, + customRules: [String: [String: String]] = [:], + source: String, + observedCoverage: Int = 0, + maximumCoverage: Int = 0 + ) throws { + let file = SwiftLintFile(contents: source) + let coverage = try coverage( + for: enabledRuleIdentifiers, + customRules: customRules, + file: file + ) + let expectedCoverage = Coverage.Coverage( + numberOfLinesOfCode: file.isEmpty ? 0 : file.lines.count, + observedCoverage: observedCoverage, + maximumCoverage: maximumCoverage + ) + XCTAssertEqual(coverage.coverage, expectedCoverage) + } + + private func coverage( + for enabledRuleIdentifiers: [String] = CoverageTests.ruleIdentifiers, + customRules: [String: [String: String]] = [:], + file: SwiftLintFile + ) throws -> Coverage { + var configurationDictionary: [String: Any] = ["only_rules": enabledRuleIdentifiers] + if customRules.isNotEmpty { + configurationDictionary[CustomRules.identifier] = customRules + } + let configuration = try Configuration(dict: configurationDictionary) + var coverage = Coverage(mode: .lint, configuration: configuration) + let linter = Linter(file: file, configuration: configuration) + let collectedLinter = linter.collect(into: RuleStorage()) + coverage.addCoverage(for: collectedLinter) + return coverage + } + + private func testCoverage( + withDisabledRuleIdentifiers disabledRuleIdentifiers: [String], + for enabledRuleIdentifiers: [String] = CoverageTests.ruleIdentifiers, + customRules: [String: [String: String]] = [:], + observedCoverage: Int, + maximumCoverage: Int + ) throws { + let filler = String(repeating: "\n", count: 10) + try testCoverage( + for: enabledRuleIdentifiers, + customRules: customRules, + source: "// swiftlint:disable \(disabledRuleIdentifiers.joined(separator: " "))" + filler, + observedCoverage: observedCoverage, + maximumCoverage: maximumCoverage + ) + } +} + +private extension String { + func replacingLastLine(with string: String) -> String { + components(separatedBy: "\n").dropLast().joined(separator: "\n") + "\n\(string)" + } +} diff --git a/Tests/FrameworkTests/LintOrAnalyzeOptionsTests.swift b/Tests/FrameworkTests/LintOrAnalyzeOptionsTests.swift index 07ec490791..5435424ca5 100644 --- a/Tests/FrameworkTests/LintOrAnalyzeOptionsTests.swift +++ b/Tests/FrameworkTests/LintOrAnalyzeOptionsTests.swift @@ -51,6 +51,7 @@ private extension LintOrAnalyzeOptions { useScriptInputFiles: false, useScriptInputFileLists: false, benchmark: false, + reportCoverage: false, reporter: nil, baseline: nil, writeBaseline: nil, diff --git a/Tests/FrameworkTests/RegionTests.swift b/Tests/FrameworkTests/RegionTests.swift index 3935f583c7..3fd3d284fc 100644 --- a/Tests/FrameworkTests/RegionTests.swift +++ b/Tests/FrameworkTests/RegionTests.swift @@ -6,12 +6,12 @@ final class RegionTests: SwiftLintTestCase { func testNoRegionsInEmptyFile() { let file = SwiftLintFile(contents: "") - XCTAssertEqual(file.regions(), []) + XCTAssertEqual(file.regions, []) } func testNoRegionsInFileWithNoCommands() { let file = SwiftLintFile(contents: String(repeating: "\n", count: 100)) - XCTAssertEqual(file.regions(), []) + XCTAssertEqual(file.regions, []) } func testRegionsFromSingleCommand() { @@ -20,14 +20,14 @@ final class RegionTests: SwiftLintTestCase { let file = SwiftLintFile(contents: "// swiftlint:disable rule_id\n") let start = Location(file: nil, line: 1, character: 29) let end = Location(file: nil, line: .max, character: .max) - XCTAssertEqual(file.regions(), [Region(start: start, end: end, disabledRuleIdentifiers: ["rule_id"])]) + XCTAssertEqual(file.regions, [Region(start: start, end: end, disabledRuleIdentifiers: ["rule_id"])]) } // enable do { let file = SwiftLintFile(contents: "// swiftlint:enable rule_id\n") let start = Location(file: nil, line: 1, character: 28) let end = Location(file: nil, line: .max, character: .max) - XCTAssertEqual(file.regions(), [Region(start: start, end: end, disabledRuleIdentifiers: [])]) + XCTAssertEqual(file.regions, [Region(start: start, end: end, disabledRuleIdentifiers: [])]) } } @@ -35,7 +35,7 @@ final class RegionTests: SwiftLintTestCase { // disable/enable do { let file = SwiftLintFile(contents: "// swiftlint:disable rule_id\n// swiftlint:enable rule_id\n") - XCTAssertEqual(file.regions(), [ + XCTAssertEqual(file.regions, [ Region(start: Location(file: nil, line: 1, character: 29), end: Location(file: nil, line: 2, character: 27), disabledRuleIdentifiers: ["rule_id"]), @@ -47,7 +47,7 @@ final class RegionTests: SwiftLintTestCase { // enable/disable do { let file = SwiftLintFile(contents: "// swiftlint:enable rule_id\n// swiftlint:disable rule_id\n") - XCTAssertEqual(file.regions(), [ + XCTAssertEqual(file.regions, [ Region(start: Location(file: nil, line: 1, character: 28), end: Location(file: nil, line: 2, character: 28), disabledRuleIdentifiers: []), @@ -62,7 +62,7 @@ final class RegionTests: SwiftLintTestCase { let file = SwiftLintFile(contents: "// swiftlint:disable:next 1\n" + "// swiftlint:disable:this 2\n" + "// swiftlint:disable:previous 3\n") - XCTAssertEqual(file.regions(), [ + XCTAssertEqual(file.regions, [ Region(start: Location(file: nil, line: 2, character: nil), end: Location(file: nil, line: 2, character: .max - 1), disabledRuleIdentifiers: ["1", "2", "3"]), @@ -79,7 +79,7 @@ final class RegionTests: SwiftLintTestCase { "// swiftlint:enable 1\n" + "// swiftlint:enable 2\n" + "// swiftlint:enable 3\n") - XCTAssertEqual(file.regions(), [ + XCTAssertEqual(file.regions, [ Region(start: Location(file: nil, line: 1, character: 23), end: Location(file: nil, line: 2, character: 22), disabledRuleIdentifiers: ["1"]),