Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: configuration validation #2133

Draft
wants to merge 25 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
0f394b3
feat: configuration validation (#1778)
maschad Oct 5, 2023
e8f3158
Merge branch 'master' into feat/config-validation
maschad Oct 8, 2023
80fd92e
refactor: updated functions according to pr comments
maschad Oct 9, 2023
5ccd6bf
chore: ensure inboundConnectionThreshold is a number
maschad Oct 9, 2023
94e089b
Merge branch 'master' into feat/config-validation
achingbrain Oct 10, 2023
4ece976
refactor: address PR comments
maschad Oct 10, 2023
78a2c4a
chore: refactor variable name from pr feedback
maschad Oct 13, 2023
e3046d8
Merge branch 'main' into feat/config-validation
maschad Nov 18, 2023
cba7c2f
refactor: updated config validation
maschad Nov 20, 2023
7f38cdc
Merge branch 'main' into feat/config-validation
maschad Nov 20, 2023
355824d
deps: add yup as dep
maschad Nov 21, 2023
6ef1271
chore: fix identify issue
maschad Nov 21, 2023
e836612
Merge branch 'main' into feat/config-validation
maschad Nov 22, 2023
e25bdb8
fix: remove max parallel dials per peer param
maschad Nov 22, 2023
9ff9ad9
Merge branch 'main' into feat/config-validation
maschad Nov 23, 2023
cf3dd84
fix: update transport with config
maschad Nov 23, 2023
1770c3d
feat: add config validation to perf service
maschad Nov 23, 2023
f8e3193
fix: update config to sort addresses properly
maschad Nov 24, 2023
fd3a1b1
Merge branch 'main' into feat/config-validation
maschad Dec 7, 2023
94e19d8
fix: fixes related to PR feedback
maschad Nov 29, 2023
d1c1f25
fix: update error + transport imports
maschad Dec 7, 2023
c63c235
Merge branch 'main' into feat/config-validation
maschad Dec 12, 2023
ce55e11
Merge branch 'main' into feat/config-validation
maschad Jan 11, 2024
1eb436e
adjust imports
maschad Jan 11, 2024
1b072f4
Merge remote-tracking branch 'origin/main' into feat/config-validation
achingbrain Jan 12, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions packages/integration-tests/.aegir.js
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ export default {
const peerId = await createEd25519PeerId()
const libp2p = await createLibp2p({
connectionManager: {
inboundConnectionThreshold: Infinity,
inboundConnectionThreshold: 10000,
minConnections: 0
},
addresses: {
Expand All @@ -47,7 +47,7 @@ export default {
identify: identify(),
relay: circuitRelayServer({
reservations: {
maxReservations: Infinity
maxReservations: 10000
}
})
}
Expand Down
6 changes: 4 additions & 2 deletions packages/libp2p/.aegir.js
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@ export default {
const peerId = await createEd25519PeerId()
const libp2p = await createLibp2p({
connectionManager: {
inboundConnectionThreshold: Infinity,
inboundConnectionThreshold: 1000,
maxIncomingPendingConnections: 1000,
maxConnections: 1000,
minConnections: 0
},
addresses: {
Expand All @@ -44,7 +46,7 @@ export default {
identify: identify(),
relay: circuitRelayServer({
reservations: {
maxReservations: Infinity
maxReservations: 100000
}
}),
echo: echo()
Expand Down
3 changes: 2 additions & 1 deletion packages/libp2p/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,8 @@
"merge-options": "^3.0.4",
"multiformats": "^13.0.0",
"private-ip": "^3.0.1",
"uint8arrays": "^5.0.0"
"uint8arrays": "^5.0.0",
"yup": "^1.3.2"
},
"devDependencies": {
"@chainsafe/libp2p-yamux": "^6.0.1",
Expand Down
13 changes: 13 additions & 0 deletions packages/libp2p/src/address-manager/utils.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
import { type ObjectSchema, object, array, string, mixed } from 'yup'
import { validateMultiaddr } from '../config/helpers.js'
import type { Multiaddr } from '@multiformats/multiaddr'

export function debounce (func: () => void, wait: number): () => void {
let timeout: ReturnType<typeof setTimeout> | undefined

Expand All @@ -11,3 +15,12 @@ export function debounce (func: () => void, wait: number): () => void {
timeout = setTimeout(later, wait)
}
}

export function validateAddressManagerConfig (): ObjectSchema<Record<string, unknown>> {
return object({
listen: array().of(string()).test('is multiaddr', validateMultiaddr).default([]),
announce: array().of(string()).test('is multiaddr', validateMultiaddr).default([]),
noAnnounce: array().of(string()).test('is multiaddr', validateMultiaddr).default([]),
announceFilter: mixed().default(() => (addrs: Multiaddr[]): Multiaddr[] => addrs)
})
}
36 changes: 0 additions & 36 deletions packages/libp2p/src/config.ts

This file was deleted.

35 changes: 35 additions & 0 deletions packages/libp2p/src/config/config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
import { FaultTolerance } from '@libp2p/interface'
import { defaultAddressSort } from '@libp2p/utils/address-sort'
import { dnsaddrResolver } from '@multiformats/multiaddr/resolvers'
import mergeOptions from 'merge-options'
import { object } from 'yup'
import { validateAddressManagerConfig } from '../address-manager/utils.js'
import { validateConnectionManagerConfig } from '../connection-manager/utils.js'
import type { ConnectionManagerInit } from '../connection-manager/index.js'
import type { Libp2pInit } from '../index.js'
import type { ServiceMap, RecursivePartial } from '@libp2p/interface'

const defaultConfig: Partial<Libp2pInit> = {
connectionManager: {
resolvers: {
dnsaddr: dnsaddrResolver
},
addressSorter: defaultAddressSort
},
transportManager: {
faultTolerance: FaultTolerance.FATAL_ALL
}
}

export function validateConfig<T extends ServiceMap = Record<string, unknown>> (opts: RecursivePartial<Libp2pInit<T>>): Libp2pInit<T> {
const libp2pConfig = object({
addresses: validateAddressManagerConfig(),
connectionManager: validateConnectionManagerConfig(opts?.connectionManager as ConnectionManagerInit)
})

const parsedOpts = libp2pConfig.validateSync(opts)

const resultingOptions: Libp2pInit<T> = mergeOptions(defaultConfig, parsedOpts)

return resultingOptions
}
19 changes: 19 additions & 0 deletions packages/libp2p/src/config/helpers.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import { CodeError } from '@libp2p/interface'
import { multiaddr } from '@multiformats/multiaddr'
import { codes } from '../errors.js'

export const validateMultiaddr = (value: Array<string | undefined> | undefined): boolean => {
if (value == null || value === undefined) {
return false
}

Check warning on line 8 in packages/libp2p/src/config/helpers.ts

View check run for this annotation

Codecov / codecov/patch

packages/libp2p/src/config/helpers.ts#L7-L8

Added lines #L7 - L8 were not covered by tests

value?.forEach((addr) => {
try {
multiaddr(addr)
} catch (err) {
throw new CodeError(`invalid multiaddr: ${addr}`, codes.ERR_INVALID_MULTIADDR)
}

Check warning on line 15 in packages/libp2p/src/config/helpers.ts

View check run for this annotation

Codecov / codecov/patch

packages/libp2p/src/config/helpers.ts#L14-L15

Added lines #L14 - L15 were not covered by tests
})

return true
maschad marked this conversation as resolved.
Show resolved Hide resolved
}
23 changes: 23 additions & 0 deletions packages/libp2p/src/connection-manager/utils.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
import { type AbortOptions, multiaddr, type Multiaddr } from '@multiformats/multiaddr'
import { type ObjectSchema, array, number, object, string } from 'yup'
import { validateMultiaddr } from '../config/helpers.js'
import { AUTO_DIAL_INTERVAL, AUTO_DIAL_CONCURRENCY, AUTO_DIAL_PRIORITY, MAX_PEER_ADDRS_TO_DIAL, DIAL_TIMEOUT, INBOUND_UPGRADE_TIMEOUT, INBOUND_CONNECTION_THRESHOLD, MAX_INCOMING_PENDING_CONNECTIONS } from './constants.defaults.js'
import { MIN_CONNECTIONS, MAX_CONNECTIONS, MAX_PARALLEL_DIALS } from './constants.js'
import type { ConnectionManagerInit } from '.'
import type { LoggerOptions } from '@libp2p/interface'

/**
Expand Down Expand Up @@ -45,3 +50,21 @@ async function resolveRecord (ma: Multiaddr, options: AbortOptions & LoggerOptio
return []
}
}

export const validateConnectionManagerConfig = (opts: ConnectionManagerInit): ObjectSchema<Record<string, unknown>> => {
return object({
maxConnections: number().integer().min(opts?.minConnections ?? MIN_CONNECTIONS, `maxConnections must be greater than or equal to minConnections: ${opts?.minConnections ?? MIN_CONNECTIONS}`).default(MAX_CONNECTIONS),
maschad marked this conversation as resolved.
Show resolved Hide resolved
minConnections: number().integer().min(0).max(opts?.maxConnections ?? MAX_CONNECTIONS, `minConnections must be less than or equal to maxConnections : ${opts?.maxConnections ?? MAX_CONNECTIONS}`).default(MIN_CONNECTIONS),
autoDialInterval: number().integer().min(0).default(AUTO_DIAL_INTERVAL),
autoDialConcurrency: number().integer().min(0).default(AUTO_DIAL_CONCURRENCY),
autoDialPriority: number().integer().min(0).default(AUTO_DIAL_PRIORITY),
maxParallelDials: number().integer().min(0).default(MAX_PARALLEL_DIALS),
maxPeerAddrsToDialed: number().integer().min(0).default(MAX_PEER_ADDRS_TO_DIAL),
dialTimeout: number().integer().min(0).default(DIAL_TIMEOUT),
inboundUpgradeTimeout: number().integer().min(0).default(INBOUND_UPGRADE_TIMEOUT),
allow: array().of(string()).test('is multiaddr', validateMultiaddr).default([]),
deny: array().of(string()).test('is multiaddr', validateMultiaddr).default([]),
inboundConnectionThreshold: number().integer().min(0).default(INBOUND_CONNECTION_THRESHOLD),
maxIncomingPendingConnections: number().integer().min(0).default(MAX_INCOMING_PENDING_CONNECTIONS)
})
}
2 changes: 1 addition & 1 deletion packages/libp2p/src/libp2p.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ import { concat as uint8ArrayConcat } from 'uint8arrays/concat'
import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
import { DefaultAddressManager } from './address-manager/index.js'
import { defaultComponents } from './components.js'
import { validateConfig } from './config/config.js'
import { connectionGater } from './config/connection-gater.js'
import { validateConfig } from './config.js'
import { DefaultConnectionManager } from './connection-manager/index.js'
import { CompoundContentRouting } from './content-routing.js'
import { codes } from './errors.js'
Expand Down
4 changes: 3 additions & 1 deletion packages/libp2p/test/connection-manager/index.node.ts
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,9 @@ describe('libp2p.connections', () => {
},
connectionManager: {
minConnections,
maxConnections: 1
maxConnections: 1,
inboundConnectionThreshold: 1,
maxIncomingPendingConnections: 1
}
}
})
Expand Down
3 changes: 2 additions & 1 deletion packages/protocol-autonat/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@
"it-pipe": "^3.0.1",
"private-ip": "^3.0.1",
"protons-runtime": "^5.0.0",
"uint8arraylist": "^2.4.7"
"uint8arraylist": "^2.4.7",
"yup": "^1.3.2"
},
"devDependencies": {
"@libp2p/logger": "^4.0.4",
Expand Down
26 changes: 19 additions & 7 deletions packages/protocol-autonat/src/autonat.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import map from 'it-map'
import parallel from 'it-parallel'
import { pipe } from 'it-pipe'
import isPrivateIp from 'private-ip'
import { number, object, string } from 'yup'
import {
MAX_INBOUND_STREAMS,
MAX_OUTBOUND_STREAMS,
Expand All @@ -23,6 +24,15 @@ import type { IncomingStreamData } from '@libp2p/interface-internal'
// https://github.com/libp2p/specs/blob/master/autonat/README.md#autonat-protocol
const REQUIRED_SUCCESSFUL_DIALS = 4

const configValidator = object({
protocolPrefix: string().default(PROTOCOL_PREFIX),
timeout: number().integer().default(TIMEOUT),
startupDelay: number().integer().default(STARTUP_DELAY),
refreshInterval: number().integer().default(REFRESH_INTERVAL),
maxInboundStreams: number().integer().default(MAX_INBOUND_STREAMS),
maxOutboundStreams: number().integer().default(MAX_OUTBOUND_STREAMS)
})

export class AutoNATService implements Startable {
private readonly components: AutoNATComponents
private readonly startupDelay: number
Expand All @@ -36,15 +46,17 @@ export class AutoNATService implements Startable {
private readonly log: Logger

constructor (components: AutoNATComponents, init: AutoNATServiceInit) {
const config = configValidator.validateSync(init)

this.components = components
this.log = components.logger.forComponent('libp2p:autonat')
this.started = false
this.protocol = `/${init.protocolPrefix ?? PROTOCOL_PREFIX}/${PROTOCOL_NAME}/${PROTOCOL_VERSION}`
this.timeout = init.timeout ?? TIMEOUT
this.maxInboundStreams = init.maxInboundStreams ?? MAX_INBOUND_STREAMS
this.maxOutboundStreams = init.maxOutboundStreams ?? MAX_OUTBOUND_STREAMS
this.startupDelay = init.startupDelay ?? STARTUP_DELAY
this.refreshInterval = init.refreshInterval ?? REFRESH_INTERVAL
this.protocol = `/${config.protocolPrefix}/${PROTOCOL_NAME}/${PROTOCOL_VERSION}`
this.timeout = config.timeout
this.maxInboundStreams = config.maxInboundStreams
this.maxOutboundStreams = config.maxOutboundStreams
this.startupDelay = config.startupDelay
this.refreshInterval = config.refreshInterval
this._verifyExternalAddresses = this._verifyExternalAddresses.bind(this)
}

Expand Down Expand Up @@ -382,7 +394,7 @@ export class AutoNATService implements Startable {
const networkSegments: string[] = []

const verifyAddress = async (peer: PeerInfo): Promise<Message.DialResponse | undefined> => {
let onAbort = (): void => {}
let onAbort = (): void => { }

try {
this.log('asking %p to verify multiaddr', peer.id)
Expand Down
3 changes: 2 additions & 1 deletion packages/protocol-dcutr/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,8 @@
"it-protobuf-stream": "^1.1.1",
"private-ip": "^3.0.1",
"protons-runtime": "^5.0.0",
"uint8arraylist": "^2.4.7"
"uint8arraylist": "^2.4.7",
"yup": "^1.3.2"
},
"devDependencies": {
"aegir": "^42.0.0",
Expand Down
12 changes: 12 additions & 0 deletions packages/protocol-dcutr/src/constants.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
// https://github.com/libp2p/specs/blob/master/relay/DCUtR.md#rpc-messages
export const MAX_DCUTR_MESSAGE_SIZE = 1024 * 4
// ensure the dial has a high priority to jump to the head of the dial queue
export const DCUTR_DIAL_PRIORITY = 100

export const DEFAULT_MAX_INBOUND_STREAMS = 1

export const DEFAULT_MAX_OUTBOUND_STREAMS = 1

export const DEFAULT_TIMEOUT = 5000

export const DEFAULT_RETRIES = 3
31 changes: 15 additions & 16 deletions packages/protocol-dcutr/src/dcutr.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,26 +2,23 @@ import { CodeError, ERR_INVALID_MESSAGE } from '@libp2p/interface'
import { type Multiaddr, multiaddr } from '@multiformats/multiaddr'
import delay from 'delay'
import { pbStream } from 'it-protobuf-stream'
import { number, object } from 'yup'
import { DCUTR_DIAL_PRIORITY, DEFAULT_MAX_INBOUND_STREAMS, DEFAULT_MAX_OUTBOUND_STREAMS, DEFAULT_RETRIES, DEFAULT_TIMEOUT, MAX_DCUTR_MESSAGE_SIZE } from './constants.js'
import { HolePunch } from './pb/message.js'
import { isPublicAndDialable } from './utils.js'
import { multicodec } from './index.js'
import type { DCUtRServiceComponents, DCUtRServiceInit } from './index.js'
import type { Logger, Connection, Stream, PeerStore, Startable } from '@libp2p/interface'
import type { AddressManager, ConnectionManager, Registrar, TransportManager } from '@libp2p/interface-internal'

// https://github.com/libp2p/specs/blob/master/relay/DCUtR.md#rpc-messages
const MAX_DCUTR_MESSAGE_SIZE = 1024 * 4
// ensure the dial has a high priority to jump to the head of the dial queue
const DCUTR_DIAL_PRIORITY = 100

const defaultValues = {
const configValidator = object({
// https://github.com/libp2p/go-libp2p/blob/8d2e54e1637041d5cf4fac1e531287560bd1f4ac/p2p/protocol/holepunch/holepuncher.go#L27
timeout: 5000,
timeout: number().integer().min(0).default(DEFAULT_TIMEOUT),
// https://github.com/libp2p/go-libp2p/blob/8d2e54e1637041d5cf4fac1e531287560bd1f4ac/p2p/protocol/holepunch/holepuncher.go#L28
retries: 3,
maxInboundStreams: 1,
maxOutboundStreams: 1
}
retries: number().integer().min(0).default(DEFAULT_RETRIES),
maxInboundStreams: number().integer().min(0).default(DEFAULT_MAX_INBOUND_STREAMS),
maxOutboundStreams: number().integer().min(0).default(DEFAULT_MAX_OUTBOUND_STREAMS)
})

export class DefaultDCUtRService implements Startable {
private started: boolean
Expand All @@ -46,10 +43,12 @@ export class DefaultDCUtRService implements Startable {
this.connectionManager = components.connectionManager
this.transportManager = components.transportManager

this.timeout = init.timeout ?? defaultValues.timeout
this.retries = init.retries ?? defaultValues.retries
this.maxInboundStreams = init.maxInboundStreams ?? defaultValues.maxInboundStreams
this.maxOutboundStreams = init.maxOutboundStreams ?? defaultValues.maxOutboundStreams
const config = configValidator.validateSync(init)

this.timeout = config.timeout
this.retries = config.retries
this.maxInboundStreams = config.maxInboundStreams
this.maxOutboundStreams = config.maxOutboundStreams
}

isStarted (): boolean {
Expand Down Expand Up @@ -363,7 +362,7 @@ export class DefaultDCUtRService implements Startable {
}

output.push(ma)
} catch {}
} catch { }
}

return output
Expand Down
3 changes: 2 additions & 1 deletion packages/protocol-fetch/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,8 @@
"it-protobuf-stream": "^1.1.1",
"protons-runtime": "^5.0.0",
"uint8arraylist": "^2.4.7",
"uint8arrays": "^5.0.0"
"uint8arrays": "^5.0.0",
"yup": "^1.3.2"
},
"devDependencies": {
"@libp2p/logger": "^4.0.4",
Expand Down
5 changes: 5 additions & 0 deletions packages/protocol-fetch/src/constants.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
// https://github.com/libp2p/specs/tree/master/fetch#wire-protocol
export const PROTOCOL_VERSION = '0.0.1'
export const PROTOCOL_NAME = 'fetch'

export const MAX_INBOUND_STREAMS = 1
export const MAX_OUTBOUND_STREAMS = 1
export const TIMEOUT = 10000
export const PROTOCOL_PREFIX = 'libp2p'
Loading
Loading