Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: configuration validation #2133

Draft
wants to merge 25 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
0f394b3
feat: configuration validation (#1778)
maschad Oct 5, 2023
e8f3158
Merge branch 'master' into feat/config-validation
maschad Oct 8, 2023
80fd92e
refactor: updated functions according to pr comments
maschad Oct 9, 2023
5ccd6bf
chore: ensure inboundConnectionThreshold is a number
maschad Oct 9, 2023
94e089b
Merge branch 'master' into feat/config-validation
achingbrain Oct 10, 2023
4ece976
refactor: address PR comments
maschad Oct 10, 2023
78a2c4a
chore: refactor variable name from pr feedback
maschad Oct 13, 2023
e3046d8
Merge branch 'main' into feat/config-validation
maschad Nov 18, 2023
cba7c2f
refactor: updated config validation
maschad Nov 20, 2023
7f38cdc
Merge branch 'main' into feat/config-validation
maschad Nov 20, 2023
355824d
deps: add yup as dep
maschad Nov 21, 2023
6ef1271
chore: fix identify issue
maschad Nov 21, 2023
e836612
Merge branch 'main' into feat/config-validation
maschad Nov 22, 2023
e25bdb8
fix: remove max parallel dials per peer param
maschad Nov 22, 2023
9ff9ad9
Merge branch 'main' into feat/config-validation
maschad Nov 23, 2023
cf3dd84
fix: update transport with config
maschad Nov 23, 2023
1770c3d
feat: add config validation to perf service
maschad Nov 23, 2023
f8e3193
fix: update config to sort addresses properly
maschad Nov 24, 2023
fd3a1b1
Merge branch 'main' into feat/config-validation
maschad Dec 7, 2023
94e19d8
fix: fixes related to PR feedback
maschad Nov 29, 2023
d1c1f25
fix: update error + transport imports
maschad Dec 7, 2023
c63c235
Merge branch 'main' into feat/config-validation
maschad Dec 12, 2023
ce55e11
Merge branch 'main' into feat/config-validation
maschad Jan 11, 2024
1eb436e
adjust imports
maschad Jan 11, 2024
1b072f4
Merge remote-tracking branch 'origin/main' into feat/config-validation
achingbrain Jan 12, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions packages/libp2p/.aegir.js
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@ export default {
const peerId = await createEd25519PeerId()
const libp2p = await createLibp2p({
connectionManager: {
inboundConnectionThreshold: Infinity,
inboundConnectionThreshold: 1000,
maxIncomingPendingConnections: 1000,
maxConnections: 1000,
minConnections: 0
},
addresses: {
Expand All @@ -51,7 +53,7 @@ export default {
fetch: fetchService(),
relay: circuitRelayServer({
reservations: {
maxReservations: Infinity
maxReservations: 100000
}
})
}
Expand Down
3 changes: 2 additions & 1 deletion packages/libp2p/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,8 @@
"uint8arraylist": "^2.4.3",
"uint8arrays": "^4.0.6",
"wherearewe": "^2.0.1",
"xsalsa20": "^1.1.0"
"xsalsa20": "^1.1.0",
"yup": "^1.2.0"
},
"devDependencies": {
"@chainsafe/libp2p-gossipsub": "^10.0.0",
Expand Down
14 changes: 14 additions & 0 deletions packages/libp2p/src/address-manager/utils.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
import { type ObjectSchema, object, array, string, mixed } from 'yup'
import { validateMultiaddr } from '../config/helpers.js'
import type { AddressManagerInit } from '.'
import type { Multiaddr } from '@multiformats/multiaddr'

export function debounce (func: () => void, wait: number): () => void {
let timeout: ReturnType<typeof setTimeout> | undefined

Expand All @@ -11,3 +16,12 @@ export function debounce (func: () => void, wait: number): () => void {
timeout = setTimeout(later, wait)
}
}

export function validateAddressManagerConfig (opts: AddressManagerInit): ObjectSchema<Record<string, unknown>> {
maschad marked this conversation as resolved.
Show resolved Hide resolved
return object({
listen: array().of(string()).test('is multiaddr', validateMultiaddr).default([]),
announce: array().of(string()).test('is multiaddr', validateMultiaddr).default([]),
noAnnounce: array().of(string()).test('is multiaddr', validateMultiaddr).default([]),
announceFilter: mixed().default(() => (addrs: Multiaddr[]): Multiaddr[] => addrs)
})
}
30 changes: 21 additions & 9 deletions packages/libp2p/src/autonat/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import map from 'it-map'
import parallel from 'it-parallel'
import { pipe } from 'it-pipe'
import isPrivateIp from 'private-ip'
import { number, object, string } from 'yup'
import { codes } from '../errors.js'
import {
MAX_INBOUND_STREAMS,
Expand Down Expand Up @@ -96,6 +97,15 @@ export interface AutoNATComponents {
peerRouting: PeerRouting
}

const configValidator = object({
protocolPrefix: string().default(PROTOCOL_PREFIX),
timeout: number().integer().default(TIMEOUT),
startupDelay: number().integer().default(STARTUP_DELAY),
refreshInterval: number().integer().default(REFRESH_INTERVAL),
maxInboundStreams: number().integer().default(MAX_INBOUND_STREAMS),
maxOutboundStreams: number().integer().default(MAX_OUTBOUND_STREAMS)
})

class DefaultAutoNATService implements Startable {
private readonly components: AutoNATComponents
private readonly startupDelay: number
Expand All @@ -108,14 +118,16 @@ class DefaultAutoNATService implements Startable {
private started: boolean

constructor (components: AutoNATComponents, init: AutoNATServiceInit) {
const validatedConfig = configValidator.validateSync(init)
maschad marked this conversation as resolved.
Show resolved Hide resolved

this.components = components
this.started = false
this.protocol = `/${init.protocolPrefix ?? PROTOCOL_PREFIX}/${PROTOCOL_NAME}/${PROTOCOL_VERSION}`
this.timeout = init.timeout ?? TIMEOUT
this.maxInboundStreams = init.maxInboundStreams ?? MAX_INBOUND_STREAMS
this.maxOutboundStreams = init.maxOutboundStreams ?? MAX_OUTBOUND_STREAMS
this.startupDelay = init.startupDelay ?? STARTUP_DELAY
this.refreshInterval = init.refreshInterval ?? REFRESH_INTERVAL
this.protocol = `/${validatedConfig.protocolPrefix}/${PROTOCOL_NAME}/${PROTOCOL_VERSION}`
this.timeout = validatedConfig.timeout
this.maxInboundStreams = validatedConfig.maxInboundStreams
this.maxOutboundStreams = validatedConfig.maxOutboundStreams
this.startupDelay = validatedConfig.startupDelay
this.refreshInterval = validatedConfig.refreshInterval
this._verifyExternalAddresses = this._verifyExternalAddresses.bind(this)
}

Expand Down Expand Up @@ -167,7 +179,7 @@ class DefaultAutoNATService implements Startable {
try {
// fails on node < 15.4
setMaxListeners?.(Infinity, signal)
} catch {}
} catch { }

const ourHosts = this.components.addressManager.getAddresses()
.map(ma => ma.toOptions().host)
Expand Down Expand Up @@ -435,7 +447,7 @@ class DefaultAutoNATService implements Startable {
try {
// fails on node < 15.4
setMaxListeners?.(Infinity, signal)
} catch {}
} catch { }

const self = this

Expand All @@ -459,7 +471,7 @@ class DefaultAutoNATService implements Startable {
const networkSegments: string[] = []

const verifyAddress = async (peer: PeerInfo): Promise<Message.DialResponse | undefined> => {
let onAbort = (): void => {}
let onAbort = (): void => { }

try {
log('asking %p to verify multiaddr', peer.id)
Expand Down
5 changes: 5 additions & 0 deletions packages/libp2p/src/circuit-relay/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,3 +70,8 @@ export const DEFAULT_HOP_TIMEOUT = 30 * second
* How long to wait before starting to advertise the relay service
*/
export const DEFAULT_ADVERT_BOOT_DELAY = 30 * second

/**
* The default timeout for Incoming STOP requests from the relay
*/
export const DEFAULT_STOP_TIMEOUT = 30 * second
32 changes: 25 additions & 7 deletions packages/libp2p/src/circuit-relay/server/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,16 @@ import { RecordEnvelope } from '@libp2p/peer-record'
import { type Multiaddr, multiaddr } from '@multiformats/multiaddr'
import { pbStream, type ProtobufStream } from 'it-protobuf-stream'
import pDefer from 'p-defer'
import { object, number, boolean } from 'yup'
import { MAX_CONNECTIONS } from '../../connection-manager/constants.js'
import { DEFAULT_MAX_INBOUND_STREAMS, DEFAULT_MAX_OUTBOUND_STREAMS } from '../../registrar.js'
import {
CIRCUIT_PROTO_CODE,
DEFAULT_DURATION_LIMIT,
DEFAULT_HOP_TIMEOUT,
DEFAULT_MAX_RESERVATION_CLEAR_INTERVAL,
DEFAULT_MAX_RESERVATION_STORE_SIZE,
DEFAULT_MAX_RESERVATION_TTL,
RELAY_SOURCE_TAG,
RELAY_V2_HOP_CODEC,
RELAY_V2_STOP_CODEC
Expand Down Expand Up @@ -95,9 +101,19 @@ export interface RelayServerEvents {
'relay:advert:error': CustomEvent<Error>
}

const defaults = {
maxOutboundStopStreams: MAX_CONNECTIONS
}
const configValidator = object({
hopTimeout: number().min(0).integer().default(DEFAULT_HOP_TIMEOUT),
maschad marked this conversation as resolved.
Show resolved Hide resolved
reservations: object({
maxReservations: number().integer().min(0).default(DEFAULT_MAX_RESERVATION_STORE_SIZE),
reservationClearInterval: number().integer().min(0).default(DEFAULT_MAX_RESERVATION_CLEAR_INTERVAL),
applyDefaultLimit: boolean().default(false),
reservationTtl: number().integer().min(0).default(DEFAULT_MAX_RESERVATION_TTL),
defaultDurationLimit: number().integer().min(0).default(DEFAULT_DURATION_LIMIT)
}),
maxInboundHopStreams: number().integer().min(0).default(DEFAULT_MAX_INBOUND_STREAMS),
maxOutboundHopStreams: number().integer().min(0).default(DEFAULT_MAX_OUTBOUND_STREAMS),
maxOutboundStopStreams: number().integer().min(0).default(MAX_CONNECTIONS)
})

class CircuitRelayServer extends EventEmitter<RelayServerEvents> implements Startable, CircuitRelayService {
private readonly registrar: Registrar
Expand All @@ -121,18 +137,20 @@ class CircuitRelayServer extends EventEmitter<RelayServerEvents> implements Star
constructor (components: CircuitRelayServerComponents, init: CircuitRelayServerInit = {}) {
super()

const validatedConfig = configValidator.validateSync(init)

this.registrar = components.registrar
this.peerStore = components.peerStore
this.addressManager = components.addressManager
this.peerId = components.peerId
this.connectionManager = components.connectionManager
this.connectionGater = components.connectionGater
this.started = false
this.hopTimeout = init?.hopTimeout ?? DEFAULT_HOP_TIMEOUT
this.hopTimeout = validatedConfig.hopTimeout
this.shutdownController = new AbortController()
this.maxInboundHopStreams = init.maxInboundHopStreams
this.maxOutboundHopStreams = init.maxOutboundHopStreams
this.maxOutboundStopStreams = init.maxOutboundStopStreams ?? defaults.maxOutboundStopStreams
this.maxInboundHopStreams = validatedConfig.maxInboundHopStreams
this.maxOutboundHopStreams = validatedConfig.maxOutboundHopStreams
this.maxOutboundStopStreams = validatedConfig.maxOutboundStopStreams

try {
// fails on node < 15.4
Expand Down
23 changes: 17 additions & 6 deletions packages/libp2p/src/circuit-relay/server/reservation-store.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { PeerMap } from '@libp2p/peer-collections'
import { object, mixed, number, boolean } from 'yup'
import { DEFAULT_DATA_LIMIT, DEFAULT_DURATION_LIMIT, DEFAULT_MAX_RESERVATION_CLEAR_INTERVAL, DEFAULT_MAX_RESERVATION_STORE_SIZE, DEFAULT_MAX_RESERVATION_TTL } from '../constants.js'
import { type Limit, Status } from '../pb/index.js'
import type { RelayReservation } from '../index.js'
Expand Down Expand Up @@ -38,6 +39,14 @@ export interface ReservationStoreInit {

export type ReservationStoreOptions = RecursivePartial<ReservationStoreInit>

const configValidator = object({
maxReservations: number().min(0).integer().default(DEFAULT_MAX_RESERVATION_STORE_SIZE),
reservationClearInterval: number().integer().min(0).default(DEFAULT_MAX_RESERVATION_CLEAR_INTERVAL),
applyDefaultLimit: boolean().default(true),
reservationTtl: number().integer().min(0).default(DEFAULT_MAX_RESERVATION_TTL),
defaultDurationLimit: number().integer().min(0).default(DEFAULT_DURATION_LIMIT),
defaultDataLimit: mixed().test('is-bigint', 'Invalid bigint', value => typeof value === 'bigint').default(DEFAULT_DATA_LIMIT)
})
export class ReservationStore implements Startable {
public readonly reservations = new PeerMap<RelayReservation>()
private _started = false
Expand All @@ -50,12 +59,14 @@ export class ReservationStore implements Startable {
private readonly defaultDataLimit: bigint

constructor (options: ReservationStoreOptions = {}) {
this.maxReservations = options.maxReservations ?? DEFAULT_MAX_RESERVATION_STORE_SIZE
this.reservationClearInterval = options.reservationClearInterval ?? DEFAULT_MAX_RESERVATION_CLEAR_INTERVAL
this.applyDefaultLimit = options.applyDefaultLimit !== false
this.reservationTtl = options.reservationTtl ?? DEFAULT_MAX_RESERVATION_TTL
this.defaultDurationLimit = options.defaultDurationLimit ?? DEFAULT_DURATION_LIMIT
this.defaultDataLimit = options.defaultDataLimit ?? DEFAULT_DATA_LIMIT
const validatedConfig = configValidator.validateSync(options)

this.maxReservations = validatedConfig.maxReservations
this.reservationClearInterval = validatedConfig.reservationClearInterval
this.applyDefaultLimit = validatedConfig.applyDefaultLimit
this.reservationTtl = validatedConfig.reservationTtl
this.defaultDurationLimit = validatedConfig.defaultDurationLimit
this.defaultDataLimit = validatedConfig.defaultDataLimit as bigint
}

isStarted (): boolean {
Expand Down
30 changes: 17 additions & 13 deletions packages/libp2p/src/circuit-relay/transport/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,10 @@ import { streamToMaConnection } from '@libp2p/utils/stream-to-ma-conn'
import * as mafmt from '@multiformats/mafmt'
import { multiaddr } from '@multiformats/multiaddr'
import { pbStream } from 'it-protobuf-stream'
import { number, object } from 'yup'
import { MAX_CONNECTIONS } from '../../connection-manager/constants.js'
import { codes } from '../../errors.js'
import { CIRCUIT_PROTO_CODE, RELAY_V2_HOP_CODEC, RELAY_V2_STOP_CODEC } from '../constants.js'
import { CIRCUIT_PROTO_CODE, DEFAULT_STOP_TIMEOUT, RELAY_V2_HOP_CODEC, RELAY_V2_STOP_CODEC } from '../constants.js'
import { StopMessage, HopMessage, Status } from '../pb/index.js'
import { RelayDiscovery, type RelayDiscoveryComponents } from './discovery.js'
import { createListener } from './listener.js'
Expand Down Expand Up @@ -100,11 +101,12 @@ export interface CircuitRelayTransportInit extends RelayStoreInit {
reservationCompletionTimeout?: number
}

const defaults = {
maxInboundStopStreams: MAX_CONNECTIONS,
maxOutboundStopStreams: MAX_CONNECTIONS,
stopTimeout: 30000
}
const configValidator = object({
discoverRelays: number().min(0).integer().default(0),
maxInboundStopStreams: number().min(0).integer().default(MAX_CONNECTIONS),
maxOutboundStopStreams: number().min(0).integer().default(MAX_CONNECTIONS),
stopTimeout: number().min(0).integer().default(DEFAULT_STOP_TIMEOUT)
})

class CircuitRelayTransport implements Transport {
private readonly discovery?: RelayDiscovery
Expand All @@ -116,24 +118,26 @@ class CircuitRelayTransport implements Transport {
private readonly addressManager: AddressManager
private readonly connectionGater: ConnectionGater
private readonly reservationStore: ReservationStore
private readonly maxInboundStopStreams: number
private readonly maxInboundStopStreams?: number
private readonly maxOutboundStopStreams?: number
private readonly stopTimeout: number
private readonly stopTimeout?: number
private started: boolean

constructor (components: CircuitRelayTransportComponents, init: CircuitRelayTransportInit) {
const validatedConfig = configValidator.validateSync(init)

this.registrar = components.registrar
this.peerStore = components.peerStore
this.connectionManager = components.connectionManager
this.peerId = components.peerId
this.upgrader = components.upgrader
this.addressManager = components.addressManager
this.connectionGater = components.connectionGater
this.maxInboundStopStreams = init.maxInboundStopStreams ?? defaults.maxInboundStopStreams
this.maxOutboundStopStreams = init.maxOutboundStopStreams ?? defaults.maxOutboundStopStreams
this.stopTimeout = init.stopTimeout ?? defaults.stopTimeout
this.maxInboundStopStreams = validatedConfig.maxInboundStopStreams
this.maxOutboundStopStreams = validatedConfig.maxOutboundStopStreams
this.stopTimeout = validatedConfig.stopTimeout

if (init.discoverRelays != null && init.discoverRelays > 0) {
if (validatedConfig.discoverRelays > 0) {
this.discovery = new RelayDiscovery(components)
this.discovery.addEventListener('relay:discover', (evt) => {
this.reservationStore.addRelay(evt.detail, 'discovered')
Expand Down Expand Up @@ -321,7 +325,7 @@ class CircuitRelayTransport implements Transport {
* An incoming STOP request means a remote peer wants to dial us via a relay
*/
async onStop ({ connection, stream }: IncomingStreamData): Promise<void> {
const signal = AbortSignal.timeout(this.stopTimeout)
const signal = AbortSignal.timeout(this.stopTimeout ?? DEFAULT_STOP_TIMEOUT)
const pbstr = pbStream(stream).pb(StopMessage)
const request = await pbstr.read({
signal
Expand Down
41 changes: 0 additions & 41 deletions packages/libp2p/src/config.ts

This file was deleted.

36 changes: 36 additions & 0 deletions packages/libp2p/src/config/config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
import { FaultTolerance } from '@libp2p/interface/transport'
import { publicAddressesFirst } from '@libp2p/utils/address-sort'
import { dnsaddrResolver } from '@multiformats/multiaddr/resolvers'
import mergeOptions from 'merge-options'
import { object } from 'yup'
import { validateAddressManagerConfig } from '../address-manager/utils.js'
import { validateConnectionManagerConfig } from '../connection-manager/utils.js'
import type { AddressManagerInit } from '../address-manager'
import type { ConnectionManagerInit } from '../connection-manager/index.js'
import type { Libp2pInit } from '../index.js'
import type { ServiceMap, RecursivePartial } from '@libp2p/interface'

const DefaultConfig: Partial<Libp2pInit> = {
maschad marked this conversation as resolved.
Show resolved Hide resolved
connectionManager: {
resolvers: {
dnsaddr: dnsaddrResolver
},
addressSorter: publicAddressesFirst
},
transportManager: {
faultTolerance: FaultTolerance.FATAL_ALL
}
}

export function validateConfig<T extends ServiceMap = Record<string, unknown>> (opts: RecursivePartial<Libp2pInit<T>>): Libp2pInit<T> {
const libp2pConfig = object({
addresses: validateAddressManagerConfig(opts?.addresses as AddressManagerInit),
connectionManager: validateConnectionManagerConfig(opts?.connectionManager as ConnectionManagerInit)
})

const parsedOpts = libp2pConfig.validateSync(opts)

const resultingOptions: Libp2pInit<T> = mergeOptions(DefaultConfig, parsedOpts)

return resultingOptions
}
Loading
Loading