Skip to content

Commit 5e81661

Browse files
pocesarscarmuegajoacohoyos
authored
Paulo/fix pubsub (#3)
* fix: use atomic operation to write cursor (txpipe#798) * Release v1.8.6 * feat: add support for GCP pubsub emulator (txpipe#803) * feat: make v1 compatible with Conway era (txpipe#807) * Release v1.9.0 * fix: running pubsub on platform * chore: formatting * chore: cargo fmt --------- Co-authored-by: Santiago Carmuega <[email protected]> Co-authored-by: Joaquin Hoyos (Clark) <[email protected]>
1 parent fd86874 commit 5e81661

21 files changed

+947
-280
lines changed

‎Cargo.lock

+120-67
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

+10-3
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
[package]
22
name = "oura"
33
description = "The tail of Cardano"
4-
version = "1.9.2"
4+
version = "1.10.0"
55
edition = "2021"
66
repository = "https://github.com/txpipe/oura"
77
homepage = "https://github.com/txpipe/oura"
@@ -12,7 +12,13 @@ authors = ["Santiago Carmuega <[email protected]>"]
1212

1313

1414
[dependencies]
15-
pallas = "0.18.2"
15+
pallas-multiplexer = "0.18.2"
16+
pallas-miniprotocols = "0.18.2"
17+
pallas-primitives = "0.29.0"
18+
pallas-traverse = "0.29.0"
19+
pallas-addresses = "0.29.0"
20+
pallas-codec = "0.29.0"
21+
pallas-crypto = "0.29.0"
1622
# pallas = { git = "https://github.com/txpipe/pallas" }
1723
# pallas = { path = "../pallas/pallas" }
1824
hex = "0.4.3"
@@ -30,6 +36,7 @@ strum = "0.26.3"
3036
strum_macros = "0.26.4"
3137
prometheus_exporter = { version = "0.8.5", default-features = false }
3238
unicode-truncate = "1.1.0"
39+
time = "0.3.36"
3340

3441
# feature logs
3542
file-rotate = { version = "0.7.1", optional = true }
@@ -80,5 +87,5 @@ elasticsink = ["elasticsearch", "tokio"]
8087
fingerprint = ["murmur3"]
8188
aws = ["aws-config", "aws-sdk-sqs", "aws-sdk-lambda", "aws-sdk-s3", "tokio"]
8289
redissink = ["redis", "tokio"]
83-
gcp = ["google-cloud-pubsub", "google-cloud-googleapis", "tokio", "web" ,"google-cloud-gax"]
90+
gcp = ["google-cloud-pubsub", "google-cloud-googleapis", "tokio", "web", "google-cloud-gax"]
8491
rabbitmqsink = ["lapin", "tokio"]

‎src/mapper/babbage.rs

+6-6
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
1-
use pallas::codec::utils::KeepRaw;
1+
use pallas_codec::utils::KeepRaw;
22

3-
use pallas::ledger::primitives::babbage::{
3+
use pallas_primitives::babbage::{
44
AuxiliaryData, MintedBlock, MintedDatumOption, MintedPostAlonzoTransactionOutput,
55
MintedTransactionBody, MintedTransactionOutput, MintedWitnessSet, NetworkId,
66
};
77

8-
use pallas::crypto::hash::Hash;
9-
use pallas::ledger::traverse::OriginalHash;
8+
use pallas_crypto::hash::Hash;
9+
use pallas_traverse::OriginalHash;
1010

1111
use crate::model::{BlockRecord, Era, TransactionRecord};
1212
use crate::utils::time::TimeProvider;
@@ -199,7 +199,7 @@ impl EventWriter {
199199
let record = self.to_post_alonzo_output_record(output)?;
200200
self.append(record.into())?;
201201

202-
let address = pallas::ledger::addresses::Address::from_bytes(&output.address)?;
202+
let address = pallas_addresses::Address::from_bytes(&output.address)?;
203203

204204
let child = &self.child_writer(EventContext {
205205
output_address: address.to_string().into(),
@@ -389,7 +389,7 @@ impl EventWriter {
389389
/// Entry-point to start crawling a blocks for events. Meant to be used when
390390
/// we haven't decoded the CBOR yet (for example, N2N).
391391
pub fn crawl_from_babbage_cbor(&self, cbor: &[u8]) -> Result<(), Error> {
392-
let (_, block): (u16, MintedBlock) = pallas::codec::minicbor::decode(cbor)?;
392+
let (_, block): (u16, MintedBlock) = pallas_codec::minicbor::decode(cbor)?;
393393
self.crawl_babbage_with_cbor(&block, cbor)
394394
}
395395
}

‎src/mapper/byron.rs

+33-33
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,12 @@ use std::ops::Deref;
33
use super::map::ToHex;
44
use super::EventWriter;
55
use crate::model::{BlockRecord, Era, EventData, TransactionRecord, TxInputRecord, TxOutputRecord};
6+
use crate::utils::time::TimeProvider;
67
use crate::{model::EventContext, Error};
78

8-
use pallas::crypto::hash::Hash;
9-
use pallas::ledger::primitives::byron;
10-
use pallas::ledger::traverse::OriginalHash;
9+
use pallas_crypto::hash::Hash;
10+
use pallas_primitives::byron;
11+
use pallas_traverse::OriginalHash;
1112

1213
impl EventWriter {
1314
fn to_byron_input_record(&self, source: &byron::TxIn) -> Option<TxInputRecord> {
@@ -41,12 +42,9 @@ impl EventWriter {
4142
}
4243

4344
fn to_byron_output_record(&self, source: &byron::TxOut) -> Result<TxOutputRecord, Error> {
44-
let address: pallas::ledger::addresses::Address =
45-
pallas::ledger::addresses::ByronAddress::new(
46-
&source.address.payload.0,
47-
source.address.crc,
48-
)
49-
.into();
45+
let address: pallas_addresses::Address =
46+
pallas_addresses::ByronAddress::new(&source.address.payload.0, source.address.crc)
47+
.into();
5048

5149
Ok(TxOutputRecord {
5250
address: address.to_string(),
@@ -168,10 +166,12 @@ impl EventWriter {
168166
hash: &Hash<32>,
169167
cbor: &[u8],
170168
) -> Result<BlockRecord, Error> {
171-
let abs_slot = pallas::ledger::traverse::time::byron_epoch_slot_to_absolute(
172-
source.header.consensus_data.0.epoch,
173-
source.header.consensus_data.0.slot,
174-
);
169+
let abs_slot = self.utils.time.as_ref().map(|time| {
170+
time.byron_epoch_slot_to_absolute(
171+
source.header.consensus_data.0.epoch,
172+
source.header.consensus_data.0.slot,
173+
)
174+
});
175175

176176
let mut record = BlockRecord {
177177
era: Era::Byron,
@@ -181,7 +181,7 @@ impl EventWriter {
181181
tx_count: source.body.tx_payload.len(),
182182
hash: hash.to_hex(),
183183
number: source.header.consensus_data.2[0],
184-
slot: abs_slot,
184+
slot: abs_slot.unwrap_or_default(),
185185
epoch: Some(source.header.consensus_data.0.epoch),
186186
epoch_slot: Some(source.header.consensus_data.0.slot),
187187
previous_hash: source.header.prev_block.to_hex(),
@@ -234,10 +234,9 @@ impl EventWriter {
234234
hash: &Hash<32>,
235235
cbor: &[u8],
236236
) -> Result<BlockRecord, Error> {
237-
let abs_slot = pallas::ledger::traverse::time::byron_epoch_slot_to_absolute(
238-
source.header.consensus_data.epoch_id,
239-
0,
240-
);
237+
let abs_slot = self.utils.time.as_ref().map(|time| {
238+
time.byron_epoch_slot_to_absolute(source.header.consensus_data.epoch_id, 0)
239+
});
241240

242241
Ok(BlockRecord {
243242
era: Era::Byron,
@@ -247,7 +246,7 @@ impl EventWriter {
247246
vrf_vkey: Default::default(),
248247
tx_count: 0,
249248
number: source.header.consensus_data.difficulty[0],
250-
slot: abs_slot,
249+
slot: abs_slot.unwrap_or_default(),
251250
epoch: Some(source.header.consensus_data.epoch_id),
252251
epoch_slot: Some(0),
253252
previous_hash: source.header.prev_block.to_hex(),
@@ -288,16 +287,18 @@ impl EventWriter {
288287
) -> Result<(), Error> {
289288
let hash = block.header.original_hash();
290289

291-
let abs_slot = pallas::ledger::traverse::time::byron_epoch_slot_to_absolute(
292-
block.header.consensus_data.0.epoch,
293-
block.header.consensus_data.0.slot,
294-
);
290+
let abs_slot = self.utils.time.as_ref().map(|time| {
291+
time.byron_epoch_slot_to_absolute(
292+
block.header.consensus_data.0.epoch,
293+
block.header.consensus_data.0.slot,
294+
)
295+
});
295296

296297
let child = self.child_writer(EventContext {
297298
block_hash: Some(hex::encode(hash)),
298299
block_number: Some(block.header.consensus_data.2[0]),
299-
slot: Some(abs_slot),
300-
timestamp: self.compute_timestamp(abs_slot),
300+
slot: abs_slot,
301+
timestamp: abs_slot.and_then(|slot| self.compute_timestamp(slot)),
301302
..EventContext::default()
302303
});
303304

@@ -311,7 +312,7 @@ impl EventWriter {
311312
/// Entry-point to start crawling a blocks for events. Meant to be used when
312313
/// we haven't decoded the CBOR yet (for example, N2N).
313314
pub fn crawl_from_byron_cbor(&self, cbor: &[u8]) -> Result<(), Error> {
314-
let (_, block): (u16, byron::MintedBlock) = pallas::codec::minicbor::decode(cbor)?;
315+
let (_, block): (u16, byron::MintedBlock) = pallas_codec::minicbor::decode(cbor)?;
315316
self.crawl_byron_with_cbor(&block, cbor)
316317
}
317318

@@ -328,16 +329,15 @@ impl EventWriter {
328329
if self.config.include_byron_ebb {
329330
let hash = block.header.original_hash();
330331

331-
let abs_slot = pallas::ledger::traverse::time::byron_epoch_slot_to_absolute(
332-
block.header.consensus_data.epoch_id,
333-
0,
334-
);
332+
let abs_slot = self.utils.time.as_ref().map(|time| {
333+
time.byron_epoch_slot_to_absolute(block.header.consensus_data.epoch_id, 0)
334+
});
335335

336336
let child = self.child_writer(EventContext {
337337
block_hash: Some(hex::encode(hash)),
338338
block_number: Some(block.header.consensus_data.difficulty[0]),
339-
slot: Some(abs_slot),
340-
timestamp: self.compute_timestamp(abs_slot),
339+
slot: abs_slot,
340+
timestamp: abs_slot.and_then(|slot| self.compute_timestamp(slot)),
341341
..EventContext::default()
342342
});
343343

@@ -352,7 +352,7 @@ impl EventWriter {
352352
/// Entry-point to start crawling a blocks for events. Meant to be used when
353353
/// we haven't decoded the CBOR yet (for example, N2N).
354354
pub fn crawl_from_ebb_cbor(&self, cbor: &[u8]) -> Result<(), Error> {
355-
let (_, block): (u16, byron::MintedEbBlock) = pallas::codec::minicbor::decode(cbor)?;
355+
let (_, block): (u16, byron::MintedEbBlock) = pallas_codec::minicbor::decode(cbor)?;
356356
self.crawl_ebb_with_cbor(&block, cbor)
357357
}
358358
}

‎src/mapper/cip15.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ use crate::model::CIP15AssetRecord;
33
use crate::Error;
44
use serde_json::Value as JsonValue;
55

6-
use pallas::ledger::primitives::alonzo::Metadatum;
6+
use pallas_primitives::alonzo::Metadatum;
77

88
fn extract_json_property<'a>(
99
json: &'a JsonValue,

‎src/mapper/cip25.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
use serde_json::Value as JsonValue;
22

3-
use pallas::ledger::primitives::alonzo::Metadatum;
3+
use pallas_primitives::alonzo::Metadatum;
44

55
use crate::{model::CIP25AssetRecord, Error};
66

‎src/mapper/collect.rs

+11-15
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,15 @@
1-
use pallas::{
2-
codec::utils::{KeepRaw, KeyValuePairs, MaybeIndefArray},
3-
ledger::{
4-
primitives::{
5-
alonzo::{
6-
AuxiliaryData, Coin, MintedBlock, Multiasset, NativeScript, PlutusData,
7-
PlutusScript, Redeemer, RewardAccount, TransactionInput, VKeyWitness, Value,
8-
},
9-
babbage::{
10-
LegacyTransactionOutput, MintedPostAlonzoTransactionOutput,
11-
MintedTransactionOutput, PlutusV2Script,
12-
},
13-
},
14-
traverse::OriginalHash,
1+
use pallas_codec::utils::{KeepRaw, KeyValuePairs, MaybeIndefArray};
2+
use pallas_primitives::{
3+
alonzo::{
4+
AuxiliaryData, Coin, MintedBlock, Multiasset, NativeScript, PlutusData, PlutusScript,
5+
Redeemer, RewardAccount, TransactionInput, VKeyWitness, Value,
6+
},
7+
babbage::{
8+
LegacyTransactionOutput, MintedPostAlonzoTransactionOutput, MintedTransactionOutput,
9+
PlutusV2Script,
1510
},
1611
};
12+
use pallas_traverse::OriginalHash;
1713

1814
use crate::{
1915
model::{
@@ -138,7 +134,7 @@ impl EventWriter {
138134

139135
pub fn collect_native_witness_records(
140136
&self,
141-
witness_set: &Option<Vec<NativeScript>>,
137+
witness_set: &Option<Vec<KeepRaw<'_, NativeScript>>>,
142138
) -> Result<Vec<NativeWitnessRecord>, Error> {
143139
match witness_set {
144140
Some(all) => all

0 commit comments

Comments
 (0)