From 7c69bc1534668829b24bec12c872707a975e8521 Mon Sep 17 00:00:00 2001 From: Martin Cederlof Date: Wed, 22 Aug 2018 14:59:02 +0200 Subject: [PATCH 01/30] WIP --- ocs-api/src/main/proto/ocs_analytics.proto | 23 +++++ .../ocsgw/data/grpc/GrpcDataSource.java | 95 +++++++++++-------- 2 files changed, 81 insertions(+), 37 deletions(-) create mode 100644 ocs-api/src/main/proto/ocs_analytics.proto diff --git a/ocs-api/src/main/proto/ocs_analytics.proto b/ocs-api/src/main/proto/ocs_analytics.proto new file mode 100644 index 000000000..ec946eb88 --- /dev/null +++ b/ocs-api/src/main/proto/ocs_analytics.proto @@ -0,0 +1,23 @@ +syntax = "proto3"; + +package org.ostelco.ocs.api; + +option java_multiple_files = true; +option java_package = "org.ostelco.ocs.api"; + +import "google/protobuf/timestamp.proto"; + +// This is used only to report to Analytics events from OCSgw to Prime + +// OCS Service +service OcsAnalyticsService { + rpc AnalyticsEvent (stream ActiveSessionsRequest) returns (ActiveSessionsReply) {} +} + +message ActiveSessionsRequest { + uint32 activeSessions = 1; + google.protobuf.Timestamp timestamp = 2; +} + +message ActiveSessionsReply { +} \ No newline at end of file diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java index ac17f4842..f12fb1112 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java @@ -22,16 +22,7 @@ import org.ostelco.diameter.model.RedirectAddressType; import org.ostelco.diameter.model.RedirectServer; import org.ostelco.diameter.model.SessionContext; -import org.ostelco.ocs.api.ActivateRequest; -import org.ostelco.ocs.api.ActivateResponse; -import org.ostelco.ocs.api.CreditControlAnswerInfo; -import org.ostelco.ocs.api.CreditControlRequestInfo; -import org.ostelco.ocs.api.CreditControlRequestType; -import org.ostelco.ocs.api.OcsServiceGrpc; -import org.ostelco.ocs.api.PsInformation; -import org.ostelco.ocs.api.ReportingReason; -import org.ostelco.ocs.api.ServiceInfo; -import org.ostelco.ocs.api.ServiceUnit; +import org.ostelco.ocs.api.*; import org.ostelco.ocsgw.OcsServer; import org.ostelco.ocsgw.data.DataSource; import org.slf4j.Logger; @@ -67,12 +58,14 @@ public class GrpcDataSource implements DataSource { private static final Logger LOG = LoggerFactory.getLogger(GrpcDataSource.class); - public static final int KEEP_ALIVE_TIMEOUT_IN_MINUTES = 1; + private static final int KEEP_ALIVE_TIMEOUT_IN_MINUTES = 1; - public static final int KEEP_ALIVE_TIME_IN_SECONDS = 50; + private static final int KEEP_ALIVE_TIME_IN_SECONDS = 50; private final OcsServiceGrpc.OcsServiceStub ocsServiceStub; + //private final OcsAnalyticsServiceGrpc.OcsAnalyticsServiceStub ocsAnalyticsServiceStub; + private final Set blocked = new HashSet<>(); private StreamObserver creditControlRequest; @@ -85,7 +78,7 @@ public class GrpcDataSource implements DataSource { private ScheduledFuture initCCRFuture = null; - private static final int MAX_ENTRIES = 3000; + private static final int MAX_ENTRIES = 50000; private final LinkedHashMap ccrMap = new LinkedHashMap(MAX_ENTRIES, .75F) { @Override protected boolean removeEldestEntry(Map.Entry eldest) { @@ -210,6 +203,8 @@ public GrpcDataSource(final String target, final boolean encrypted) throws IOExc .build(); ocsServiceStub = OcsServiceGrpc.newStub(channel) .withCallCredentials(MoreCallCredentials.from(credentials)); + + //ocsAnalyticsServiceStub = OcsAnalyticsServiceGrpc.newStub(channel).withCallCredentials(MoreCallCredentials.from(credentials)); } else { final ManagedChannelBuilder channelBuilder = ManagedChannelBuilder .forTarget(target) @@ -238,32 +233,58 @@ private void initCreditControlRequest() { creditControlRequest = ocsServiceStub.creditControlRequest( new CreditControlRequestObserver() { public void onNext(CreditControlAnswerInfo answer) { + handleGrpcCcrAnswer(answer); + } + }); + } + + private void handleGrpcCcrAnswer(CreditControlAnswerInfo answer) { + try { + LOG.info("[<<] Received data bucket for {}", answer.getMsisdn()); + final CreditControlContext ccrContext = ccrMap.remove(answer.getRequestId()); + if (ccrContext != null) { + final ServerCCASession session = OcsServer.getInstance().getStack().getSession(ccrContext.getSessionId(), ServerCCASession.class); + if (session != null && session.isValid()) { + removeFromSessionMap(ccrContext); + updateBlockedList(answer, ccrContext.getCreditControlRequest()); + if (!ccrContext.getSkipAnswer()) { + CreditControlAnswer cca = createCreditControlAnswer(answer); try { - LOG.info("[<<] Received data bucket for {}", answer.getMsisdn()); - final CreditControlContext ccrContext = ccrMap.remove(answer.getRequestId()); - if (ccrContext != null) { - final ServerCCASession session = OcsServer.getInstance().getStack().getSession(ccrContext.getSessionId(), ServerCCASession.class); - if (session != null && session.isValid()) { - updateBlockedList(answer, ccrContext.getCreditControlRequest()); - if (!ccrContext.getSkipAnswer()) { - CreditControlAnswer cca = createCreditControlAnswer(answer); - try { - session.sendCreditControlAnswer(ccrContext.createCCA(cca)); - } catch (InternalException | IllegalDiameterStateException | RouteException | OverloadException e) { - LOG.error("Failed to send Credit-Control-Answer", e); - } - } - } else { - LOG.warn("No stored CCR or Session for {}", answer.getRequestId()); - } - } else { - LOG.warn("Missing CreditControlContext for req id {}", answer.getRequestId()); - } - } catch (Exception e) { - LOG.error("Credit-Control-Request failed ", e); + session.sendCreditControlAnswer(ccrContext.createCCA(cca)); + } catch (InternalException | IllegalDiameterStateException | RouteException | OverloadException e) { + LOG.error("Failed to send Credit-Control-Answer", e); } } - }); + } else { + LOG.warn("No stored CCR or Session for {}", answer.getRequestId()); + } + } else { + LOG.warn("Missing CreditControlContext for req id {}", answer.getRequestId()); + } + } catch (Exception e) { + LOG.error("Credit-Control-Request failed ", e); + } + } + + private void addToSessionMap(CreditControlContext creditControlContext) { + switch (getRequestType(creditControlContext)) { + case INITIAL_REQUEST: + case UPDATE_REQUEST: + case TERMINATION_REQUEST: + sessionIdMap.put(creditControlContext.getCreditControlRequest().getMsisdn(), new SessionContext(creditControlContext.getSessionId(), creditControlContext.getCreditControlRequest().getOriginHost(), creditControlContext.getCreditControlRequest().getOriginRealm())); + break; + case EVENT_REQUEST: + break; + default: + LOG.warn("Unknown request type"); + break; + } + } + + private void removeFromSessionMap(CreditControlContext creditControlContext) { + if (getRequestType(creditControlContext) == CreditControlRequestType.TERMINATION_REQUEST) { + sessionIdMap.remove(creditControlContext.getCreditControlRequest().getMsisdn()); + } } private void initActivate() { @@ -310,7 +331,7 @@ private void updateBlockedList(CreditControlAnswerInfo answer, CreditControlRequ @Override public void handleRequest(final CreditControlContext context) { ccrMap.put(context.getSessionId(), context); - sessionIdMap.put(context.getCreditControlRequest().getMsisdn(), new SessionContext(context.getSessionId(), context.getCreditControlRequest().getOriginHost(), context.getCreditControlRequest().getOriginRealm())); + addToSessionMap(context); LOG.info("[>>] Requesting bytes for {}", context.getCreditControlRequest().getMsisdn()); if (creditControlRequest != null) { From 9fd67acfc7c131c04e8e63cc7815eb5b9c42c5c8 Mon Sep 17 00:00:00 2001 From: Martin Cederlof Date: Wed, 22 Aug 2018 19:41:30 +0200 Subject: [PATCH 02/30] Split ocs-api and analytics-api for grpc --- analytics-grpc-api/README.md | 1 + {ocs-api => analytics-grpc-api}/build.gradle | 0 .../src/main/proto/analytics.proto | 5 +-- .../src/main/proto/ocs_analytics.proto | 6 +-- analytics/build.gradle | 2 +- .../analytics/DataConsumptionPipeline.kt | 4 +- .../org/ostelco/analytics/PipelineTest.kt | 4 +- {ocs-api => ocs-grpc-api}/README.md | 0 ocs-grpc-api/build.gradle | 41 +++++++++++++++++++ .../src/main/proto/ocs.proto | 4 +- .../analytics/DataConsumptionInfoPublisher.kt | 2 +- .../ostelco/prime/disruptor/EventProducer.kt | 2 +- .../prime/disruptor/EventProducerImpl.kt | 4 +- .../org/ostelco/prime/disruptor/OcsEvent.kt | 2 +- .../prime/ocs/ActivateResponseHolder.kt | 2 +- .../org/ostelco/prime/ocs/EventHandlerImpl.kt | 14 +++---- .../org/ostelco/prime/ocs/OcsGrpcService.kt | 12 +++--- .../org/ostelco/prime/ocs/OcsService.kt | 8 ++-- .../prime/disruptor/PrimeEventProducerTest.kt | 6 +-- ocsgw/build.gradle | 2 +- .../ocsgw/data/grpc/GrpcDataSource.java | 23 +++++++---- .../ocsgw/data/local/LocalDataSource.java | 2 +- .../ocsgw/data/proxy/ProxyDataSource.java | 2 +- prime-api/build.gradle | 3 +- .../kotlin/org/ostelco/prime/ocs/OcsTest.kt | 18 ++++---- pseudonym-server/build.gradle | 2 +- .../pseudonym/managed/MessageProcessor.kt | 2 +- settings.gradle | 6 ++- 28 files changed, 116 insertions(+), 63 deletions(-) create mode 100644 analytics-grpc-api/README.md rename {ocs-api => analytics-grpc-api}/build.gradle (100%) rename {ocs-api => analytics-grpc-api}/src/main/proto/analytics.proto (76%) rename {ocs-api => analytics-grpc-api}/src/main/proto/ocs_analytics.proto (67%) rename {ocs-api => ocs-grpc-api}/README.md (100%) create mode 100644 ocs-grpc-api/build.gradle rename {ocs-api => ocs-grpc-api}/src/main/proto/ocs.proto (96%) diff --git a/analytics-grpc-api/README.md b/analytics-grpc-api/README.md new file mode 100644 index 000000000..ab92604dc --- /dev/null +++ b/analytics-grpc-api/README.md @@ -0,0 +1 @@ +# Analytics API diff --git a/ocs-api/build.gradle b/analytics-grpc-api/build.gradle similarity index 100% rename from ocs-api/build.gradle rename to analytics-grpc-api/build.gradle diff --git a/ocs-api/src/main/proto/analytics.proto b/analytics-grpc-api/src/main/proto/analytics.proto similarity index 76% rename from ocs-api/src/main/proto/analytics.proto rename to analytics-grpc-api/src/main/proto/analytics.proto index a44dd35dd..acff96728 100644 --- a/ocs-api/src/main/proto/analytics.proto +++ b/analytics-grpc-api/src/main/proto/analytics.proto @@ -1,14 +1,13 @@ syntax = "proto3"; -package org.ostelco.ocs.api; +package org.ostelco.analytics.grpc.api; option java_multiple_files = true; -option java_package = "org.ostelco.ocs.api"; +option java_package = "org.ostelco.analytics.grpc.api"; import "google/protobuf/timestamp.proto"; // This is used only to report to Analytics engine by Prime via Google Cloud Pub/Sub. -// This may be moved to a separate library project, in future. message DataTrafficInfo { string msisdn = 1; diff --git a/ocs-api/src/main/proto/ocs_analytics.proto b/analytics-grpc-api/src/main/proto/ocs_analytics.proto similarity index 67% rename from ocs-api/src/main/proto/ocs_analytics.proto rename to analytics-grpc-api/src/main/proto/ocs_analytics.proto index ec946eb88..13df0ceac 100644 --- a/ocs-api/src/main/proto/ocs_analytics.proto +++ b/analytics-grpc-api/src/main/proto/ocs_analytics.proto @@ -1,9 +1,9 @@ syntax = "proto3"; -package org.ostelco.ocs.api; +package org.ostelco.analytics.grpc.api; option java_multiple_files = true; -option java_package = "org.ostelco.ocs.api"; +option java_package = "org.ostelco.analytics.grpc.api"; import "google/protobuf/timestamp.proto"; @@ -11,7 +11,7 @@ import "google/protobuf/timestamp.proto"; // OCS Service service OcsAnalyticsService { - rpc AnalyticsEvent (stream ActiveSessionsRequest) returns (ActiveSessionsReply) {} + rpc AnalyticsEvent (ActiveSessionsRequest) returns (ActiveSessionsReply) {} } message ActiveSessionsRequest { diff --git a/analytics/build.gradle b/analytics/build.gradle index 857e271a9..00117c145 100644 --- a/analytics/build.gradle +++ b/analytics/build.gradle @@ -6,7 +6,7 @@ plugins { } dependencies { - implementation project(':ocs-api') + implementation project(':analytics-grpc-api') implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion" implementation "com.google.cloud:google-cloud-pubsub:$googleCloudVersion" implementation 'com.google.cloud.dataflow:google-cloud-dataflow-java-sdk-all:2.5.0' diff --git a/analytics/src/main/kotlin/org/ostelco/analytics/DataConsumptionPipeline.kt b/analytics/src/main/kotlin/org/ostelco/analytics/DataConsumptionPipeline.kt index 5e9de19fa..4e5891555 100644 --- a/analytics/src/main/kotlin/org/ostelco/analytics/DataConsumptionPipeline.kt +++ b/analytics/src/main/kotlin/org/ostelco/analytics/DataConsumptionPipeline.kt @@ -26,8 +26,8 @@ import org.joda.time.Instant import org.ostelco.analytics.ParDoFn.transform import org.ostelco.analytics.Table.HOURLY_CONSUMPTION import org.ostelco.analytics.Table.RAW_CONSUMPTION -import org.ostelco.ocs.api.AggregatedDataTrafficInfo -import org.ostelco.ocs.api.DataTrafficInfo +import org.ostelco.analytics.grpc.api.AggregatedDataTrafficInfo +import org.ostelco.analytics.grpc.api.DataTrafficInfo import java.time.ZoneOffset import java.time.ZonedDateTime import java.time.format.DateTimeFormatter diff --git a/analytics/src/test/kotlin/org/ostelco/analytics/PipelineTest.kt b/analytics/src/test/kotlin/org/ostelco/analytics/PipelineTest.kt index 15ec50f4a..42ac68458 100644 --- a/analytics/src/test/kotlin/org/ostelco/analytics/PipelineTest.kt +++ b/analytics/src/test/kotlin/org/ostelco/analytics/PipelineTest.kt @@ -11,8 +11,8 @@ import org.joda.time.Instant import org.junit.Rule import org.junit.Test import org.junit.experimental.categories.Category -import org.ostelco.ocs.api.AggregatedDataTrafficInfo -import org.ostelco.ocs.api.DataTrafficInfo +import org.ostelco.analytics.grpc.api.AggregatedDataTrafficInfo +import org.ostelco.analytics.grpc.api.DataTrafficInfo import java.time.ZoneOffset import java.time.ZonedDateTime import java.time.format.DateTimeFormatter diff --git a/ocs-api/README.md b/ocs-grpc-api/README.md similarity index 100% rename from ocs-api/README.md rename to ocs-grpc-api/README.md diff --git a/ocs-grpc-api/build.gradle b/ocs-grpc-api/build.gradle new file mode 100644 index 000000000..08b28a100 --- /dev/null +++ b/ocs-grpc-api/build.gradle @@ -0,0 +1,41 @@ +plugins { + id "java-library" + id "com.google.protobuf" version "0.8.6" + id "idea" +} + +// Keeping it version 1.13.1 to be consistent with grpc via PubSub client lib +// Keeping it version 1.13.1 to be consistent with netty via Firebase lib +ext.grpcVersion = "1.13.1" + +dependencies { + api "io.grpc:grpc-netty-shaded:$grpcVersion" + api "io.grpc:grpc-protobuf:$grpcVersion" + api "io.grpc:grpc-stub:$grpcVersion" + api "io.grpc:grpc-core:$grpcVersion" +} + +protobuf { + plugins { + grpc { + artifact = "io.grpc:protoc-gen-grpc-java:$grpcVersion" + } + } + protoc { artifact = 'com.google.protobuf:protoc:3.6.0' } + generateProtoTasks { + all()*.plugins { + grpc {} + } + } +} + +idea { + module { + sourceDirs += files("${protobuf.generatedFilesBaseDir}/main/java") + sourceDirs += files("${protobuf.generatedFilesBaseDir}/main/grpc") + } +} + +clean { + delete protobuf.generatedFilesBaseDir +} diff --git a/ocs-api/src/main/proto/ocs.proto b/ocs-grpc-api/src/main/proto/ocs.proto similarity index 96% rename from ocs-api/src/main/proto/ocs.proto rename to ocs-grpc-api/src/main/proto/ocs.proto index f4afe8537..5346f068e 100644 --- a/ocs-api/src/main/proto/ocs.proto +++ b/ocs-grpc-api/src/main/proto/ocs.proto @@ -1,9 +1,9 @@ syntax = "proto3"; -package org.ostelco.ocs.api; +package org.ostelco.ocs.grpc.api; option java_multiple_files = true; -option java_package = "org.ostelco.ocs.api"; +option java_package = "org.ostelco.ocs.grpc.api"; // OCS Service service OcsService { diff --git a/ocs/src/main/kotlin/org/ostelco/prime/analytics/DataConsumptionInfoPublisher.kt b/ocs/src/main/kotlin/org/ostelco/prime/analytics/DataConsumptionInfoPublisher.kt index 54f02c15e..82a230674 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/analytics/DataConsumptionInfoPublisher.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/analytics/DataConsumptionInfoPublisher.kt @@ -9,7 +9,7 @@ import com.google.pubsub.v1.ProjectTopicName import com.google.pubsub.v1.PubsubMessage import com.lmax.disruptor.EventHandler import io.dropwizard.lifecycle.Managed -import org.ostelco.ocs.api.DataTrafficInfo +import org.ostelco.analytics.grpc.api.DataTrafficInfo import org.ostelco.prime.disruptor.OcsEvent import org.ostelco.prime.disruptor.EventMessageType.CREDIT_CONTROL_REQUEST import org.ostelco.prime.logger diff --git a/ocs/src/main/kotlin/org/ostelco/prime/disruptor/EventProducer.kt b/ocs/src/main/kotlin/org/ostelco/prime/disruptor/EventProducer.kt index 8bc4c0526..01e7fd8a6 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/disruptor/EventProducer.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/disruptor/EventProducer.kt @@ -1,6 +1,6 @@ package org.ostelco.prime.disruptor -import org.ostelco.ocs.api.CreditControlRequestInfo +import org.ostelco.ocs.grpc.api.CreditControlRequestInfo import org.ostelco.prime.model.Bundle interface EventProducer { diff --git a/ocs/src/main/kotlin/org/ostelco/prime/disruptor/EventProducerImpl.kt b/ocs/src/main/kotlin/org/ostelco/prime/disruptor/EventProducerImpl.kt index 47edee8e0..1df0ebf3c 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/disruptor/EventProducerImpl.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/disruptor/EventProducerImpl.kt @@ -1,8 +1,8 @@ package org.ostelco.prime.disruptor import com.lmax.disruptor.RingBuffer -import org.ostelco.ocs.api.CreditControlRequestInfo -import org.ostelco.ocs.api.ReportingReason +import org.ostelco.ocs.grpc.api.CreditControlRequestInfo +import org.ostelco.ocs.grpc.api.ReportingReason import org.ostelco.prime.disruptor.EventMessageType.ADD_MSISDN_TO_BUNDLE_MAPPING import org.ostelco.prime.disruptor.EventMessageType.CREDIT_CONTROL_REQUEST import org.ostelco.prime.disruptor.EventMessageType.RELEASE_RESERVED_BUCKET diff --git a/ocs/src/main/kotlin/org/ostelco/prime/disruptor/OcsEvent.kt b/ocs/src/main/kotlin/org/ostelco/prime/disruptor/OcsEvent.kt index 815a76669..619b806df 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/disruptor/OcsEvent.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/disruptor/OcsEvent.kt @@ -1,6 +1,6 @@ package org.ostelco.prime.disruptor -import org.ostelco.ocs.api.ReportingReason +import org.ostelco.ocs.grpc.api.ReportingReason class OcsEvent { diff --git a/ocs/src/main/kotlin/org/ostelco/prime/ocs/ActivateResponseHolder.kt b/ocs/src/main/kotlin/org/ostelco/prime/ocs/ActivateResponseHolder.kt index b0858568d..baa7f8dde 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/ocs/ActivateResponseHolder.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/ocs/ActivateResponseHolder.kt @@ -1,7 +1,7 @@ package org.ostelco.prime.ocs import io.grpc.stub.StreamObserver -import org.ostelco.ocs.api.ActivateResponse +import org.ostelco.ocs.grpc.api.ActivateResponse import java.util.concurrent.locks.Lock import java.util.concurrent.locks.ReentrantReadWriteLock diff --git a/ocs/src/main/kotlin/org/ostelco/prime/ocs/EventHandlerImpl.kt b/ocs/src/main/kotlin/org/ostelco/prime/ocs/EventHandlerImpl.kt index 2402c9727..d5534a024 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/ocs/EventHandlerImpl.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/ocs/EventHandlerImpl.kt @@ -1,13 +1,13 @@ package org.ostelco.prime.ocs import com.lmax.disruptor.EventHandler -import org.ostelco.ocs.api.ActivateResponse -import org.ostelco.ocs.api.CreditControlAnswerInfo -import org.ostelco.ocs.api.FinalUnitAction -import org.ostelco.ocs.api.FinalUnitIndication -import org.ostelco.ocs.api.MultipleServiceCreditControl -import org.ostelco.ocs.api.ReportingReason -import org.ostelco.ocs.api.ServiceUnit +import org.ostelco.ocs.grpc.api.ActivateResponse +import org.ostelco.ocs.grpc.api.CreditControlAnswerInfo +import org.ostelco.ocs.grpc.api.FinalUnitAction +import org.ostelco.ocs.grpc.api.FinalUnitIndication +import org.ostelco.ocs.grpc.api.MultipleServiceCreditControl +import org.ostelco.ocs.grpc.api.ReportingReason +import org.ostelco.ocs.grpc.api.ServiceUnit import org.ostelco.prime.disruptor.EventMessageType.CREDIT_CONTROL_REQUEST import org.ostelco.prime.disruptor.EventMessageType.TOPUP_DATA_BUNDLE_BALANCE import org.ostelco.prime.disruptor.OcsEvent diff --git a/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsGrpcService.kt b/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsGrpcService.kt index b27098722..2f6cc8642 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsGrpcService.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsGrpcService.kt @@ -1,12 +1,12 @@ package org.ostelco.prime.ocs import io.grpc.stub.StreamObserver -import org.ostelco.ocs.api.ActivateRequest -import org.ostelco.ocs.api.ActivateResponse -import org.ostelco.ocs.api.CreditControlAnswerInfo -import org.ostelco.ocs.api.CreditControlRequestInfo -import org.ostelco.ocs.api.CreditControlRequestType.NONE -import org.ostelco.ocs.api.OcsServiceGrpc +import org.ostelco.ocs.grpc.api.ActivateRequest +import org.ostelco.ocs.grpc.api.ActivateResponse +import org.ostelco.ocs.grpc.api.CreditControlAnswerInfo +import org.ostelco.ocs.grpc.api.CreditControlRequestInfo +import org.ostelco.ocs.grpc.api.CreditControlRequestType.NONE +import org.ostelco.ocs.grpc.api.OcsServiceGrpc import org.ostelco.prime.logger import java.util.* diff --git a/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsService.kt b/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsService.kt index c0de0ab29..9c1c50506 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsService.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsService.kt @@ -2,10 +2,10 @@ package org.ostelco.prime.ocs import com.lmax.disruptor.EventHandler import io.grpc.stub.StreamObserver -import org.ostelco.ocs.api.ActivateResponse -import org.ostelco.ocs.api.CreditControlAnswerInfo -import org.ostelco.ocs.api.CreditControlRequestInfo -import org.ostelco.ocs.api.OcsServiceGrpc +import org.ostelco.ocs.grpc.api.ActivateResponse +import org.ostelco.ocs.grpc.api.CreditControlAnswerInfo +import org.ostelco.ocs.grpc.api.CreditControlRequestInfo +import org.ostelco.ocs.grpc.api.OcsServiceGrpc import org.ostelco.prime.disruptor.EventProducer import org.ostelco.prime.disruptor.OcsEvent import java.util.concurrent.ConcurrentHashMap diff --git a/ocs/src/test/kotlin/org/ostelco/prime/disruptor/PrimeEventProducerTest.kt b/ocs/src/test/kotlin/org/ostelco/prime/disruptor/PrimeEventProducerTest.kt index e410bae73..9a018915c 100644 --- a/ocs/src/test/kotlin/org/ostelco/prime/disruptor/PrimeEventProducerTest.kt +++ b/ocs/src/test/kotlin/org/ostelco/prime/disruptor/PrimeEventProducerTest.kt @@ -9,9 +9,9 @@ import org.junit.Assert.assertNotNull import org.junit.Assert.assertTrue import org.junit.Before import org.junit.Test -import org.ostelco.ocs.api.CreditControlRequestInfo -import org.ostelco.ocs.api.MultipleServiceCreditControl -import org.ostelco.ocs.api.ServiceUnit +import org.ostelco.ocs.grpc.api.CreditControlRequestInfo +import org.ostelco.ocs.grpc.api.MultipleServiceCreditControl +import org.ostelco.ocs.grpc.api.ServiceUnit import org.ostelco.prime.disruptor.EventMessageType.CREDIT_CONTROL_REQUEST import org.ostelco.prime.disruptor.EventMessageType.TOPUP_DATA_BUNDLE_BALANCE import java.util.concurrent.CountDownLatch diff --git a/ocsgw/build.gradle b/ocsgw/build.gradle index 29b9950c3..6e5a73640 100644 --- a/ocsgw/build.gradle +++ b/ocsgw/build.gradle @@ -4,7 +4,7 @@ plugins { } dependencies { - implementation project(':ocs-api') + implementation project(':ocs-grpc-api') implementation project(':diameter-stack') implementation "com.google.cloud:google-cloud-core-grpc:$googleCloudVersion" diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java index f12fb1112..47c2496cd 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java @@ -22,7 +22,7 @@ import org.ostelco.diameter.model.RedirectAddressType; import org.ostelco.diameter.model.RedirectServer; import org.ostelco.diameter.model.SessionContext; -import org.ostelco.ocs.api.*; +import org.ostelco.ocs.grpc.api.*; import org.ostelco.ocsgw.OcsServer; import org.ostelco.ocsgw.data.DataSource; import org.slf4j.Logger; @@ -272,6 +272,7 @@ private void addToSessionMap(CreditControlContext creditControlContext) { case UPDATE_REQUEST: case TERMINATION_REQUEST: sessionIdMap.put(creditControlContext.getCreditControlRequest().getMsisdn(), new SessionContext(creditControlContext.getSessionId(), creditControlContext.getCreditControlRequest().getOriginHost(), creditControlContext.getCreditControlRequest().getOriginRealm())); + updateAnalytics(); break; case EVENT_REQUEST: break; @@ -284,9 +285,17 @@ private void addToSessionMap(CreditControlContext creditControlContext) { private void removeFromSessionMap(CreditControlContext creditControlContext) { if (getRequestType(creditControlContext) == CreditControlRequestType.TERMINATION_REQUEST) { sessionIdMap.remove(creditControlContext.getCreditControlRequest().getMsisdn()); + updateAnalytics(); } } + private void updateAnalytics() { + LOG.info("Number of active sesssions is {}", sessionIdMap.size()); + + // ToDo: Send to analytics + + } + private void initActivate() { ActivateRequest dummyActivate = ActivateRequest.newBuilder().build(); ocsServiceStub.activate(dummyActivate, new ActivateObserver() { @@ -318,7 +327,7 @@ private void initKeepAlive() { private void updateBlockedList(CreditControlAnswerInfo answer, CreditControlRequest request) { // This suffers from the fact that one Credit-Control-Request can have multiple MSCC - for (org.ostelco.ocs.api.MultipleServiceCreditControl msccAnswer : answer.getMsccList()) { + for (org.ostelco.ocs.grpc.api.MultipleServiceCreditControl msccAnswer : answer.getMsccList()) { for (org.ostelco.diameter.model.MultipleServiceCreditControl msccRequest : request.getMultipleServiceCreditControls()) { if ((msccAnswer.getServiceIdentifier() == msccRequest.getServiceIdentifier()) && (msccAnswer.getRatingGroup() == msccRequest.getRatingGroup())) { updateBlockedList(msccAnswer, msccRequest, answer.getMsisdn()); @@ -342,7 +351,7 @@ public void handleRequest(final CreditControlContext context) { for (MultipleServiceCreditControl mscc : context.getCreditControlRequest().getMultipleServiceCreditControls()) { - org.ostelco.ocs.api.MultipleServiceCreditControl.Builder protoMscc = org.ostelco.ocs.api.MultipleServiceCreditControl.newBuilder(); + org.ostelco.ocs.grpc.api.MultipleServiceCreditControl.Builder protoMscc = org.ostelco.ocs.grpc.api.MultipleServiceCreditControl.newBuilder(); if (!mscc.getRequested().isEmpty()) { @@ -434,13 +443,13 @@ private CreditControlAnswer createCreditControlAnswer(CreditControlAnswerInfo re } final LinkedList multipleServiceCreditControls = new LinkedList<>(); - for (org.ostelco.ocs.api.MultipleServiceCreditControl mscc : response.getMsccList()) { + for (org.ostelco.ocs.grpc.api.MultipleServiceCreditControl mscc : response.getMsccList()) { multipleServiceCreditControls.add(convertMSCC(mscc)); } return new CreditControlAnswer(multipleServiceCreditControls); } - private void updateBlockedList(org.ostelco.ocs.api.MultipleServiceCreditControl msccAnswer, org.ostelco.diameter.model.MultipleServiceCreditControl msccRequest, String msisdn) { + private void updateBlockedList(org.ostelco.ocs.grpc.api.MultipleServiceCreditControl msccAnswer, org.ostelco.diameter.model.MultipleServiceCreditControl msccRequest, String msisdn) { if (!msccRequest.getRequested().isEmpty()) { if (msccAnswer.getGranted().getTotalOctets() < msccRequest.getRequested().get(0).getTotal()) { blocked.add(msisdn); @@ -450,7 +459,7 @@ private void updateBlockedList(org.ostelco.ocs.api.MultipleServiceCreditControl } } - private MultipleServiceCreditControl convertMSCC(org.ostelco.ocs.api.MultipleServiceCreditControl msccGRPC) { + private MultipleServiceCreditControl convertMSCC(org.ostelco.ocs.grpc.api.MultipleServiceCreditControl msccGRPC) { return new MultipleServiceCreditControl( msccGRPC.getRatingGroup(), (int) msccGRPC.getServiceIdentifier(), @@ -461,7 +470,7 @@ private MultipleServiceCreditControl convertMSCC(org.ostelco.ocs.api.MultipleSer convertFinalUnitIndication(msccGRPC.getFinalUnitIndication())); } - private FinalUnitIndication convertFinalUnitIndication(org.ostelco.ocs.api.FinalUnitIndication fuiGrpc) { + private FinalUnitIndication convertFinalUnitIndication(org.ostelco.ocs.grpc.api.FinalUnitIndication fuiGrpc) { if (!fuiGrpc.getIsSet()) { return null; } diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/local/LocalDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/local/LocalDataSource.java index 3b45e09f7..5c2729ab1 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/local/LocalDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/local/LocalDataSource.java @@ -13,7 +13,7 @@ import org.ostelco.diameter.model.RedirectAddressType; import org.ostelco.diameter.model.RedirectServer; import org.ostelco.diameter.model.ServiceUnit; -import org.ostelco.ocs.api.CreditControlRequestType; +import org.ostelco.ocs.grpc.api.CreditControlRequestType; import org.ostelco.ocsgw.OcsServer; import org.ostelco.ocsgw.data.DataSource; import org.slf4j.Logger; diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/proxy/ProxyDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/proxy/ProxyDataSource.java index c7744df98..e779bc2da 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/proxy/ProxyDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/proxy/ProxyDataSource.java @@ -3,7 +3,7 @@ import org.ostelco.ocsgw.data.DataSource; import org.ostelco.ocsgw.data.local.LocalDataSource; import org.ostelco.diameter.CreditControlContext; -import org.ostelco.ocs.api.CreditControlRequestType; +import org.ostelco.ocs.grpc.api.CreditControlRequestType; /** * Proxy DataSource is a combination of the Local DataSource and any other * DataSource. diff --git a/prime-api/build.gradle b/prime-api/build.gradle index 9f95213f2..5ffc69148 100644 --- a/prime-api/build.gradle +++ b/prime-api/build.gradle @@ -9,7 +9,8 @@ dependencies { api "io.jsonwebtoken:jjwt:0.9.0" - api project(':ocs-api') + api project(':ocs-grpc-api') + api project(':analytics-grpc-api') api project(':model') api "io.dropwizard:dropwizard-core:$dropwizardVersion" diff --git a/prime/src/integration-tests/kotlin/org/ostelco/prime/ocs/OcsTest.kt b/prime/src/integration-tests/kotlin/org/ostelco/prime/ocs/OcsTest.kt index 8627bd178..3fd3d9ebd 100644 --- a/prime/src/integration-tests/kotlin/org/ostelco/prime/ocs/OcsTest.kt +++ b/prime/src/integration-tests/kotlin/org/ostelco/prime/ocs/OcsTest.kt @@ -12,15 +12,15 @@ import org.junit.Assert.assertEquals import org.junit.BeforeClass import org.junit.ClassRule import org.junit.Test -import org.ostelco.ocs.api.ActivateRequest -import org.ostelco.ocs.api.ActivateResponse -import org.ostelco.ocs.api.CreditControlAnswerInfo -import org.ostelco.ocs.api.CreditControlRequestInfo -import org.ostelco.ocs.api.CreditControlRequestType.INITIAL_REQUEST -import org.ostelco.ocs.api.MultipleServiceCreditControl -import org.ostelco.ocs.api.OcsServiceGrpc -import org.ostelco.ocs.api.OcsServiceGrpc.OcsServiceStub -import org.ostelco.ocs.api.ServiceUnit +import org.ostelco.ocs.grpc.api.ActivateRequest +import org.ostelco.ocs.grpc.api.ActivateResponse +import org.ostelco.ocs.grpc.api.CreditControlAnswerInfo +import org.ostelco.ocs.grpc.api.CreditControlRequestInfo +import org.ostelco.ocs.grpc.api.CreditControlRequestType.INITIAL_REQUEST +import org.ostelco.ocs.grpc.api.MultipleServiceCreditControl +import org.ostelco.ocs.grpc.api.OcsServiceGrpc +import org.ostelco.ocs.grpc.api.OcsServiceGrpc.OcsServiceStub +import org.ostelco.ocs.grpc.api.ServiceUnit import org.ostelco.prime.disruptor.EventProducerImpl import org.ostelco.prime.disruptor.OcsDisruptor import org.ostelco.prime.logger diff --git a/pseudonym-server/build.gradle b/pseudonym-server/build.gradle index 3b17980cd..14e6978bb 100644 --- a/pseudonym-server/build.gradle +++ b/pseudonym-server/build.gradle @@ -9,7 +9,7 @@ dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion" implementation "io.dropwizard:dropwizard-core:$dropwizardVersion" implementation "io.dropwizard:dropwizard-client:$dropwizardVersion" - implementation project(':ocs-api') + implementation project(':analytics-grpc-api') implementation 'com.google.guava:guava:25.1-jre' // Match with grpc-netty-shaded via PubSub // removing io.grpc:grpc-netty-shaded:1.13.1 causes ALPN error diff --git a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/managed/MessageProcessor.kt b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/managed/MessageProcessor.kt index 92355f055..f9451f00a 100644 --- a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/managed/MessageProcessor.kt +++ b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/managed/MessageProcessor.kt @@ -21,7 +21,7 @@ import com.google.pubsub.v1.PubsubMessage import io.dropwizard.lifecycle.Managed import io.grpc.ManagedChannel import io.grpc.ManagedChannelBuilder -import org.ostelco.ocs.api.DataTrafficInfo +import org.ostelco.analytics.grpc.api.DataTrafficInfo import org.ostelco.prime.model.PseudonymEntity import org.ostelco.pseudonym.resources.DateBounds import org.slf4j.LoggerFactory diff --git a/settings.gradle b/settings.gradle index f2fee1545..bbc094fbf 100644 --- a/settings.gradle +++ b/settings.gradle @@ -3,6 +3,7 @@ rootProject.name = 'ostelco-core' include ':acceptance-tests' include ':app-notifier' include ':admin-api' +include ':analytics-grpc-api' include ':analytics' include ':auth-server' include ':client-api' @@ -14,7 +15,7 @@ include ':model' include ':neo4j-admin-tools' include ':neo4j-store' include ':ocs' -include ':ocs-api' +include ':ocs-grpc-api' include ':ocsgw' include ':ostelco-lib' include ':payment-processor' @@ -26,6 +27,7 @@ include ':pseudonym-server' project(':acceptance-tests').projectDir = "$rootDir/acceptance-tests" as File project(':app-notifier').projectDir = "$rootDir/app-notifier" as File project(':admin-api').projectDir = "$rootDir/admin-api" as File +project(':analytics-grpc-api').projectDir = "$rootDir/analytics-grpc-api" as File project(':analytics').projectDir = "$rootDir/analytics" as File project(':auth-server').projectDir = "$rootDir/auth-server" as File project(':client-api').projectDir = "$rootDir/client-api" as File @@ -37,7 +39,7 @@ project(':model').projectDir = "$rootDir/model" as File project(':neo4j-admin-tools').projectDir = "$rootDir/tools/neo4j-admin-tools" as File project(':neo4j-store').projectDir = "$rootDir/neo4j-store" as File project(':ocs').projectDir = "$rootDir/ocs" as File -project(':ocs-api').projectDir = "$rootDir/ocs-api" as File +project(':ocs-grpc-api').projectDir = "$rootDir/ocs-grpc-api" as File project(':ocsgw').projectDir = "$rootDir/ocsgw" as File project(':ostelco-lib').projectDir = "$rootDir/ostelco-lib" as File project(':payment-processor').projectDir = "$rootDir/payment-processor" as File From ca016bcce4067f4aabad531e4f5a13338e63218e Mon Sep 17 00:00:00 2001 From: Martin Cederlof Date: Wed, 22 Aug 2018 21:33:30 +0200 Subject: [PATCH 03/30] Send analytics event to prime from ocsgw using grpc --- .../src/main/proto/ocs_analytics.proto | 11 ++-- ocsgw/build.gradle | 1 + .../ocsgw/data/grpc/GrpcDataSource.java | 63 +++++++++++++++++-- 3 files changed, 63 insertions(+), 12 deletions(-) diff --git a/analytics-grpc-api/src/main/proto/ocs_analytics.proto b/analytics-grpc-api/src/main/proto/ocs_analytics.proto index 13df0ceac..310f410b2 100644 --- a/analytics-grpc-api/src/main/proto/ocs_analytics.proto +++ b/analytics-grpc-api/src/main/proto/ocs_analytics.proto @@ -7,17 +7,16 @@ option java_package = "org.ostelco.analytics.grpc.api"; import "google/protobuf/timestamp.proto"; -// This is used only to report to Analytics events from OCSgw to Prime +// This is used to report Analytics events from OCSgw to Prime -// OCS Service -service OcsAnalyticsService { - rpc AnalyticsEvent (ActiveSessionsRequest) returns (ActiveSessionsReply) {} +service OcsgwAnalyticsService { + rpc OcsgwAnalyticsEvent (stream OcsgwAnalyticsReport) returns (OcsgwAnalyticsReply) {} } -message ActiveSessionsRequest { +message OcsgwAnalyticsReport { uint32 activeSessions = 1; google.protobuf.Timestamp timestamp = 2; } -message ActiveSessionsReply { +message OcsgwAnalyticsReply { } \ No newline at end of file diff --git a/ocsgw/build.gradle b/ocsgw/build.gradle index 6e5a73640..b8ac38f13 100644 --- a/ocsgw/build.gradle +++ b/ocsgw/build.gradle @@ -5,6 +5,7 @@ plugins { dependencies { implementation project(':ocs-grpc-api') + implementation project(':analytics-grpc-api') implementation project(':diameter-stack') implementation "com.google.cloud:google-cloud-core-grpc:$googleCloudVersion" diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java index 47c2496cd..f260dc79c 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java @@ -23,6 +23,7 @@ import org.ostelco.diameter.model.RedirectServer; import org.ostelco.diameter.model.SessionContext; import org.ostelco.ocs.grpc.api.*; +import org.ostelco.analytics.grpc.api.*; import org.ostelco.ocsgw.OcsServer; import org.ostelco.ocsgw.data.DataSource; import org.slf4j.Logger; @@ -64,12 +65,14 @@ public class GrpcDataSource implements DataSource { private final OcsServiceGrpc.OcsServiceStub ocsServiceStub; - //private final OcsAnalyticsServiceGrpc.OcsAnalyticsServiceStub ocsAnalyticsServiceStub; - private final Set blocked = new HashSet<>(); private StreamObserver creditControlRequest; + private final OcsgwAnalyticsServiceGrpc.OcsgwAnalyticsServiceStub ocsgwAnalyticsServiceStub; + + private StreamObserver ocsgwAnalyticsReport; + private ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); private ScheduledFuture keepAliveFuture = null; @@ -78,6 +81,8 @@ public class GrpcDataSource implements DataSource { private ScheduledFuture initCCRFuture = null; + private ScheduledFuture initAnalyticsFuture = null; + private static final int MAX_ENTRIES = 50000; private final LinkedHashMap ccrMap = new LinkedHashMap(MAX_ENTRIES, .75F) { @Override @@ -107,6 +112,19 @@ public final void onCompleted() { } } + private abstract class AnalyticsRequestObserver implements StreamObserver { + public final void onError(Throwable t) { + LOG.error("AnalyticsRequestObserver error", t); + if (t instanceof StatusRuntimeException) { + reconnectAnalyticsReport(); + } + } + + public final void onCompleted() { + // Nothing to do here + } + } + private abstract class ActivateObserver implements StreamObserver { public final void onError(Throwable t) { LOG.error("ActivateObserver error", t); @@ -137,6 +155,24 @@ private void reconnectActivate() { TimeUnit.SECONDS); } + + private void reconnectAnalyticsReport() { + LOG.info("reconnectAnalyticsReport called"); + + if (initAnalyticsFuture != null) { + initAnalyticsFuture.cancel(true); + } + + LOG.info("Schedule new Callable initAnalyticsRequest"); + initAnalyticsFuture = executorService.schedule((Callable) () -> { + LOG.info("Calling initAnalyticsRequest"); + initAnalyticsRequest(); + return "Called!"; + }, + 5, + TimeUnit.SECONDS); + } + private void reconnectCcrKeepAlive() { LOG.info("reconnectCreditControlRequest called"); if (keepAliveFuture != null) { @@ -204,7 +240,8 @@ public GrpcDataSource(final String target, final boolean encrypted) throws IOExc ocsServiceStub = OcsServiceGrpc.newStub(channel) .withCallCredentials(MoreCallCredentials.from(credentials)); - //ocsAnalyticsServiceStub = OcsAnalyticsServiceGrpc.newStub(channel).withCallCredentials(MoreCallCredentials.from(credentials)); + ocsgwAnalyticsServiceStub = OcsgwAnalyticsServiceGrpc.newStub(channel) + .withCallCredentials(MoreCallCredentials.from(credentials)); } else { final ManagedChannelBuilder channelBuilder = ManagedChannelBuilder .forTarget(target) @@ -216,6 +253,8 @@ public GrpcDataSource(final String target, final boolean encrypted) throws IOExc .usePlaintext() .build(); ocsServiceStub = OcsServiceGrpc.newStub(channel); + + ocsgwAnalyticsServiceStub = OcsgwAnalyticsServiceGrpc.newStub(channel); } } @@ -227,6 +266,8 @@ public void init() { initActivate(); initKeepAlive(); + + initAnalyticsRequest(); } private void initCreditControlRequest() { @@ -238,6 +279,18 @@ public void onNext(CreditControlAnswerInfo answer) { }); } + private void initAnalyticsRequest() { + ocsgwAnalyticsReport = ocsgwAnalyticsServiceStub.ocsgwAnalyticsEvent( + new AnalyticsRequestObserver() { + + @Override + public void onNext(OcsgwAnalyticsReply value) { + // Ignore reply from Prime + } + } + ); + } + private void handleGrpcCcrAnswer(CreditControlAnswerInfo answer) { try { LOG.info("[<<] Received data bucket for {}", answer.getMsisdn()); @@ -291,9 +344,7 @@ private void removeFromSessionMap(CreditControlContext creditControlContext) { private void updateAnalytics() { LOG.info("Number of active sesssions is {}", sessionIdMap.size()); - - // ToDo: Send to analytics - + ocsgwAnalyticsReport.onNext(OcsgwAnalyticsReport.newBuilder().setActiveSessions(sessionIdMap.size()).build()); } private void initActivate() { From 9e6c7bfe2d27a200b42ea4f319ae0beb24a975cc Mon Sep 17 00:00:00 2001 From: Martin Cederlof Date: Thu, 23 Aug 2018 10:31:05 +0200 Subject: [PATCH 04/30] Moved ocsgw analytics reporting to new class --- .../ocsgw/data/grpc/GrpcDataSource.java | 58 ++----------- .../ocsgw/data/grpc/OcsgwAnalytics.java | 84 +++++++++++++++++++ 2 files changed, 90 insertions(+), 52 deletions(-) create mode 100644 ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwAnalytics.java diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java index f260dc79c..b2babfeb5 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java @@ -69,9 +69,7 @@ public class GrpcDataSource implements DataSource { private StreamObserver creditControlRequest; - private final OcsgwAnalyticsServiceGrpc.OcsgwAnalyticsServiceStub ocsgwAnalyticsServiceStub; - - private StreamObserver ocsgwAnalyticsReport; + private OcsgwAnalytics ocsgwAnalytics; private ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); @@ -81,8 +79,6 @@ public class GrpcDataSource implements DataSource { private ScheduledFuture initCCRFuture = null; - private ScheduledFuture initAnalyticsFuture = null; - private static final int MAX_ENTRIES = 50000; private final LinkedHashMap ccrMap = new LinkedHashMap(MAX_ENTRIES, .75F) { @Override @@ -112,19 +108,6 @@ public final void onCompleted() { } } - private abstract class AnalyticsRequestObserver implements StreamObserver { - public final void onError(Throwable t) { - LOG.error("AnalyticsRequestObserver error", t); - if (t instanceof StatusRuntimeException) { - reconnectAnalyticsReport(); - } - } - - public final void onCompleted() { - // Nothing to do here - } - } - private abstract class ActivateObserver implements StreamObserver { public final void onError(Throwable t) { LOG.error("ActivateObserver error", t); @@ -155,24 +138,6 @@ private void reconnectActivate() { TimeUnit.SECONDS); } - - private void reconnectAnalyticsReport() { - LOG.info("reconnectAnalyticsReport called"); - - if (initAnalyticsFuture != null) { - initAnalyticsFuture.cancel(true); - } - - LOG.info("Schedule new Callable initAnalyticsRequest"); - initAnalyticsFuture = executorService.schedule((Callable) () -> { - LOG.info("Calling initAnalyticsRequest"); - initAnalyticsRequest(); - return "Called!"; - }, - 5, - TimeUnit.SECONDS); - } - private void reconnectCcrKeepAlive() { LOG.info("reconnectCreditControlRequest called"); if (keepAliveFuture != null) { @@ -240,8 +205,7 @@ public GrpcDataSource(final String target, final boolean encrypted) throws IOExc ocsServiceStub = OcsServiceGrpc.newStub(channel) .withCallCredentials(MoreCallCredentials.from(credentials)); - ocsgwAnalyticsServiceStub = OcsgwAnalyticsServiceGrpc.newStub(channel) - .withCallCredentials(MoreCallCredentials.from(credentials)); + ocsgwAnalytics = new OcsgwAnalytics(channel, credentials); } else { final ManagedChannelBuilder channelBuilder = ManagedChannelBuilder .forTarget(target) @@ -254,7 +218,7 @@ public GrpcDataSource(final String target, final boolean encrypted) throws IOExc .build(); ocsServiceStub = OcsServiceGrpc.newStub(channel); - ocsgwAnalyticsServiceStub = OcsgwAnalyticsServiceGrpc.newStub(channel); + ocsgwAnalytics = new OcsgwAnalytics(channel, null); } } @@ -267,7 +231,7 @@ public void init() { initKeepAlive(); - initAnalyticsRequest(); + ocsgwAnalytics.initAnalyticsRequest(); } private void initCreditControlRequest() { @@ -279,17 +243,6 @@ public void onNext(CreditControlAnswerInfo answer) { }); } - private void initAnalyticsRequest() { - ocsgwAnalyticsReport = ocsgwAnalyticsServiceStub.ocsgwAnalyticsEvent( - new AnalyticsRequestObserver() { - - @Override - public void onNext(OcsgwAnalyticsReply value) { - // Ignore reply from Prime - } - } - ); - } private void handleGrpcCcrAnswer(CreditControlAnswerInfo answer) { try { @@ -344,7 +297,8 @@ private void removeFromSessionMap(CreditControlContext creditControlContext) { private void updateAnalytics() { LOG.info("Number of active sesssions is {}", sessionIdMap.size()); - ocsgwAnalyticsReport.onNext(OcsgwAnalyticsReport.newBuilder().setActiveSessions(sessionIdMap.size()).build()); + + ocsgwAnalytics.sendAnalytics(sessionIdMap.size()); } private void initActivate() { diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwAnalytics.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwAnalytics.java new file mode 100644 index 000000000..daafe2b0b --- /dev/null +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwAnalytics.java @@ -0,0 +1,84 @@ +package org.ostelco.ocsgw.data.grpc; + +import com.google.auth.oauth2.ServiceAccountJwtAccessCredentials; +import io.grpc.ManagedChannel; +import io.grpc.StatusRuntimeException; +import io.grpc.auth.MoreCallCredentials; +import io.grpc.stub.StreamObserver; +import org.ostelco.analytics.grpc.api.OcsgwAnalyticsReply; +import org.ostelco.analytics.grpc.api.OcsgwAnalyticsReport; +import org.ostelco.analytics.grpc.api.OcsgwAnalyticsServiceGrpc; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.*; + +public class OcsgwAnalytics { + + private static final Logger LOG = LoggerFactory.getLogger(OcsgwAnalytics.class); + + private final OcsgwAnalyticsServiceGrpc.OcsgwAnalyticsServiceStub ocsgwAnalyticsServiceStub; + + private ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); + + private StreamObserver ocsgwAnalyticsReport; + + private ScheduledFuture initAnalyticsFuture = null; + + + public OcsgwAnalytics(ManagedChannel channel, ServiceAccountJwtAccessCredentials credentials) { + if (credentials != null) { + ocsgwAnalyticsServiceStub = OcsgwAnalyticsServiceGrpc.newStub(channel) + .withCallCredentials(MoreCallCredentials.from(credentials)); + } else { + ocsgwAnalyticsServiceStub = OcsgwAnalyticsServiceGrpc.newStub(channel); + } + } + + private abstract class AnalyticsRequestObserver implements StreamObserver { + public final void onError(Throwable t) { + LOG.error("AnalyticsRequestObserver error", t); + if (t instanceof StatusRuntimeException) { + reconnectAnalyticsReport(); + } + } + + public final void onCompleted() { + // Nothing to do here + } + } + + private void reconnectAnalyticsReport() { + LOG.info("reconnectAnalyticsReport called"); + + if (initAnalyticsFuture != null) { + initAnalyticsFuture.cancel(true); + } + + LOG.info("Schedule new Callable initAnalyticsRequest"); + initAnalyticsFuture = executorService.schedule((Callable) () -> { + LOG.info("Calling initAnalyticsRequest"); + initAnalyticsRequest(); + return "Called!"; + }, + 5, + TimeUnit.SECONDS); + } + + public void initAnalyticsRequest() { + ocsgwAnalyticsReport = ocsgwAnalyticsServiceStub.ocsgwAnalyticsEvent( + new AnalyticsRequestObserver() { + + @Override + public void onNext(OcsgwAnalyticsReply value) { + // Ignore reply from Prime + } + } + ); + } + + public void sendAnalytics(int size) { + ocsgwAnalyticsReport.onNext(OcsgwAnalyticsReport.newBuilder().setActiveSessions(size).build()); + } + +} \ No newline at end of file From 08fecc6c2ebfd6e2ec561f671f8295a13f786f05 Mon Sep 17 00:00:00 2001 From: Martin Cederlof Date: Thu, 23 Aug 2018 10:41:24 +0200 Subject: [PATCH 05/30] Remove unused import --- .../java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java index b2babfeb5..e8a177b2f 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java @@ -23,7 +23,6 @@ import org.ostelco.diameter.model.RedirectServer; import org.ostelco.diameter.model.SessionContext; import org.ostelco.ocs.grpc.api.*; -import org.ostelco.analytics.grpc.api.*; import org.ostelco.ocsgw.OcsServer; import org.ostelco.ocsgw.data.DataSource; import org.slf4j.Logger; @@ -333,7 +332,7 @@ private void initKeepAlive() { private void updateBlockedList(CreditControlAnswerInfo answer, CreditControlRequest request) { // This suffers from the fact that one Credit-Control-Request can have multiple MSCC for (org.ostelco.ocs.grpc.api.MultipleServiceCreditControl msccAnswer : answer.getMsccList()) { - for (org.ostelco.diameter.model.MultipleServiceCreditControl msccRequest : request.getMultipleServiceCreditControls()) { + for (MultipleServiceCreditControl msccRequest : request.getMultipleServiceCreditControls()) { if ((msccAnswer.getServiceIdentifier() == msccRequest.getServiceIdentifier()) && (msccAnswer.getRatingGroup() == msccRequest.getRatingGroup())) { updateBlockedList(msccAnswer, msccRequest, answer.getMsisdn()); return; @@ -454,7 +453,7 @@ private CreditControlAnswer createCreditControlAnswer(CreditControlAnswerInfo re return new CreditControlAnswer(multipleServiceCreditControls); } - private void updateBlockedList(org.ostelco.ocs.grpc.api.MultipleServiceCreditControl msccAnswer, org.ostelco.diameter.model.MultipleServiceCreditControl msccRequest, String msisdn) { + private void updateBlockedList(org.ostelco.ocs.grpc.api.MultipleServiceCreditControl msccAnswer, MultipleServiceCreditControl msccRequest, String msisdn) { if (!msccRequest.getRequested().isEmpty()) { if (msccAnswer.getGranted().getTotalOctets() < msccRequest.getRequested().get(0).getTotal()) { blocked.add(msisdn); From 805ad1edac6e1ba3f287035143da5231e51282aa Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Thu, 23 Aug 2018 15:19:08 +0200 Subject: [PATCH 06/30] Adds analyics module for collection events (wip) --- analytics-module/build.gradle | 17 ++++++ .../prime/analytics/AnalyticsGrpcServer.kt | 56 +++++++++++++++++++ .../prime/analytics/AnalyticsGrpcService.kt | 54 ++++++++++++++++++ .../prime/analytics/AnalyticsModule.kt | 29 ++++++++++ .../prime/analytics/AnalyticsService.kt | 2 + .../prime/analytics/metrics/OcsgwMetrics.kt | 11 ++++ settings.gradle | 2 + 7 files changed, 171 insertions(+) create mode 100644 analytics-module/build.gradle create mode 100644 analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcServer.kt create mode 100644 analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt create mode 100644 analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt create mode 100644 analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt create mode 100644 analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt diff --git a/analytics-module/build.gradle b/analytics-module/build.gradle new file mode 100644 index 000000000..80fc3a36e --- /dev/null +++ b/analytics-module/build.gradle @@ -0,0 +1,17 @@ +plugins { + id "org.jetbrains.kotlin.jvm" version "1.2.61" + id "java-library" +} + +dependencies { + implementation project(":prime-api") + + implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion" + implementation "io.dropwizard:dropwizard-core:$dropwizardVersion" + + testImplementation "io.dropwizard:dropwizard-testing:$dropwizardVersion" + testImplementation 'org.mockito:mockito-core:2.18.3' + testImplementation 'org.assertj:assertj-core:3.10.0' +} + +apply from: '../jacoco.gradle' diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcServer.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcServer.kt new file mode 100644 index 000000000..8a2515f06 --- /dev/null +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcServer.kt @@ -0,0 +1,56 @@ +package org.ostelco.prime.analytics + +import io.dropwizard.lifecycle.Managed +import io.grpc.BindableService +import io.grpc.Server +import io.grpc.ServerBuilder +import org.ostelco.prime.logger +import java.io.IOException + +/** + * This is Analytics Server running on gRPC protocol. + * Its startup and shutdown are managed by Dropwizard's lifecycle + * through the Managed interface. + * + */ +class AnalyticsGrpcServer(private val port: Int, service: BindableService) : Managed { + + private val logger by logger() + + // may add Transport Security with Certificates if needed. + // may add executor for control over number of threads + private val server: Server = ServerBuilder.forPort(port).addService(service).build() + + /** + * Startup is managed by Dropwizard's lifecycle. + * + * @throws IOException ... sometimes, perhaps. + */ + override fun start() { + server.start() + logger.info("Analytics Server started, listening for incoming gRPC traffic on {}", port) + } + + /** + * Shutdown is managed by Dropwizard's lifecycle. + * + * @throws InterruptedException When something goes wrong. + */ + override fun stop() { + logger.info("Stopping Analytics Server listening for gRPC traffic on {}", port) + server.shutdown() + blockUntilShutdown() + } + + /** + * Used for unit testing + */ + fun forceStop() { + logger.info("Stopping forcefully Analytics Server listening for gRPC traffic on {}", port) + server.shutdownNow() + } + + private fun blockUntilShutdown() { + server.awaitTermination() + } +} diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt new file mode 100644 index 000000000..832726aaf --- /dev/null +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt @@ -0,0 +1,54 @@ +package org.ostelco.prime.analytics + +import io.grpc.stub.StreamObserver +import org.ostelco.analytics.grpc.api.OcsgwAnalyticsReport +import org.ostelco.analytics.grpc.api.OcsgwAnalyticsReply +import org.ostelco.analytics.grpc.api.OcsgwAnalyticsServiceGrpc +import org.ostelco.prime.analytics.metrics.OcsgwMetrics +import org.ostelco.prime.logger +import java.util.* + + +class AnalyticsGrpcService(private val metrics : OcsgwMetrics) : OcsgwAnalyticsServiceGrpc.OcsgwAnalyticsServiceImplBase() { + + private val logger by logger() + + // rpc OcsgwAnalyticsEvent (stream OcsgwAnalyticsReport) returns (OcsgwAnalyticsReply) {} + + override fun ocsgwAnalyticsEvent(ocsgwAnalyticsReply: StreamObserver): StreamObserver { + + val streamId = newUniqueStreamId() + + return StreamObserverForStreamWithId(streamId) + } + + /** + * Return an unique ID based on Java's UUID generator that uniquely + * identifies a stream of values. + * @return A new unique identifier. + */ + private fun newUniqueStreamId(): String { + return UUID.randomUUID().toString() + } + + private inner class StreamObserverForStreamWithId internal constructor(private val streamId: String) : StreamObserver { + + /** + * This method gets called every time a Credit-Control-Request is received + * from the OCS. + * @param request + */ + override fun onNext(request: OcsgwAnalyticsReport) { + // + } + + override fun onError(t: Throwable) { + // TODO vihang: handle onError for stream observers + } + + override fun onCompleted() { + logger.info("AnalyticsGrpcService with streamId: {} completed", streamId) + + } + } +} diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt new file mode 100644 index 000000000..917c3a069 --- /dev/null +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt @@ -0,0 +1,29 @@ +package org.ostelco.prime.analytics + +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.annotation.JsonTypeName +import io.dropwizard.setup.Environment +import com.codahale.metrics.SharedMetricRegistries +import org.hibernate.validator.constraints.NotEmpty +import org.ostelco.prime.module.PrimeModule +import org.ostelco.prime.analytics.metrics.OcsgwMetrics + +@JsonTypeName("analytics") +class AnalyticsModule : PrimeModule { + + @JsonProperty("config") + lateinit var config: AnalyticsConfig + + override fun init(env: Environment) { + + val ocsgwMetrics = OcsgwMetrics(SharedMetricRegistries.getOrCreate("ocsgw-metrics")) + + val server = AnalyticsGrpcServer(8082, AnalyticsGrpcService(ocsgwMetrics)) + + env.lifecycle().manage(server) + } +} + +class AnalyticsConfig { + +} \ No newline at end of file diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt new file mode 100644 index 000000000..500f4733e --- /dev/null +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt @@ -0,0 +1,2 @@ +package org.ostelco.prime.analytics + diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt new file mode 100644 index 000000000..6d1dfeaf1 --- /dev/null +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt @@ -0,0 +1,11 @@ +package org.ostelco.prime.analytics.metrics + +import com.codahale.metrics.MetricRegistry + + +class OcsgwMetrics(private val registry: MetricRegistry) { + + init { + + } +} diff --git a/settings.gradle b/settings.gradle index bbc094fbf..b00b5c3ed 100644 --- a/settings.gradle +++ b/settings.gradle @@ -4,6 +4,7 @@ include ':acceptance-tests' include ':app-notifier' include ':admin-api' include ':analytics-grpc-api' +include ':analytics-module' include ':analytics' include ':auth-server' include ':client-api' @@ -28,6 +29,7 @@ project(':acceptance-tests').projectDir = "$rootDir/acceptance-tests" as File project(':app-notifier').projectDir = "$rootDir/app-notifier" as File project(':admin-api').projectDir = "$rootDir/admin-api" as File project(':analytics-grpc-api').projectDir = "$rootDir/analytics-grpc-api" as File +project(':analytics-module').projectDir = "$rootDir/analytics-module" as File project(':analytics').projectDir = "$rootDir/analytics" as File project(':auth-server').projectDir = "$rootDir/auth-server" as File project(':client-api').projectDir = "$rootDir/client-api" as File From 648722af6c07ddc1ec3ef113a8bccf84cb25ba4b Mon Sep 17 00:00:00 2001 From: Martin Cederlof Date: Thu, 23 Aug 2018 16:22:56 +0200 Subject: [PATCH 07/30] Start moving DataConsumptionInfoPublisher to analytics module --- .../org/ostelco/prime/analytics/AnalyticsModule.kt | 6 ++++++ .../org/ostelco/prime/analytics/AnalyticsService.kt | 2 -- .../ostelco/prime/analytics/AnalyticsServiceImpl.kt | 12 ++++++++++++ .../services/io.dropwizard.jackson.Discoverable | 1 + .../org.ostelco.prime.analytics.AnalyticsService | 1 + .../services/org.ostelco.prime.module.PrimeModule | 1 + .../prime/analytics/DataConsumptionInfoPublisher.kt | 9 +++++++-- .../main/kotlin/org/ostelco/prime/ocs/OcsService.kt | 2 +- .../org/ostelco/prime/analytics/AnalyticsService.kt | 5 +++++ prime/build.gradle | 1 + 10 files changed, 35 insertions(+), 5 deletions(-) delete mode 100644 analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt create mode 100644 analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsServiceImpl.kt create mode 100644 analytics-module/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable create mode 100644 analytics-module/src/main/resources/META-INF/services/org.ostelco.prime.analytics.AnalyticsService create mode 100644 analytics-module/src/main/resources/META-INF/services/org.ostelco.prime.module.PrimeModule create mode 100644 prime-api/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt index 917c3a069..9a24e362a 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt @@ -25,5 +25,11 @@ class AnalyticsModule : PrimeModule { } class AnalyticsConfig { + @NotEmpty + @JsonProperty("projectId") + lateinit var projectId: String + @NotEmpty + @JsonProperty("topicId") + lateinit var topicId: String } \ No newline at end of file diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt deleted file mode 100644 index 500f4733e..000000000 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt +++ /dev/null @@ -1,2 +0,0 @@ -package org.ostelco.prime.analytics - diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsServiceImpl.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsServiceImpl.kt new file mode 100644 index 000000000..6253d0e9e --- /dev/null +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsServiceImpl.kt @@ -0,0 +1,12 @@ +package org.ostelco.prime.analytics + +import org.ostelco.prime.logger + +class AnalyticsServiceImpl : AnalyticsService { + + private val logger by logger() + + override fun reportTrafficInfo(msisdn: String, usedBytes: Long, bundleBytes: Long) { + logger.info("reportTrafficInfo : msisdn {} usedBytes {} bundleBytes {}", msisdn, usedBytes, bundleBytes) + } +} \ No newline at end of file diff --git a/analytics-module/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable b/analytics-module/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable new file mode 100644 index 000000000..8056fe23b --- /dev/null +++ b/analytics-module/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable @@ -0,0 +1 @@ +org.ostelco.prime.module.PrimeModule \ No newline at end of file diff --git a/analytics-module/src/main/resources/META-INF/services/org.ostelco.prime.analytics.AnalyticsService b/analytics-module/src/main/resources/META-INF/services/org.ostelco.prime.analytics.AnalyticsService new file mode 100644 index 000000000..f4f73dd16 --- /dev/null +++ b/analytics-module/src/main/resources/META-INF/services/org.ostelco.prime.analytics.AnalyticsService @@ -0,0 +1 @@ +org.ostelco.prime.analytics.AnalyticsServiceImpl \ No newline at end of file diff --git a/analytics-module/src/main/resources/META-INF/services/org.ostelco.prime.module.PrimeModule b/analytics-module/src/main/resources/META-INF/services/org.ostelco.prime.module.PrimeModule new file mode 100644 index 000000000..46d3739c1 --- /dev/null +++ b/analytics-module/src/main/resources/META-INF/services/org.ostelco.prime.module.PrimeModule @@ -0,0 +1 @@ +org.ostelco.prime.analytics.AnalyticsModule \ No newline at end of file diff --git a/ocs/src/main/kotlin/org/ostelco/prime/analytics/DataConsumptionInfoPublisher.kt b/ocs/src/main/kotlin/org/ostelco/prime/analytics/DataConsumptionInfoPublisher.kt index 82a230674..a2d3cdff6 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/analytics/DataConsumptionInfoPublisher.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/analytics/DataConsumptionInfoPublisher.kt @@ -13,6 +13,7 @@ import org.ostelco.analytics.grpc.api.DataTrafficInfo import org.ostelco.prime.disruptor.OcsEvent import org.ostelco.prime.disruptor.EventMessageType.CREDIT_CONTROL_REQUEST import org.ostelco.prime.logger +import org.ostelco.prime.module.getResource import java.io.IOException import java.time.Instant @@ -23,6 +24,8 @@ class DataConsumptionInfoPublisher(private val projectId: String, private val to private val logger by logger() + private val analyticsReporter by lazy { getResource() } + private lateinit var publisher: Publisher @Throws(IOException::class) @@ -49,10 +52,12 @@ class DataConsumptionInfoPublisher(private val projectId: String, private val to return } - // FIXME vihang: We only report the requested bucket. Should probably report the Used-Units instead + // FIXME martin: Move rest of this code to the analytics module + if (event.msisdn != null) analyticsReporter.reportTrafficInfo(event.msisdn!!, event.usedBucketBytes, event.bundleBytes) + val data = DataTrafficInfo.newBuilder() .setMsisdn(event.msisdn) - .setBucketBytes(event.requestedBucketBytes) + .setBucketBytes(event.usedBucketBytes) .setBundleBytes(event.bundleBytes) .setTimestamp(Timestamps.fromMillis(Instant.now().toEpochMilli())) .build() diff --git a/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsService.kt b/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsService.kt index 9c1c50506..fdec7d1fa 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsService.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsService.kt @@ -56,7 +56,7 @@ class OcsService(private val producer: EventProducer) { * build(); ` * * - * @return The service that can receive incoming GPRS messages + * @return The service that can receive incoming GRPC messages */ fun asOcsServiceImplBase(): OcsServiceGrpc.OcsServiceImplBase { return this.ocsServerImplBaseImpl diff --git a/prime-api/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt b/prime-api/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt new file mode 100644 index 000000000..325d83403 --- /dev/null +++ b/prime-api/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt @@ -0,0 +1,5 @@ +package org.ostelco.prime.analytics + +interface AnalyticsService { + fun reportTrafficInfo(msisdn: String, usedBytes: Long, bundleBytes: Long) +} diff --git a/prime/build.gradle b/prime/build.gradle index 6178de08b..92b41f6d8 100644 --- a/prime/build.gradle +++ b/prime/build.gradle @@ -38,6 +38,7 @@ dependencies { runtimeOnly project(':admin-api') runtimeOnly project(':app-notifier') runtimeOnly project(':payment-processor') + runtimeOnly project(':analytics-module') implementation "com.fasterxml.jackson.module:jackson-module-kotlin:$jacksonVersion" implementation "io.dropwizard:dropwizard-http2:$dropwizardVersion" From bb62c979fb5fd7233ff452c6a550deb329086325 Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Fri, 24 Aug 2018 09:28:45 +0200 Subject: [PATCH 08/30] Adds counter metrics to count active user sessions --- .../prime/analytics/AnalyticsGrpcService.kt | 28 +++++++++++++------ .../prime/analytics/metrics/OcsgwMetrics.kt | 18 +++++++++++- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt index 832726aaf..27b0798f1 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt @@ -9,16 +9,27 @@ import org.ostelco.prime.logger import java.util.* +/** + * Serves incoming GRPC analytcs requests. + * + * It's implemented as a subclass of [OcsServiceGrpc.OcsServiceImplBase] overriding + * methods that together implements the protocol described in the analytics protobuf + * file: ocs_analytics.proto + *` + * service OcsgwAnalyticsService { + * rpc OcsgwAnalyticsEvent (stream OcsgwAnalyticsReport) returns (OcsgwAnalyticsReply) {} + * } + */ + class AnalyticsGrpcService(private val metrics : OcsgwMetrics) : OcsgwAnalyticsServiceGrpc.OcsgwAnalyticsServiceImplBase() { private val logger by logger() - // rpc OcsgwAnalyticsEvent (stream OcsgwAnalyticsReport) returns (OcsgwAnalyticsReply) {} - + /** + * Handles the OcsgwAnalyticsEvent message. + */ override fun ocsgwAnalyticsEvent(ocsgwAnalyticsReply: StreamObserver): StreamObserver { - val streamId = newUniqueStreamId() - return StreamObserverForStreamWithId(streamId) } @@ -34,12 +45,12 @@ class AnalyticsGrpcService(private val metrics : OcsgwMetrics) : OcsgwAnalyticsS private inner class StreamObserverForStreamWithId internal constructor(private val streamId: String) : StreamObserver { /** - * This method gets called every time a Credit-Control-Request is received - * from the OCS. - * @param request + * This method gets called every time a new active session count is sent + * from the OCS GW. + * @param request provides current active session as a counter with a timestamp */ override fun onNext(request: OcsgwAnalyticsReport) { - // + metrics.setActiveSessions(request.timestamp.seconds, request.activeSessions.toLong()) } override fun onError(t: Throwable) { @@ -48,7 +59,6 @@ class AnalyticsGrpcService(private val metrics : OcsgwMetrics) : OcsgwAnalyticsS override fun onCompleted() { logger.info("AnalyticsGrpcService with streamId: {} completed", streamId) - } } } diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt index 6d1dfeaf1..f51c24289 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt @@ -1,11 +1,27 @@ package org.ostelco.prime.analytics.metrics +import com.codahale.metrics.Counter import com.codahale.metrics.MetricRegistry - class OcsgwMetrics(private val registry: MetricRegistry) { + private val activeSessions : Counter + init { + this.activeSessions = Counter() + } + /** + * Records active user sessions. + * @param timestamp - seconds since epoch + * @param count - active sessions + */ + @Synchronized fun setActiveSessions(timestamp: Long, count: Long) { + val currentCount = activeSessions.getCount() + if (currentCount < count) { + activeSessions.inc(count - currentCount) + } else if (currentCount > count) { + activeSessions.dec(currentCount - count) + } } } From bfb25005269f61c2b56fe08c37c7209d6d2a1857 Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Fri, 24 Aug 2018 11:41:13 +0200 Subject: [PATCH 09/30] Updates counter metric to be correctly set up with metrics framework --- .../org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt index f51c24289..9759da2bb 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt @@ -2,13 +2,17 @@ package org.ostelco.prime.analytics.metrics import com.codahale.metrics.Counter import com.codahale.metrics.MetricRegistry +import com.codahale.metrics.MetricRegistry.name +import org.ostelco.prime.analytics.AnalyticsGrpcService + class OcsgwMetrics(private val registry: MetricRegistry) { private val activeSessions : Counter init { - this.activeSessions = Counter() + this.activeSessions = registry.counter(name(AnalyticsGrpcService::class.java, + "active-sessions")) } /** From 3c69bdcdf5abe0bcf628ec034dd411dbbd5e042f Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Fri, 24 Aug 2018 13:49:11 +0200 Subject: [PATCH 10/30] Updates to use the dropwizards gauge metric for active sessions --- .../prime/analytics/metrics/OcsgwMetrics.kt | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt index 9759da2bb..bb0a317d3 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt @@ -1,6 +1,6 @@ package org.ostelco.prime.analytics.metrics -import com.codahale.metrics.Counter +import com.codahale.metrics.Gauge import com.codahale.metrics.MetricRegistry import com.codahale.metrics.MetricRegistry.name import org.ostelco.prime.analytics.AnalyticsGrpcService @@ -8,11 +8,17 @@ import org.ostelco.prime.analytics.AnalyticsGrpcService class OcsgwMetrics(private val registry: MetricRegistry) { - private val activeSessions : Counter + private val activeSessions: Gauge + + private val currentActiveSessions = object { + var timestamp: Long = 0 + var count: Long = 0 + } init { - this.activeSessions = registry.counter(name(AnalyticsGrpcService::class.java, - "active-sessions")) + this.activeSessions = registry.register( + name(AnalyticsGrpcService::class.java, "active-sessions"), + Gauge { currentActiveSessions.count }) } /** @@ -20,12 +26,9 @@ class OcsgwMetrics(private val registry: MetricRegistry) { * @param timestamp - seconds since epoch * @param count - active sessions */ - @Synchronized fun setActiveSessions(timestamp: Long, count: Long) { - val currentCount = activeSessions.getCount() - if (currentCount < count) { - activeSessions.inc(count - currentCount) - } else if (currentCount > count) { - activeSessions.dec(currentCount - count) - } + @Synchronized + fun setActiveSessions(timestamp: Long, count: Long) { + currentActiveSessions.timestamp = timestamp + currentActiveSessions.count = count } } From 5af4b96abf30a6ada4337f4832ae046cff7ae3c3 Mon Sep 17 00:00:00 2001 From: Martin Cederlof Date: Fri, 24 Aug 2018 13:48:52 +0200 Subject: [PATCH 11/30] Move DataConsumtionInfo to analytics module --- analytics-module/build.gradle | 1 + .../prime/analytics/AnalyticsModule.kt | 15 +++++++-- .../prime/analytics/AnalyticsServiceImpl.kt | 2 ++ .../DataConsumptionInfoPublisher.kt | 33 +++++-------------- ocs/build.gradle | 1 - .../prime/analytics/DataConsumptionInfo.kt | 32 ++++++++++++++++++ .../kotlin/org/ostelco/prime/ocs/OcsModule.kt | 19 ++--------- prime/config/config.yaml | 4 ++- prime/config/test.yaml | 4 ++- .../integration-tests/resources/config.yaml | 4 ++- 10 files changed, 69 insertions(+), 46 deletions(-) rename {ocs/src/main/kotlin/org/ostelco/prime/analytics => analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers}/DataConsumptionInfoPublisher.kt (67%) create mode 100644 ocs/src/main/kotlin/org/ostelco/prime/analytics/DataConsumptionInfo.kt diff --git a/analytics-module/build.gradle b/analytics-module/build.gradle index 80fc3a36e..adcf08f5a 100644 --- a/analytics-module/build.gradle +++ b/analytics-module/build.gradle @@ -8,6 +8,7 @@ dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion" implementation "io.dropwizard:dropwizard-core:$dropwizardVersion" + implementation "com.google.cloud:google-cloud-pubsub:$googleCloudVersion" testImplementation "io.dropwizard:dropwizard-testing:$dropwizardVersion" testImplementation 'org.mockito:mockito-core:2.18.3' diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt index 9a24e362a..d9217ed46 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt @@ -5,22 +5,29 @@ import com.fasterxml.jackson.annotation.JsonTypeName import io.dropwizard.setup.Environment import com.codahale.metrics.SharedMetricRegistries import org.hibernate.validator.constraints.NotEmpty +import org.ostelco.prime.analytics.ConfigRegistry.config import org.ostelco.prime.module.PrimeModule import org.ostelco.prime.analytics.metrics.OcsgwMetrics +import org.ostelco.prime.analytics.publishers.DataConsumptionInfoPublisher @JsonTypeName("analytics") class AnalyticsModule : PrimeModule { @JsonProperty("config") - lateinit var config: AnalyticsConfig + fun setConfig(config: AnalyticsConfig) { + ConfigRegistry.config = config + } override fun init(env: Environment) { val ocsgwMetrics = OcsgwMetrics(SharedMetricRegistries.getOrCreate("ocsgw-metrics")) - val server = AnalyticsGrpcServer(8082, AnalyticsGrpcService(ocsgwMetrics)) + val server = AnalyticsGrpcServer(8083, AnalyticsGrpcService(ocsgwMetrics)) env.lifecycle().manage(server) + + // dropwizard starts Analytics events publisher + env.lifecycle().manage(DataConsumptionInfoPublisher) } } @@ -32,4 +39,8 @@ class AnalyticsConfig { @NotEmpty @JsonProperty("topicId") lateinit var topicId: String +} + +object ConfigRegistry { + lateinit var config: AnalyticsConfig } \ No newline at end of file diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsServiceImpl.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsServiceImpl.kt index 6253d0e9e..6e2151fe8 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsServiceImpl.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsServiceImpl.kt @@ -1,5 +1,6 @@ package org.ostelco.prime.analytics +import org.ostelco.prime.analytics.publishers.DataConsumptionInfoPublisher import org.ostelco.prime.logger class AnalyticsServiceImpl : AnalyticsService { @@ -8,5 +9,6 @@ class AnalyticsServiceImpl : AnalyticsService { override fun reportTrafficInfo(msisdn: String, usedBytes: Long, bundleBytes: Long) { logger.info("reportTrafficInfo : msisdn {} usedBytes {} bundleBytes {}", msisdn, usedBytes, bundleBytes) + DataConsumptionInfoPublisher.publish(msisdn, usedBytes, bundleBytes) } } \ No newline at end of file diff --git a/ocs/src/main/kotlin/org/ostelco/prime/analytics/DataConsumptionInfoPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DataConsumptionInfoPublisher.kt similarity index 67% rename from ocs/src/main/kotlin/org/ostelco/prime/analytics/DataConsumptionInfoPublisher.kt rename to analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DataConsumptionInfoPublisher.kt index a2d3cdff6..df8f00c49 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/analytics/DataConsumptionInfoPublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DataConsumptionInfoPublisher.kt @@ -1,4 +1,4 @@ -package org.ostelco.prime.analytics +package org.ostelco.prime.analytics.publishers import com.google.api.core.ApiFutureCallback import com.google.api.core.ApiFutures @@ -7,31 +7,26 @@ import com.google.cloud.pubsub.v1.Publisher import com.google.protobuf.util.Timestamps import com.google.pubsub.v1.ProjectTopicName import com.google.pubsub.v1.PubsubMessage -import com.lmax.disruptor.EventHandler import io.dropwizard.lifecycle.Managed import org.ostelco.analytics.grpc.api.DataTrafficInfo -import org.ostelco.prime.disruptor.OcsEvent -import org.ostelco.prime.disruptor.EventMessageType.CREDIT_CONTROL_REQUEST +import org.ostelco.prime.analytics.ConfigRegistry.config import org.ostelco.prime.logger -import org.ostelco.prime.module.getResource import java.io.IOException import java.time.Instant /** * This class publishes the data consumption information events to the Google Cloud Pub/Sub. */ -class DataConsumptionInfoPublisher(private val projectId: String, private val topicId: String) : EventHandler, Managed { +object DataConsumptionInfoPublisher : Managed { private val logger by logger() - private val analyticsReporter by lazy { getResource() } - private lateinit var publisher: Publisher @Throws(IOException::class) override fun start() { - val topicName = ProjectTopicName.of(projectId, topicId) + val topicName = ProjectTopicName.of(config.projectId, config.topicId) // Create a publisher instance with default settings bound to the topic publisher = Publisher.newBuilder(topicName).build() @@ -43,22 +38,12 @@ class DataConsumptionInfoPublisher(private val projectId: String, private val to publisher.shutdown() } - override fun onEvent( - event: OcsEvent, - sequence: Long, - endOfBatch: Boolean) { - - if (event.messageType != CREDIT_CONTROL_REQUEST) { - return - } - - // FIXME martin: Move rest of this code to the analytics module - if (event.msisdn != null) analyticsReporter.reportTrafficInfo(event.msisdn!!, event.usedBucketBytes, event.bundleBytes) + fun publish(msisdn: String, usedBucketBytes: Long, bundleBytes: Long) { val data = DataTrafficInfo.newBuilder() - .setMsisdn(event.msisdn) - .setBucketBytes(event.usedBucketBytes) - .setBundleBytes(event.bundleBytes) + .setMsisdn(msisdn) + .setBucketBytes(usedBucketBytes) + .setBundleBytes(bundleBytes) .setTimestamp(Timestamps.fromMillis(Instant.now().toEpochMilli())) .build() .toByteString() @@ -79,7 +64,7 @@ class DataConsumptionInfoPublisher(private val projectId: String, private val to logger.warn("Status code: {}", throwable.statusCode.code) logger.warn("Retrying: {}", throwable.isRetryable) } - logger.warn("Error publishing message for msisdn: {}", event.msisdn) + logger.warn("Error publishing message for msisdn: {}", msisdn) } override fun onSuccess(messageId: String) { diff --git a/ocs/build.gradle b/ocs/build.gradle index 979532b84..cf6c53e4d 100644 --- a/ocs/build.gradle +++ b/ocs/build.gradle @@ -6,7 +6,6 @@ plugins { dependencies { implementation project(':prime-api') - implementation "com.google.cloud:google-cloud-pubsub:$googleCloudVersion" implementation 'com.lmax:disruptor:3.4.2' testImplementation "org.jetbrains.kotlin:kotlin-test:$kotlinVersion" diff --git a/ocs/src/main/kotlin/org/ostelco/prime/analytics/DataConsumptionInfo.kt b/ocs/src/main/kotlin/org/ostelco/prime/analytics/DataConsumptionInfo.kt new file mode 100644 index 000000000..576ab059e --- /dev/null +++ b/ocs/src/main/kotlin/org/ostelco/prime/analytics/DataConsumptionInfo.kt @@ -0,0 +1,32 @@ +package org.ostelco.prime.analytics + +import com.lmax.disruptor.EventHandler +import org.ostelco.prime.disruptor.OcsEvent +import org.ostelco.prime.disruptor.EventMessageType.CREDIT_CONTROL_REQUEST +import org.ostelco.prime.logger +import org.ostelco.prime.module.getResource + +/** + * This class publishes the data consumption information events analytics. + */ +class DataConsumptionInfo() : EventHandler { + + private val logger by logger() + + private val analyticsReporter by lazy { getResource() } + + override fun onEvent( + event: OcsEvent, + sequence: Long, + endOfBatch: Boolean) { + + if (event.messageType != CREDIT_CONTROL_REQUEST) { + return + } + + if (event.msisdn != null) { + logger.info("Sent DataConsumptionInfo event to analytics") + analyticsReporter.reportTrafficInfo(event.msisdn!!, event.usedBucketBytes, event.bundleBytes) + } + } +} diff --git a/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsModule.kt b/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsModule.kt index aa559d007..af54645ee 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsModule.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsModule.kt @@ -4,7 +4,7 @@ import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.annotation.JsonTypeName import io.dropwizard.setup.Environment import org.hibernate.validator.constraints.NotEmpty -import org.ostelco.prime.analytics.DataConsumptionInfoPublisher +import org.ostelco.prime.analytics.DataConsumptionInfo import org.ostelco.prime.disruptor.ClearingEventHandler import org.ostelco.prime.disruptor.EventProducerImpl import org.ostelco.prime.disruptor.OcsDisruptor @@ -34,9 +34,7 @@ class OcsModule : PrimeModule { // OcsServer assigns OcsService as handler for gRPC requests val server = OcsGrpcServer(8082, ocsService.asOcsServiceImplBase()) - val dataConsumptionInfoPublisher = DataConsumptionInfoPublisher( - config.projectId, - config.topicId) + val dataConsumptionInfo = DataConsumptionInfo() val thresholdChecker = ThresholdChecker(config.lowBalanceThreshold) @@ -48,11 +46,9 @@ class OcsModule : PrimeModule { disruptor.disruptor .handleEventsWith(OcsState()) - .then(ocsService.asEventHandler(), EventProcessor(), thresholdChecker, dataConsumptionInfoPublisher) + .then(ocsService.asEventHandler(), EventProcessor(), thresholdChecker, dataConsumptionInfo) .then(ClearingEventHandler()) - // dropwizard starts Analytics events publisher - env.lifecycle().manage(dataConsumptionInfoPublisher) // dropwizard starts disruptor env.lifecycle().manage(disruptor) // dropwizard starts server @@ -62,15 +58,6 @@ class OcsModule : PrimeModule { class OcsConfig { - @NotEmpty - @JsonProperty("projectId") - lateinit var projectId: String - - @NotEmpty - @JsonProperty("topicId") - lateinit var topicId: String - - @NotEmpty @JsonProperty("lowBalanceThreshold") var lowBalanceThreshold: Long = 0 diff --git a/prime/config/config.yaml b/prime/config/config.yaml index cd52ffb82..84c757eb1 100644 --- a/prime/config/config.yaml +++ b/prime/config/config.yaml @@ -9,10 +9,12 @@ modules: host: neo4j protocol: bolt+routing - type: ocs + config: + lowBalanceThreshold: 100000000 + - type: analytics config: projectId: pantel-2decb topicId: data-traffic - lowBalanceThreshold: 100000000 - type: api config: authenticationCachePolicy: maximumSize=10000, expireAfterAccess=10m diff --git a/prime/config/test.yaml b/prime/config/test.yaml index efa68475e..c2af8a9be 100644 --- a/prime/config/test.yaml +++ b/prime/config/test.yaml @@ -11,10 +11,12 @@ modules: host: neo4j protocol: bolt - type: ocs + config: + lowBalanceThreshold: 0 + - type: analytics config: projectId: pantel-2decb topicId: data-traffic - lowBalanceThreshold: 0 - type: api config: authenticationCachePolicy: maximumSize=10000, expireAfterAccess=10m diff --git a/prime/src/integration-tests/resources/config.yaml b/prime/src/integration-tests/resources/config.yaml index 7d593c8d5..3379ef631 100644 --- a/prime/src/integration-tests/resources/config.yaml +++ b/prime/src/integration-tests/resources/config.yaml @@ -9,10 +9,12 @@ modules: host: 0.0.0.0 protocol: bolt - type: ocs + config: + lowBalanceThreshold: 0 + - type: analytics config: projectId: pantel-2decb topicId: data-traffic - lowBalanceThreshold: 0 - type: api config: authenticationCachePolicy: maximumSize=10000, expireAfterAccess=10m From 8611ae969e901ef96ff4fa2ef71e0b4997a04f21 Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Fri, 24 Aug 2018 14:02:16 +0200 Subject: [PATCH 12/30] Adds a prometheus module for dropwizard to prime enabling the /prometheus-metrics endpoint Note that the /prometheus-metrics endpoint will be on the 'admin' port Ref.: https://github.com/dhatim/dropwizard-prometheus --- prime/build.gradle | 2 ++ prime/src/main/kotlin/org/ostelco/prime/PrimeApplication.kt | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/prime/build.gradle b/prime/build.gradle index 92b41f6d8..bf1dc8015 100644 --- a/prime/build.gradle +++ b/prime/build.gradle @@ -44,11 +44,13 @@ dependencies { implementation "io.dropwizard:dropwizard-http2:$dropwizardVersion" implementation "io.dropwizard:dropwizard-json-logging:$dropwizardVersion" implementation 'com.google.guava:guava:25.1-jre' + implementation 'org.dhatim:dropwizard-prometheus:2.2.0' testImplementation "io.dropwizard:dropwizard-testing:$dropwizardVersion" testImplementation 'org.mockito:mockito-core:2.18.3' testImplementation 'com.lmax:disruptor:3.4.2' testImplementation 'com.palantir.docker.compose:docker-compose-rule-junit4:0.34.0' + testImplementation 'org.dhatim:dropwizard-prometheus:2.2.0' integrationImplementation "io.dropwizard:dropwizard-testing:$dropwizardVersion" integrationImplementation 'org.mockito:mockito-core:2.18.3' diff --git a/prime/src/main/kotlin/org/ostelco/prime/PrimeApplication.kt b/prime/src/main/kotlin/org/ostelco/prime/PrimeApplication.kt index a09cf6ba0..f8206a8ca 100644 --- a/prime/src/main/kotlin/org/ostelco/prime/PrimeApplication.kt +++ b/prime/src/main/kotlin/org/ostelco/prime/PrimeApplication.kt @@ -6,7 +6,7 @@ import io.dropwizard.configuration.EnvironmentVariableSubstitutor import io.dropwizard.configuration.SubstitutingSourceProvider import io.dropwizard.setup.Bootstrap import io.dropwizard.setup.Environment - +import org.dhatim.dropwizard.prometheus.PrometheusBundle fun main(args: Array) { PrimeApplication().run(*args) @@ -19,6 +19,7 @@ class PrimeApplication : Application() { bootstrap.configurationSourceProvider, EnvironmentVariableSubstitutor()) bootstrap.objectMapper.registerModule(KotlinModule()) + bootstrap.addBundle(PrometheusBundle()); } override fun run( From 956aea3a51e9511caa9f0a9e4e3eb6e6937a8b73 Mon Sep 17 00:00:00 2001 From: Martin Cederlof Date: Fri, 24 Aug 2018 14:53:14 +0200 Subject: [PATCH 13/30] Rollback name change on protobuf grpc --- .../src/main/proto/analytics.proto | 2 +- ...s_analytics.proto => prime_analytics.proto} | 4 ++-- .../prime/analytics/AnalyticsGrpcService.kt | 6 +++--- .../publishers/DataConsumptionInfoPublisher.kt | 2 +- .../analytics/DataConsumptionPipeline.kt | 4 ++-- .../org/ostelco/analytics/PipelineTest.kt | 4 ++-- ocs-grpc-api/src/main/proto/ocs.proto | 4 ++-- .../ostelco/prime/disruptor/EventProducer.kt | 2 +- .../prime/disruptor/EventProducerImpl.kt | 4 ++-- .../org/ostelco/prime/disruptor/OcsEvent.kt | 2 +- .../prime/ocs/ActivateResponseHolder.kt | 2 +- .../org/ostelco/prime/ocs/EventHandlerImpl.kt | 14 +++++++------- .../org/ostelco/prime/ocs/OcsGrpcService.kt | 12 ++++++------ .../kotlin/org/ostelco/prime/ocs/OcsService.kt | 8 ++++---- .../prime/disruptor/PrimeEventProducerTest.kt | 6 +++--- .../ocsgw/data/grpc/GrpcDataSource.java | 14 +++++++------- .../ocsgw/data/grpc/OcsgwAnalytics.java | 6 +++--- .../ocsgw/data/local/LocalDataSource.java | 2 +- .../ocsgw/data/proxy/ProxyDataSource.java | 2 +- .../kotlin/org/ostelco/prime/ocs/OcsTest.kt | 18 +++++++++--------- .../pseudonym/managed/MessageProcessor.kt | 2 +- 21 files changed, 60 insertions(+), 60 deletions(-) rename analytics-grpc-api/src/main/proto/{ocs_analytics.proto => prime_analytics.proto} (81%) diff --git a/analytics-grpc-api/src/main/proto/analytics.proto b/analytics-grpc-api/src/main/proto/analytics.proto index acff96728..f437c517c 100644 --- a/analytics-grpc-api/src/main/proto/analytics.proto +++ b/analytics-grpc-api/src/main/proto/analytics.proto @@ -3,7 +3,7 @@ syntax = "proto3"; package org.ostelco.analytics.grpc.api; option java_multiple_files = true; -option java_package = "org.ostelco.analytics.grpc.api"; +option java_package = "org.ostelco.analytics.api"; import "google/protobuf/timestamp.proto"; diff --git a/analytics-grpc-api/src/main/proto/ocs_analytics.proto b/analytics-grpc-api/src/main/proto/prime_analytics.proto similarity index 81% rename from analytics-grpc-api/src/main/proto/ocs_analytics.proto rename to analytics-grpc-api/src/main/proto/prime_analytics.proto index 310f410b2..327c1d62f 100644 --- a/analytics-grpc-api/src/main/proto/ocs_analytics.proto +++ b/analytics-grpc-api/src/main/proto/prime_analytics.proto @@ -1,9 +1,9 @@ syntax = "proto3"; -package org.ostelco.analytics.grpc.api; +package org.ostelco.analytics.api; option java_multiple_files = true; -option java_package = "org.ostelco.analytics.grpc.api"; +option java_package = "org.ostelco.prime.analytics.api"; import "google/protobuf/timestamp.proto"; diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt index 27b0798f1..bd70847ff 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt @@ -1,9 +1,9 @@ package org.ostelco.prime.analytics import io.grpc.stub.StreamObserver -import org.ostelco.analytics.grpc.api.OcsgwAnalyticsReport -import org.ostelco.analytics.grpc.api.OcsgwAnalyticsReply -import org.ostelco.analytics.grpc.api.OcsgwAnalyticsServiceGrpc +import org.ostelco.prime.analytics.api.OcsgwAnalyticsReply +import org.ostelco.prime.analytics.api.OcsgwAnalyticsReport +import org.ostelco.prime.analytics.api.OcsgwAnalyticsServiceGrpc import org.ostelco.prime.analytics.metrics.OcsgwMetrics import org.ostelco.prime.logger import java.util.* diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DataConsumptionInfoPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DataConsumptionInfoPublisher.kt index df8f00c49..feec66edc 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DataConsumptionInfoPublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DataConsumptionInfoPublisher.kt @@ -8,7 +8,7 @@ import com.google.protobuf.util.Timestamps import com.google.pubsub.v1.ProjectTopicName import com.google.pubsub.v1.PubsubMessage import io.dropwizard.lifecycle.Managed -import org.ostelco.analytics.grpc.api.DataTrafficInfo +import org.ostelco.analytics.api.DataTrafficInfo import org.ostelco.prime.analytics.ConfigRegistry.config import org.ostelco.prime.logger import java.io.IOException diff --git a/analytics/src/main/kotlin/org/ostelco/analytics/DataConsumptionPipeline.kt b/analytics/src/main/kotlin/org/ostelco/analytics/DataConsumptionPipeline.kt index 4e5891555..62fff2a56 100644 --- a/analytics/src/main/kotlin/org/ostelco/analytics/DataConsumptionPipeline.kt +++ b/analytics/src/main/kotlin/org/ostelco/analytics/DataConsumptionPipeline.kt @@ -26,8 +26,8 @@ import org.joda.time.Instant import org.ostelco.analytics.ParDoFn.transform import org.ostelco.analytics.Table.HOURLY_CONSUMPTION import org.ostelco.analytics.Table.RAW_CONSUMPTION -import org.ostelco.analytics.grpc.api.AggregatedDataTrafficInfo -import org.ostelco.analytics.grpc.api.DataTrafficInfo +import org.ostelco.analytics.api.AggregatedDataTrafficInfo +import org.ostelco.analytics.api.DataTrafficInfo import java.time.ZoneOffset import java.time.ZonedDateTime import java.time.format.DateTimeFormatter diff --git a/analytics/src/test/kotlin/org/ostelco/analytics/PipelineTest.kt b/analytics/src/test/kotlin/org/ostelco/analytics/PipelineTest.kt index 42ac68458..7f9787e5d 100644 --- a/analytics/src/test/kotlin/org/ostelco/analytics/PipelineTest.kt +++ b/analytics/src/test/kotlin/org/ostelco/analytics/PipelineTest.kt @@ -11,8 +11,8 @@ import org.joda.time.Instant import org.junit.Rule import org.junit.Test import org.junit.experimental.categories.Category -import org.ostelco.analytics.grpc.api.AggregatedDataTrafficInfo -import org.ostelco.analytics.grpc.api.DataTrafficInfo +import org.ostelco.analytics.api.AggregatedDataTrafficInfo +import org.ostelco.analytics.api.DataTrafficInfo import java.time.ZoneOffset import java.time.ZonedDateTime import java.time.format.DateTimeFormatter diff --git a/ocs-grpc-api/src/main/proto/ocs.proto b/ocs-grpc-api/src/main/proto/ocs.proto index 5346f068e..f4afe8537 100644 --- a/ocs-grpc-api/src/main/proto/ocs.proto +++ b/ocs-grpc-api/src/main/proto/ocs.proto @@ -1,9 +1,9 @@ syntax = "proto3"; -package org.ostelco.ocs.grpc.api; +package org.ostelco.ocs.api; option java_multiple_files = true; -option java_package = "org.ostelco.ocs.grpc.api"; +option java_package = "org.ostelco.ocs.api"; // OCS Service service OcsService { diff --git a/ocs/src/main/kotlin/org/ostelco/prime/disruptor/EventProducer.kt b/ocs/src/main/kotlin/org/ostelco/prime/disruptor/EventProducer.kt index 01e7fd8a6..8bc4c0526 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/disruptor/EventProducer.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/disruptor/EventProducer.kt @@ -1,6 +1,6 @@ package org.ostelco.prime.disruptor -import org.ostelco.ocs.grpc.api.CreditControlRequestInfo +import org.ostelco.ocs.api.CreditControlRequestInfo import org.ostelco.prime.model.Bundle interface EventProducer { diff --git a/ocs/src/main/kotlin/org/ostelco/prime/disruptor/EventProducerImpl.kt b/ocs/src/main/kotlin/org/ostelco/prime/disruptor/EventProducerImpl.kt index 1df0ebf3c..47edee8e0 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/disruptor/EventProducerImpl.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/disruptor/EventProducerImpl.kt @@ -1,8 +1,8 @@ package org.ostelco.prime.disruptor import com.lmax.disruptor.RingBuffer -import org.ostelco.ocs.grpc.api.CreditControlRequestInfo -import org.ostelco.ocs.grpc.api.ReportingReason +import org.ostelco.ocs.api.CreditControlRequestInfo +import org.ostelco.ocs.api.ReportingReason import org.ostelco.prime.disruptor.EventMessageType.ADD_MSISDN_TO_BUNDLE_MAPPING import org.ostelco.prime.disruptor.EventMessageType.CREDIT_CONTROL_REQUEST import org.ostelco.prime.disruptor.EventMessageType.RELEASE_RESERVED_BUCKET diff --git a/ocs/src/main/kotlin/org/ostelco/prime/disruptor/OcsEvent.kt b/ocs/src/main/kotlin/org/ostelco/prime/disruptor/OcsEvent.kt index 619b806df..815a76669 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/disruptor/OcsEvent.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/disruptor/OcsEvent.kt @@ -1,6 +1,6 @@ package org.ostelco.prime.disruptor -import org.ostelco.ocs.grpc.api.ReportingReason +import org.ostelco.ocs.api.ReportingReason class OcsEvent { diff --git a/ocs/src/main/kotlin/org/ostelco/prime/ocs/ActivateResponseHolder.kt b/ocs/src/main/kotlin/org/ostelco/prime/ocs/ActivateResponseHolder.kt index baa7f8dde..b0858568d 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/ocs/ActivateResponseHolder.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/ocs/ActivateResponseHolder.kt @@ -1,7 +1,7 @@ package org.ostelco.prime.ocs import io.grpc.stub.StreamObserver -import org.ostelco.ocs.grpc.api.ActivateResponse +import org.ostelco.ocs.api.ActivateResponse import java.util.concurrent.locks.Lock import java.util.concurrent.locks.ReentrantReadWriteLock diff --git a/ocs/src/main/kotlin/org/ostelco/prime/ocs/EventHandlerImpl.kt b/ocs/src/main/kotlin/org/ostelco/prime/ocs/EventHandlerImpl.kt index d5534a024..2402c9727 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/ocs/EventHandlerImpl.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/ocs/EventHandlerImpl.kt @@ -1,13 +1,13 @@ package org.ostelco.prime.ocs import com.lmax.disruptor.EventHandler -import org.ostelco.ocs.grpc.api.ActivateResponse -import org.ostelco.ocs.grpc.api.CreditControlAnswerInfo -import org.ostelco.ocs.grpc.api.FinalUnitAction -import org.ostelco.ocs.grpc.api.FinalUnitIndication -import org.ostelco.ocs.grpc.api.MultipleServiceCreditControl -import org.ostelco.ocs.grpc.api.ReportingReason -import org.ostelco.ocs.grpc.api.ServiceUnit +import org.ostelco.ocs.api.ActivateResponse +import org.ostelco.ocs.api.CreditControlAnswerInfo +import org.ostelco.ocs.api.FinalUnitAction +import org.ostelco.ocs.api.FinalUnitIndication +import org.ostelco.ocs.api.MultipleServiceCreditControl +import org.ostelco.ocs.api.ReportingReason +import org.ostelco.ocs.api.ServiceUnit import org.ostelco.prime.disruptor.EventMessageType.CREDIT_CONTROL_REQUEST import org.ostelco.prime.disruptor.EventMessageType.TOPUP_DATA_BUNDLE_BALANCE import org.ostelco.prime.disruptor.OcsEvent diff --git a/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsGrpcService.kt b/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsGrpcService.kt index 2f6cc8642..b27098722 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsGrpcService.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsGrpcService.kt @@ -1,12 +1,12 @@ package org.ostelco.prime.ocs import io.grpc.stub.StreamObserver -import org.ostelco.ocs.grpc.api.ActivateRequest -import org.ostelco.ocs.grpc.api.ActivateResponse -import org.ostelco.ocs.grpc.api.CreditControlAnswerInfo -import org.ostelco.ocs.grpc.api.CreditControlRequestInfo -import org.ostelco.ocs.grpc.api.CreditControlRequestType.NONE -import org.ostelco.ocs.grpc.api.OcsServiceGrpc +import org.ostelco.ocs.api.ActivateRequest +import org.ostelco.ocs.api.ActivateResponse +import org.ostelco.ocs.api.CreditControlAnswerInfo +import org.ostelco.ocs.api.CreditControlRequestInfo +import org.ostelco.ocs.api.CreditControlRequestType.NONE +import org.ostelco.ocs.api.OcsServiceGrpc import org.ostelco.prime.logger import java.util.* diff --git a/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsService.kt b/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsService.kt index fdec7d1fa..b5f20e01e 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsService.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsService.kt @@ -2,10 +2,10 @@ package org.ostelco.prime.ocs import com.lmax.disruptor.EventHandler import io.grpc.stub.StreamObserver -import org.ostelco.ocs.grpc.api.ActivateResponse -import org.ostelco.ocs.grpc.api.CreditControlAnswerInfo -import org.ostelco.ocs.grpc.api.CreditControlRequestInfo -import org.ostelco.ocs.grpc.api.OcsServiceGrpc +import org.ostelco.ocs.api.ActivateResponse +import org.ostelco.ocs.api.CreditControlAnswerInfo +import org.ostelco.ocs.api.CreditControlRequestInfo +import org.ostelco.ocs.api.OcsServiceGrpc import org.ostelco.prime.disruptor.EventProducer import org.ostelco.prime.disruptor.OcsEvent import java.util.concurrent.ConcurrentHashMap diff --git a/ocs/src/test/kotlin/org/ostelco/prime/disruptor/PrimeEventProducerTest.kt b/ocs/src/test/kotlin/org/ostelco/prime/disruptor/PrimeEventProducerTest.kt index 9a018915c..e410bae73 100644 --- a/ocs/src/test/kotlin/org/ostelco/prime/disruptor/PrimeEventProducerTest.kt +++ b/ocs/src/test/kotlin/org/ostelco/prime/disruptor/PrimeEventProducerTest.kt @@ -9,9 +9,9 @@ import org.junit.Assert.assertNotNull import org.junit.Assert.assertTrue import org.junit.Before import org.junit.Test -import org.ostelco.ocs.grpc.api.CreditControlRequestInfo -import org.ostelco.ocs.grpc.api.MultipleServiceCreditControl -import org.ostelco.ocs.grpc.api.ServiceUnit +import org.ostelco.ocs.api.CreditControlRequestInfo +import org.ostelco.ocs.api.MultipleServiceCreditControl +import org.ostelco.ocs.api.ServiceUnit import org.ostelco.prime.disruptor.EventMessageType.CREDIT_CONTROL_REQUEST import org.ostelco.prime.disruptor.EventMessageType.TOPUP_DATA_BUNDLE_BALANCE import java.util.concurrent.CountDownLatch diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java index e8a177b2f..7b965fb78 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java @@ -22,7 +22,7 @@ import org.ostelco.diameter.model.RedirectAddressType; import org.ostelco.diameter.model.RedirectServer; import org.ostelco.diameter.model.SessionContext; -import org.ostelco.ocs.grpc.api.*; +import org.ostelco.ocs.api.*; import org.ostelco.ocsgw.OcsServer; import org.ostelco.ocsgw.data.DataSource; import org.slf4j.Logger; @@ -331,7 +331,7 @@ private void initKeepAlive() { private void updateBlockedList(CreditControlAnswerInfo answer, CreditControlRequest request) { // This suffers from the fact that one Credit-Control-Request can have multiple MSCC - for (org.ostelco.ocs.grpc.api.MultipleServiceCreditControl msccAnswer : answer.getMsccList()) { + for (org.ostelco.ocs.api.MultipleServiceCreditControl msccAnswer : answer.getMsccList()) { for (MultipleServiceCreditControl msccRequest : request.getMultipleServiceCreditControls()) { if ((msccAnswer.getServiceIdentifier() == msccRequest.getServiceIdentifier()) && (msccAnswer.getRatingGroup() == msccRequest.getRatingGroup())) { updateBlockedList(msccAnswer, msccRequest, answer.getMsisdn()); @@ -355,7 +355,7 @@ public void handleRequest(final CreditControlContext context) { for (MultipleServiceCreditControl mscc : context.getCreditControlRequest().getMultipleServiceCreditControls()) { - org.ostelco.ocs.grpc.api.MultipleServiceCreditControl.Builder protoMscc = org.ostelco.ocs.grpc.api.MultipleServiceCreditControl.newBuilder(); + org.ostelco.ocs.api.MultipleServiceCreditControl.Builder protoMscc = org.ostelco.ocs.api.MultipleServiceCreditControl.newBuilder(); if (!mscc.getRequested().isEmpty()) { @@ -447,13 +447,13 @@ private CreditControlAnswer createCreditControlAnswer(CreditControlAnswerInfo re } final LinkedList multipleServiceCreditControls = new LinkedList<>(); - for (org.ostelco.ocs.grpc.api.MultipleServiceCreditControl mscc : response.getMsccList()) { + for (org.ostelco.ocs.api.MultipleServiceCreditControl mscc : response.getMsccList()) { multipleServiceCreditControls.add(convertMSCC(mscc)); } return new CreditControlAnswer(multipleServiceCreditControls); } - private void updateBlockedList(org.ostelco.ocs.grpc.api.MultipleServiceCreditControl msccAnswer, MultipleServiceCreditControl msccRequest, String msisdn) { + private void updateBlockedList(org.ostelco.ocs.api.MultipleServiceCreditControl msccAnswer, MultipleServiceCreditControl msccRequest, String msisdn) { if (!msccRequest.getRequested().isEmpty()) { if (msccAnswer.getGranted().getTotalOctets() < msccRequest.getRequested().get(0).getTotal()) { blocked.add(msisdn); @@ -463,7 +463,7 @@ private void updateBlockedList(org.ostelco.ocs.grpc.api.MultipleServiceCreditCon } } - private MultipleServiceCreditControl convertMSCC(org.ostelco.ocs.grpc.api.MultipleServiceCreditControl msccGRPC) { + private MultipleServiceCreditControl convertMSCC(org.ostelco.ocs.api.MultipleServiceCreditControl msccGRPC) { return new MultipleServiceCreditControl( msccGRPC.getRatingGroup(), (int) msccGRPC.getServiceIdentifier(), @@ -474,7 +474,7 @@ private MultipleServiceCreditControl convertMSCC(org.ostelco.ocs.grpc.api.Multip convertFinalUnitIndication(msccGRPC.getFinalUnitIndication())); } - private FinalUnitIndication convertFinalUnitIndication(org.ostelco.ocs.grpc.api.FinalUnitIndication fuiGrpc) { + private FinalUnitIndication convertFinalUnitIndication(org.ostelco.ocs.api.FinalUnitIndication fuiGrpc) { if (!fuiGrpc.getIsSet()) { return null; } diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwAnalytics.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwAnalytics.java index daafe2b0b..afe5172d0 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwAnalytics.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwAnalytics.java @@ -5,9 +5,9 @@ import io.grpc.StatusRuntimeException; import io.grpc.auth.MoreCallCredentials; import io.grpc.stub.StreamObserver; -import org.ostelco.analytics.grpc.api.OcsgwAnalyticsReply; -import org.ostelco.analytics.grpc.api.OcsgwAnalyticsReport; -import org.ostelco.analytics.grpc.api.OcsgwAnalyticsServiceGrpc; +import org.ostelco.prime.analytics.api.OcsgwAnalyticsReply; +import org.ostelco.prime.analytics.api.OcsgwAnalyticsReport; +import org.ostelco.prime.analytics.api.OcsgwAnalyticsServiceGrpc; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/local/LocalDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/local/LocalDataSource.java index 5c2729ab1..3b45e09f7 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/local/LocalDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/local/LocalDataSource.java @@ -13,7 +13,7 @@ import org.ostelco.diameter.model.RedirectAddressType; import org.ostelco.diameter.model.RedirectServer; import org.ostelco.diameter.model.ServiceUnit; -import org.ostelco.ocs.grpc.api.CreditControlRequestType; +import org.ostelco.ocs.api.CreditControlRequestType; import org.ostelco.ocsgw.OcsServer; import org.ostelco.ocsgw.data.DataSource; import org.slf4j.Logger; diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/proxy/ProxyDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/proxy/ProxyDataSource.java index e779bc2da..c7744df98 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/proxy/ProxyDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/proxy/ProxyDataSource.java @@ -3,7 +3,7 @@ import org.ostelco.ocsgw.data.DataSource; import org.ostelco.ocsgw.data.local.LocalDataSource; import org.ostelco.diameter.CreditControlContext; -import org.ostelco.ocs.grpc.api.CreditControlRequestType; +import org.ostelco.ocs.api.CreditControlRequestType; /** * Proxy DataSource is a combination of the Local DataSource and any other * DataSource. diff --git a/prime/src/integration-tests/kotlin/org/ostelco/prime/ocs/OcsTest.kt b/prime/src/integration-tests/kotlin/org/ostelco/prime/ocs/OcsTest.kt index 3fd3d9ebd..8627bd178 100644 --- a/prime/src/integration-tests/kotlin/org/ostelco/prime/ocs/OcsTest.kt +++ b/prime/src/integration-tests/kotlin/org/ostelco/prime/ocs/OcsTest.kt @@ -12,15 +12,15 @@ import org.junit.Assert.assertEquals import org.junit.BeforeClass import org.junit.ClassRule import org.junit.Test -import org.ostelco.ocs.grpc.api.ActivateRequest -import org.ostelco.ocs.grpc.api.ActivateResponse -import org.ostelco.ocs.grpc.api.CreditControlAnswerInfo -import org.ostelco.ocs.grpc.api.CreditControlRequestInfo -import org.ostelco.ocs.grpc.api.CreditControlRequestType.INITIAL_REQUEST -import org.ostelco.ocs.grpc.api.MultipleServiceCreditControl -import org.ostelco.ocs.grpc.api.OcsServiceGrpc -import org.ostelco.ocs.grpc.api.OcsServiceGrpc.OcsServiceStub -import org.ostelco.ocs.grpc.api.ServiceUnit +import org.ostelco.ocs.api.ActivateRequest +import org.ostelco.ocs.api.ActivateResponse +import org.ostelco.ocs.api.CreditControlAnswerInfo +import org.ostelco.ocs.api.CreditControlRequestInfo +import org.ostelco.ocs.api.CreditControlRequestType.INITIAL_REQUEST +import org.ostelco.ocs.api.MultipleServiceCreditControl +import org.ostelco.ocs.api.OcsServiceGrpc +import org.ostelco.ocs.api.OcsServiceGrpc.OcsServiceStub +import org.ostelco.ocs.api.ServiceUnit import org.ostelco.prime.disruptor.EventProducerImpl import org.ostelco.prime.disruptor.OcsDisruptor import org.ostelco.prime.logger diff --git a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/managed/MessageProcessor.kt b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/managed/MessageProcessor.kt index f9451f00a..55f6a3422 100644 --- a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/managed/MessageProcessor.kt +++ b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/managed/MessageProcessor.kt @@ -21,7 +21,7 @@ import com.google.pubsub.v1.PubsubMessage import io.dropwizard.lifecycle.Managed import io.grpc.ManagedChannel import io.grpc.ManagedChannelBuilder -import org.ostelco.analytics.grpc.api.DataTrafficInfo +import org.ostelco.analytics.api.DataTrafficInfo import org.ostelco.prime.model.PseudonymEntity import org.ostelco.pseudonym.resources.DateBounds import org.slf4j.LoggerFactory From 11497742011ae04ee32b2c75d4d993351354ed6c Mon Sep 17 00:00:00 2001 From: Martin Cederlof Date: Fri, 24 Aug 2018 14:59:39 +0200 Subject: [PATCH 14/30] Add FixMe --- .../main/java/org/ostelco/ocsgw/data/grpc/OcsgwAnalytics.java | 1 + 1 file changed, 1 insertion(+) diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwAnalytics.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwAnalytics.java index afe5172d0..8bd67490c 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwAnalytics.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwAnalytics.java @@ -26,6 +26,7 @@ public class OcsgwAnalytics { private ScheduledFuture initAnalyticsFuture = null; + // FixMe Martin : Can not reuse channel public OcsgwAnalytics(ManagedChannel channel, ServiceAccountJwtAccessCredentials credentials) { if (credentials != null) { ocsgwAnalyticsServiceStub = OcsgwAnalyticsServiceGrpc.newStub(channel) From a6dc5e9b3de9b1c18ff009f33b5f714742fc4805 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Sun, 26 Aug 2018 11:09:07 +0200 Subject: [PATCH 15/30] Made (most of the) entity classes immutable. Fixed AT. --- .../org/ostelco/prime/admin/api/Resources.kt | 2 - .../client/api/store/SubscriberDAOImpl.kt | 33 +++-- .../resources/ApplicationTokenResourceTest.kt | 8 +- .../api/resources/ProductsResourceTest.kt | 3 + .../api/resources/ProfileResourceTest.kt | 20 +--- .../api/resources/PurchasesResourceTest.kt | 11 +- .../api/resources/SubscriptionResourceTest.kt | 8 +- .../resources/SubscriptionsResourceTest.kt | 5 +- firebase-store/build.gradle | 3 +- .../prime/storage/firebase/FirebaseStorage.kt | 21 +--- model/build.gradle | 3 + .../org/ostelco/prime/model/Entities.kt | 113 ++++++++---------- .../prime/storage/graph/Neo4jModule.kt | 44 +++---- .../ostelco/prime/storage/graph/Neo4jStore.kt | 14 +-- .../prime/storage/graph/GraphStoreTest.kt | 20 ++-- prime/script/wait.sh | 39 +++++- .../prime/storage/firebase/FbStorageTest.kt | 13 +- .../prime/storage/graph/Neo4jStorageTest.kt | 6 +- .../tools/migration/FirebaseExporter.kt | 27 +++-- 19 files changed, 208 insertions(+), 185 deletions(-) diff --git a/admin-api/src/main/kotlin/org/ostelco/prime/admin/api/Resources.kt b/admin-api/src/main/kotlin/org/ostelco/prime/admin/api/Resources.kt index bafd53998..cd4e48477 100644 --- a/admin-api/src/main/kotlin/org/ostelco/prime/admin/api/Resources.kt +++ b/admin-api/src/main/kotlin/org/ostelco/prime/admin/api/Resources.kt @@ -82,8 +82,6 @@ class SegmentResource { @PathParam("segment-id") segmentId: String, segment: Segment): Response { - segment.id = segmentId - return adminDataSource.updateSegment(segment) .fold({ Response.status(Response.Status.NOT_MODIFIED).entity(it.message).build() }, { Response.ok().build() }) diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt index 7ae1ba76b..153cd552f 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt @@ -60,14 +60,13 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu logger.error("Failed to create profile. Invalid profile.") return Either.left(BadRequestError("Incomplete profile description")) } - try { - profile.referralId = profile.email - return storage.addSubscriber(profile, referredBy) + return try { + storage.addSubscriber(profile, referredBy) .mapLeft { ForbiddenError("Failed to create profile. ${it.message}") } .flatMap { getProfile(subscriberId) } } catch (e: Exception) { logger.error("Failed to create profile", e) - return Either.left(ForbiddenError("Failed to create profile")) + Either.left(ForbiddenError("Failed to create profile")) } } @@ -102,7 +101,6 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu return Either.left(BadRequestError("Incomplete profile description")) } try { - profile.referralId = profile.email storage.updateSubscriber(profile) } catch (e: Exception) { logger.error("Failed to update profile", e) @@ -156,27 +154,24 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu } override fun getMsisdn(subscriberId: String): Either { - try { - return storage.getMsisdn(subscriberId).mapLeft { + return try { + storage.getMsisdn(subscriberId).mapLeft { NotFoundError("Did not find msisdn for this subscription. ${it.message}") } } catch (e: Exception) { logger.error("Did not find msisdn for this subscription", e) - return Either.left(NotFoundError("Did not find subscription")) + Either.left(NotFoundError("Did not find subscription")) } } override fun getProducts(subscriberId: String): Either> { - try { - return storage.getProducts(subscriberId).bimap( + return try { + storage.getProducts(subscriberId).bimap( { NotFoundError(it.message) }, - { products -> - products.forEach { key, value -> value.sku = key } - products.values - }) + { products -> products.values }) } catch (e: Exception) { logger.error("Failed to get Products", e) - return Either.left(NotFoundError("Failed to get Products")) + Either.left(NotFoundError("Failed to get Products")) } } @@ -202,11 +197,11 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu // If we can't find the product, return not-found .mapLeft { NotFoundError("Product unavailable") } .flatMap { product -> - product.sku = sku val purchaseRecord = PurchaseRecord( id = UUID.randomUUID().toString(), product = product, - timestamp = Instant.now().toEpochMilli()) + timestamp = Instant.now().toEpochMilli(), + msisdn = "") // Create purchase record storage.addPurchaseRecord(subscriberId, purchaseRecord) .mapLeft { storeError -> @@ -255,11 +250,11 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu .map { chargeId -> Tuple4(profileInfo, savedSourceId, chargeId, product) } } .flatMap { (profileInfo, savedSourceId, chargeId, product) -> - product.sku = sku val purchaseRecord = PurchaseRecord( id = chargeId, product = product, - timestamp = Instant.now().toEpochMilli()) + timestamp = Instant.now().toEpochMilli(), + msisdn = "") // Create purchase record storage.addPurchaseRecord(subscriberId, purchaseRecord) .mapLeft { storeError -> diff --git a/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/ApplicationTokenResourceTest.kt b/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/ApplicationTokenResourceTest.kt index 5724d7e82..e68cfe8e6 100644 --- a/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/ApplicationTokenResourceTest.kt +++ b/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/ApplicationTokenResourceTest.kt @@ -1,10 +1,12 @@ package org.ostelco.prime.client.api.resources import arrow.core.Either +import com.fasterxml.jackson.module.kotlin.KotlinModule import com.nhaarman.mockito_kotlin.argumentCaptor import io.dropwizard.auth.AuthDynamicFeature import io.dropwizard.auth.AuthValueFactoryProvider import io.dropwizard.auth.oauth.OAuthCredentialAuthFilter +import io.dropwizard.jackson.Jackson import io.dropwizard.testing.junit.ResourceTestRule import org.assertj.core.api.Assertions.assertThat import org.glassfish.jersey.test.grizzly.GrizzlyWebTestContainerFactory @@ -38,7 +40,10 @@ class ApplicationTokenResourceTest { private val applicationID = "myAppID:4378932" private val tokenType = "FCM" - private val applicationToken = ApplicationToken() + private val applicationToken = ApplicationToken( + applicationID = applicationID, + token = token, + tokenType = tokenType) @Before @Throws(Exception::class) @@ -87,6 +92,7 @@ class ApplicationTokenResourceTest { @JvmField @ClassRule val RULE = ResourceTestRule.builder() + .setMapper(Jackson.newObjectMapper().registerModule(KotlinModule())) .addResource(AuthDynamicFeature( OAuthCredentialAuthFilter.Builder() .setAuthenticator(AUTHENTICATOR) diff --git a/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/ProductsResourceTest.kt b/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/ProductsResourceTest.kt index 57395652c..af9ca85a7 100644 --- a/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/ProductsResourceTest.kt +++ b/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/ProductsResourceTest.kt @@ -1,10 +1,12 @@ package org.ostelco.prime.client.api.resources import arrow.core.Either +import com.fasterxml.jackson.module.kotlin.KotlinModule import com.nhaarman.mockito_kotlin.argumentCaptor import io.dropwizard.auth.AuthDynamicFeature import io.dropwizard.auth.AuthValueFactoryProvider import io.dropwizard.auth.oauth.OAuthCredentialAuthFilter +import io.dropwizard.jackson.Jackson import io.dropwizard.testing.junit.ResourceTestRule import org.assertj.core.api.Assertions.assertThat import org.glassfish.jersey.test.grizzly.GrizzlyWebTestContainerFactory @@ -123,6 +125,7 @@ class ProductsResourceTest { @JvmField @ClassRule val RULE = ResourceTestRule.builder() + .setMapper(Jackson.newObjectMapper().registerModule(KotlinModule())) .addResource(AuthDynamicFeature( OAuthCredentialAuthFilter.Builder() .setAuthenticator(AUTHENTICATOR) diff --git a/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/ProfileResourceTest.kt b/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/ProfileResourceTest.kt index c88eb851f..105c62ff1 100644 --- a/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/ProfileResourceTest.kt +++ b/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/ProfileResourceTest.kt @@ -1,10 +1,12 @@ package org.ostelco.prime.client.api.resources import arrow.core.Either +import com.fasterxml.jackson.module.kotlin.KotlinModule import com.nhaarman.mockito_kotlin.argumentCaptor import io.dropwizard.auth.AuthDynamicFeature import io.dropwizard.auth.AuthValueFactoryProvider import io.dropwizard.auth.oauth.OAuthCredentialAuthFilter +import io.dropwizard.jackson.Jackson import io.dropwizard.testing.junit.ResourceTestRule import org.assertj.core.api.Assertions.assertThat import org.glassfish.jersey.test.grizzly.GrizzlyWebTestContainerFactory @@ -18,8 +20,6 @@ import org.ostelco.prime.client.api.auth.AccessTokenPrincipal import org.ostelco.prime.client.api.auth.OAuthAuthenticator import org.ostelco.prime.client.api.store.SubscriberDAO import org.ostelco.prime.client.api.util.AccessToken -import org.ostelco.prime.core.ApiError -import org.ostelco.prime.core.NotFoundError import org.ostelco.prime.model.Subscriber import java.util.* import javax.ws.rs.client.Entity @@ -38,7 +38,7 @@ class ProfileResourceTest { private val postCode = "132 23" private val city = "Oslo" - private val profile = Subscriber() + private val profile = Subscriber(email) @Before @Throws(Exception::class) @@ -62,10 +62,6 @@ class ProfileResourceTest { assertThat(resp.status).isEqualTo(Response.Status.OK.statusCode) assertThat(resp.mediaType.toString()).isEqualTo(MediaType.APPLICATION_JSON) - /* Requires that: - lombok.anyConstructor.addConstructorProperties=true - is added to the lombok config file ('lombok.config'). - Ref.: lombok changelog for ver. 1.16.20. */ assertThat(resp.readEntity(Subscriber::class.java)).isEqualTo(profile) assertThat(arg.firstValue).isEqualTo(email) } @@ -174,12 +170,6 @@ class ProfileResourceTest { @Test @Throws(Exception::class) fun updateWithIncompleteProfile() { - val arg1 = argumentCaptor() - val arg2 = argumentCaptor() - - `when`(DAO.updateProfile(arg1.capture(), arg2.capture())) - .thenReturn(Either.left(NotFoundError("No profile found"))) - val resp = RULE.target("/profile") .request(MediaType.APPLICATION_JSON) .header("Authorization", "Bearer ${AccessToken.withEmail(email)}") @@ -187,8 +177,7 @@ class ProfileResourceTest { " \"name\": \"" + name + "\"\n" + "}\n")) - assertThat(resp.status).isEqualTo(Response.Status.NOT_FOUND.statusCode) - assertThat(arg1.firstValue).isEqualTo(email) + assertThat(resp.status).isEqualTo(Response.Status.BAD_REQUEST.statusCode) } companion object { @@ -199,6 +188,7 @@ class ProfileResourceTest { @JvmField @ClassRule val RULE = ResourceTestRule.builder() + .setMapper(Jackson.newObjectMapper().registerModule(KotlinModule())) .addResource(AuthDynamicFeature( OAuthCredentialAuthFilter.Builder() .setAuthenticator(AUTHENTICATOR) diff --git a/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/PurchasesResourceTest.kt b/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/PurchasesResourceTest.kt index c30470a24..f07fdc6de 100644 --- a/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/PurchasesResourceTest.kt +++ b/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/PurchasesResourceTest.kt @@ -1,10 +1,12 @@ package org.ostelco.prime.client.api.resources import arrow.core.Either +import com.fasterxml.jackson.module.kotlin.KotlinModule import com.nhaarman.mockito_kotlin.argumentCaptor import io.dropwizard.auth.AuthDynamicFeature import io.dropwizard.auth.AuthValueFactoryProvider import io.dropwizard.auth.oauth.OAuthCredentialAuthFilter.Builder +import io.dropwizard.jackson.Jackson import io.dropwizard.testing.junit.ResourceTestRule import org.assertj.core.api.Assertions import org.glassfish.jersey.test.grizzly.GrizzlyWebTestContainerFactory @@ -15,9 +17,9 @@ import org.mockito.ArgumentMatchers import org.mockito.Mockito import org.ostelco.prime.client.api.auth.AccessTokenPrincipal import org.ostelco.prime.client.api.auth.OAuthAuthenticator -import org.ostelco.prime.core.ApiError import org.ostelco.prime.client.api.store.SubscriberDAO import org.ostelco.prime.client.api.util.AccessToken +import org.ostelco.prime.core.ApiError import org.ostelco.prime.model.Price import org.ostelco.prime.model.Product import org.ostelco.prime.model.PurchaseRecord @@ -52,7 +54,11 @@ class PurchasesResourceTest { val product = Product("1", Price(10, "NOK"), Collections.emptyMap(), Collections.emptyMap()) val now = Instant.now().toEpochMilli() - val purchaseRecord = PurchaseRecord(msisdn = "msisdn", product = product, timestamp = now) + val purchaseRecord = PurchaseRecord( + product = product, + timestamp = now, + id = UUID.randomUUID().toString(), + msisdn = "") Mockito.`when`>>(DAO.getPurchaseHistory(arg1.capture())) .thenReturn(Either.right(listOf(purchaseRecord))) @@ -74,6 +80,7 @@ class PurchasesResourceTest { @JvmField @ClassRule val RULE = ResourceTestRule.builder() + .setMapper(Jackson.newObjectMapper().registerModule(KotlinModule())) .addResource(AuthDynamicFeature( Builder() .setAuthenticator(AUTHENTICATOR) diff --git a/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/SubscriptionResourceTest.kt b/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/SubscriptionResourceTest.kt index 9192b1cf7..ad6926288 100644 --- a/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/SubscriptionResourceTest.kt +++ b/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/SubscriptionResourceTest.kt @@ -2,10 +2,12 @@ package org.ostelco.prime.client.api.resources import arrow.core.Either import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.KotlinModule import com.nhaarman.mockito_kotlin.argumentCaptor import io.dropwizard.auth.AuthDynamicFeature import io.dropwizard.auth.AuthValueFactoryProvider import io.dropwizard.auth.oauth.OAuthCredentialAuthFilter +import io.dropwizard.jackson.Jackson import io.dropwizard.testing.junit.ResourceTestRule import org.assertj.core.api.Assertions.assertThat import org.glassfish.jersey.test.grizzly.GrizzlyWebTestContainerFactory @@ -41,9 +43,10 @@ class SubscriptionResourceTest { private val purchaseRecords = listOf( PurchaseRecord( - msisdn = "msisdn", product = Product(sku = "1", price = Price(10, "NOK")), - timestamp = Instant.now().toEpochMilli())) + timestamp = Instant.now().toEpochMilli(), + id = UUID.randomUUID().toString(), + msisdn = "")) @Before fun setUp() { @@ -103,6 +106,7 @@ class SubscriptionResourceTest { @JvmField @ClassRule val RULE: ResourceTestRule = ResourceTestRule.builder() + .setMapper(Jackson.newObjectMapper().registerModule(KotlinModule())) .addResource(AuthDynamicFeature( OAuthCredentialAuthFilter.Builder() .setAuthenticator(AUTHENTICATOR) diff --git a/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/SubscriptionsResourceTest.kt b/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/SubscriptionsResourceTest.kt index 49213151e..1ba221fcb 100644 --- a/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/SubscriptionsResourceTest.kt +++ b/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/SubscriptionsResourceTest.kt @@ -1,10 +1,12 @@ package org.ostelco.prime.client.api.resources import arrow.core.Either +import com.fasterxml.jackson.module.kotlin.KotlinModule import com.nhaarman.mockito_kotlin.argumentCaptor import io.dropwizard.auth.AuthDynamicFeature import io.dropwizard.auth.AuthValueFactoryProvider import io.dropwizard.auth.oauth.OAuthCredentialAuthFilter +import io.dropwizard.jackson.Jackson import io.dropwizard.testing.junit.ResourceTestRule import org.assertj.core.api.Assertions.assertThat import org.glassfish.jersey.test.grizzly.GrizzlyWebTestContainerFactory @@ -16,9 +18,9 @@ import org.mockito.Mockito.`when` import org.mockito.Mockito.mock import org.ostelco.prime.client.api.auth.AccessTokenPrincipal import org.ostelco.prime.client.api.auth.OAuthAuthenticator -import org.ostelco.prime.core.ApiError import org.ostelco.prime.client.api.store.SubscriberDAO import org.ostelco.prime.client.api.util.AccessToken +import org.ostelco.prime.core.ApiError import org.ostelco.prime.model.Subscription import java.util.* import javax.ws.rs.client.Invocation @@ -72,6 +74,7 @@ class SubscriptionsResourceTest { @JvmField @ClassRule val RULE: ResourceTestRule = ResourceTestRule.builder() + .setMapper(Jackson.newObjectMapper().registerModule(KotlinModule())) .addResource(AuthDynamicFeature( OAuthCredentialAuthFilter.Builder() .setAuthenticator(AUTHENTICATOR) diff --git a/firebase-store/build.gradle b/firebase-store/build.gradle index 9da55192a..8d3dc3fe9 100644 --- a/firebase-store/build.gradle +++ b/firebase-store/build.gradle @@ -6,7 +6,8 @@ plugins { dependencies { implementation project(":prime-api") // Match netty via ocs-api - implementation 'com.google.firebase:firebase-admin:6.2.0' + api 'com.google.firebase:firebase-admin:6.2.0' + implementation "com.fasterxml.jackson.module:jackson-module-kotlin:$jacksonVersion" testImplementation "org.jetbrains.kotlin:kotlin-test:$kotlinVersion" testImplementation "org.jetbrains.kotlin:kotlin-test-junit:$kotlinVersion" diff --git a/firebase-store/src/main/kotlin/org/ostelco/prime/storage/firebase/FirebaseStorage.kt b/firebase-store/src/main/kotlin/org/ostelco/prime/storage/firebase/FirebaseStorage.kt index 52a3821c1..aab071b0f 100644 --- a/firebase-store/src/main/kotlin/org/ostelco/prime/storage/firebase/FirebaseStorage.kt +++ b/firebase-store/src/main/kotlin/org/ostelco/prime/storage/firebase/FirebaseStorage.kt @@ -4,16 +4,9 @@ import com.google.auth.oauth2.GoogleCredentials import com.google.firebase.FirebaseApp import com.google.firebase.FirebaseOptions import com.google.firebase.database.FirebaseDatabase -import org.ostelco.prime.logger import org.ostelco.prime.model.ApplicationToken -import org.ostelco.prime.model.Product -import org.ostelco.prime.model.PurchaseRecord -import org.ostelco.prime.model.Subscriber import org.ostelco.prime.storage.DocumentStore import java.io.FileInputStream -import java.net.URLDecoder -import java.net.URLEncoder -import java.nio.charset.StandardCharsets import java.nio.file.Files import java.nio.file.Paths @@ -25,23 +18,11 @@ class FirebaseStorage : DocumentStore by FirebaseStorageSingleton object FirebaseStorageSingleton : DocumentStore { - private val LOG by logger() - - private val balanceEntity = EntityType("balance", Long::class.java) - private val productEntity = EntityType("products", Product::class.java) - private val subscriptionEntity = EntityType("subscriptions", String::class.java) - private val subscriberEntity = EntityType("subscribers", Subscriber::class.java) - private val paymentHistoryEntity = EntityType("paymentHistory", PurchaseRecord::class.java) private val fcmTokenEntity = EntityType("notificationTokens", ApplicationToken::class.java) private val paymentIdEntity = EntityType("paymentId", String::class.java) private val firebaseDatabase = setupFirebaseInstance() - val balanceStore = EntityStore(firebaseDatabase, balanceEntity) - val productStore = EntityStore(firebaseDatabase, productEntity) - val subscriptionStore = EntityStore(firebaseDatabase, subscriptionEntity) - val subscriberStore = EntityStore(firebaseDatabase, subscriberEntity) - private val paymentHistoryStore = EntityStore(firebaseDatabase, paymentHistoryEntity) private val fcmTokenStore = EntityStore(firebaseDatabase, fcmTokenEntity) private val paymentIdStore = EntityStore(firebaseDatabase, paymentIdEntity) @@ -75,7 +56,7 @@ object FirebaseStorageSingleton : DocumentStore { } override fun getNotificationToken(msisdn: String, applicationID: String): ApplicationToken? { - return fcmTokenStore.get(applicationID) { databaseReference.child(msisdn) } + return fcmTokenStore.get(applicationID) { databaseReference.child(urlEncode(msisdn)) } } override fun getNotificationTokens(msisdn: String): Collection { diff --git a/model/build.gradle b/model/build.gradle index f634e4d5f..b465dbcef 100644 --- a/model/build.gradle +++ b/model/build.gradle @@ -6,5 +6,8 @@ plugins { dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion" implementation "com.fasterxml.jackson.core:jackson-annotations:$jacksonVersion" + + // TODO vihang: this dependency is added only for @Exclude annotation for firebase + implementation 'com.google.firebase:firebase-admin:6.2.0' implementation "org.slf4j:slf4j-api:1.7.25" } \ No newline at end of file diff --git a/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt b/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt index 09ab0898a..fd7659c2b 100644 --- a/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt +++ b/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt @@ -1,114 +1,95 @@ package org.ostelco.prime.model import com.fasterxml.jackson.annotation.JsonIgnore +import com.google.firebase.database.Exclude interface HasId { - var id: String -} - -open class Entity : HasId { - @JsonIgnore - override var id: String = "" + val id: String } data class Offer( - var segments: Collection = emptyList(), - var products: Collection = emptyList()) : Entity() + override val id: String, + @JsonIgnore val segments: Collection = emptyList(), + @JsonIgnore val products: Collection = emptyList()) : HasId data class Segment( - var subscribers: Collection = emptyList()) : Entity() + override val id: String, + @JsonIgnore val subscribers: Collection = emptyList()) : HasId data class Subscriber( - var email: String = "", - var name: String = "", - var address: String = "", - var postCode: String = "", - var city: String = "", - var country: String = "", - var referralId: String = email) : HasId { - - constructor(email: String) : this() { - this.email = email - } - - override var id: String + val email: String, + val name: String = "", + val address: String = "", + val postCode: String = "", + val city: String = "", + val country: String = "", + private val referralId: String = email) : HasId { + + constructor(email: String): this(email = email, referralId = email) + + fun getReferralId() = email + + override val id: String @JsonIgnore get() = email - @JsonIgnore - set(value) { - email = value - } } +// TODO vihang: make ApplicationToken data class immutable +// this data class is treated differently since it is stored in Firebase. data class ApplicationToken( var token: String = "", var applicationID: String = "", var tokenType: String = "") : HasId { - constructor(applicationID: String) : this() { - this.applicationID = applicationID - } - - override var id: String + override val id: String + @Exclude @JsonIgnore get() = applicationID - @JsonIgnore - set(value) { - applicationID = value - } } data class Subscription( - var msisdn: String = "") : HasId { + val msisdn: String) : HasId { - override var id: String + override val id: String @JsonIgnore get() = msisdn - @JsonIgnore - set(value) { - msisdn = value - } } data class Bundle( - override var id: String = "", - var balance: Long = 0) : HasId + override val id: String, + val balance: Long) : HasId data class Price( - var amount: Int = 0, - var currency: String = "") + val amount: Int, + val currency: String) data class Product( - var sku: String = "", - var price: Price = Price(0, ""), - var properties: Map = mapOf(), - var presentation: Map = mapOf()) : HasId { + val sku: String, + val price: Price, + val properties: Map = emptyMap(), + val presentation: Map = emptyMap()) : HasId { - override var id: String + override val id: String @JsonIgnore get() = sku - @JsonIgnore - set(value) { - sku = value - } } data class ProductClass( - override var id: String = "", - var properties: List = listOf()) : HasId + override val id: String, + val properties: List = listOf()) : HasId data class PurchaseRecord( - override var id: String = "", - @Deprecated("Will be removed in future") var msisdn: String = "", - var product: Product = Product(), - var timestamp: Long = 0L) : HasId + override val id: String, + @Deprecated("Will be removed in future") val msisdn: String, + val product: Product, + val timestamp: Long) : HasId data class PseudonymEntity( - var msisdn: String, - var pseudonym: String, - var start: Long, - var end: Long) + val msisdn: String, + val pseudonym: String, + val start: Long, + val end: Long) data class ActivePseudonyms( - var current: PseudonymEntity, - var next: PseudonymEntity) \ No newline at end of file + val current: PseudonymEntity, + val next: PseudonymEntity) \ No newline at end of file diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt index 73a3983f5..d9c3ea8ce 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt @@ -37,20 +37,27 @@ class Neo4jModule : PrimeModule { } fun initDatabase() { - Neo4jStoreSingleton.createProduct(createProduct("1GB_249NOK", 24900)) - Neo4jStoreSingleton.createProduct(createProduct("2GB_299NOK", 29900)) - Neo4jStoreSingleton.createProduct(createProduct("3GB_349NOK", 34900)) - Neo4jStoreSingleton.createProduct(createProduct("5GB_399NOK", 39900)) - - Neo4jStoreSingleton.createProduct(Product("100MB_FREE_ON_JOINING", Price(0, "NOK"), mapOf("noOfBytes" to "100_000_000"))) - Neo4jStoreSingleton.createProduct(Product("1GB_FREE_ON_REFERRED", Price(0, "NOK"), mapOf("noOfBytes" to "1_000_000_000"))) - - val segment = Segment() - segment.id = "all" + Neo4jStoreSingleton.createProduct(createProduct(sku = "1GB_249NOK", amount = 24900)) + Neo4jStoreSingleton.createProduct(createProduct(sku = "2GB_299NOK", amount = 29900)) + Neo4jStoreSingleton.createProduct(createProduct(sku = "3GB_349NOK", amount = 34900)) + Neo4jStoreSingleton.createProduct(createProduct(sku = "5GB_399NOK", amount = 39900)) + + Neo4jStoreSingleton.createProduct(Product( + sku = "100MB_FREE_ON_JOINING", + price = Price(0, "NOK"), + properties = mapOf("noOfBytes" to "100_000_000"))) + Neo4jStoreSingleton.createProduct(Product( + sku = "1GB_FREE_ON_REFERRED", + price = Price(0, "NOK"), + properties = mapOf("noOfBytes" to "1_000_000_000"))) + + val segment = Segment(id = "all") Neo4jStoreSingleton.createSegment(segment) - val offer = Offer(listOf("all"), listOf("1GB_249NOK", "2GB_299NOK", "3GB_349NOK", "5GB_399NOK")) - offer.id = "default_offer" + val offer = Offer( + id = "default_offer", + segments = listOf("all"), + products = listOf("1GB_249NOK", "2GB_299NOK", "3GB_349NOK", "5GB_399NOK")) Neo4jStoreSingleton.createOffer(offer) } @@ -86,16 +93,13 @@ object Neo4jClient : Managed { } fun createProduct(sku: String, amount: Int): Product { - val product = Product() - product.sku = sku - product.price = Price() - product.price.amount = amount - product.price.currency = "NOK" // This is messy code val gbs: Long = "${sku[0]}".toLong() - product.properties = mapOf("noOfBytes" to "${gbs}_000_000_000") - product.presentation = mapOf("label" to "$gbs GB for ${amount / 100}") - return product + return Product( + sku = sku, + price = Price(amount = amount, currency = "NOK"), + properties = mapOf("noOfBytes" to "${gbs}_000_000_000"), + presentation = mapOf("label" to "$gbs GB for ${amount / 100}")) } diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index 95f4cc0b8..273dcd4e5 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -5,7 +5,6 @@ import arrow.core.flatMap import org.neo4j.driver.v1.Transaction import org.ostelco.prime.logger import org.ostelco.prime.model.Bundle -import org.ostelco.prime.model.Entity import org.ostelco.prime.model.Offer import org.ostelco.prime.model.Product import org.ostelco.prime.model.ProductClass @@ -154,7 +153,7 @@ object Neo4jStoreSingleton : GraphStore { .flatMap { createPurchaseRecordRelation( subscriber.id, - PurchaseRecord(id = UUID.randomUUID().toString(), product = it, timestamp = Instant.now().toEpochMilli()), + PurchaseRecord(id = UUID.randomUUID().toString(), product = it, timestamp = Instant.now().toEpochMilli(), msisdn = ""), transaction) } } @@ -172,7 +171,7 @@ object Neo4jStoreSingleton : GraphStore { .flatMap { createPurchaseRecordRelation( subscriber.id, - PurchaseRecord(id = UUID.randomUUID().toString(), product = it, timestamp = Instant.now().toEpochMilli()), + PurchaseRecord(id = UUID.randomUUID().toString(), product = it, timestamp = Instant.now().toEpochMilli(), msisdn = ""), transaction) } } @@ -327,11 +326,6 @@ object Neo4jStoreSingleton : GraphStore { return subscriberStore.get(subscriberId, transaction).flatMap { subscriber -> productStore.get(purchase.product.sku, transaction).flatMap { product -> - - if (purchase.id.isBlank()) { - logger.warn("Purchase Id not set, generating a UUID") - purchase.id = UUID.randomUUID().toString() - } purchaseRecordRelationStore.create(subscriber, purchase, product, transaction) .map { purchase.id } } @@ -411,10 +405,10 @@ object Neo4jStoreSingleton : GraphStore { } } - private val offerEntity = EntityType(Entity::class.java, "Offer") + private val offerEntity = EntityType(Offer::class.java) private val offerStore = EntityStore(offerEntity) - private val segmentEntity = EntityType(Entity::class.java, "Segment") + private val segmentEntity = EntityType(Segment::class.java) private val segmentStore = EntityStore(segmentEntity) private val offerToSegmentRelation = RelationType(OFFERED_TO_SEGMENT, offerEntity, segmentEntity, Void::class.java) diff --git a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt index 0072f8b1d..f7c5d439f 100644 --- a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt +++ b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt @@ -17,6 +17,7 @@ import org.ostelco.prime.model.Subscriber import org.ostelco.prime.model.Subscription import org.ostelco.prime.ocs.OcsAdminService import java.time.Instant +import java.util.* import kotlin.test.BeforeTest import kotlin.test.Test import kotlin.test.assertEquals @@ -47,8 +48,7 @@ class GraphStoreTest { price = Price(0, "NOK"), properties = mapOf("noOfBytes" to "1_000_000_000"))) - val allSegment = Segment() - allSegment.id = "all" + val allSegment = Segment(id = "all") Neo4jStoreSingleton.createSegment(allSegment) } @@ -115,7 +115,7 @@ class GraphStoreTest { Neo4jStoreSingleton.createProduct(product) .mapLeft { fail(it.message) } - val purchaseRecord = PurchaseRecord(product = product, timestamp = now) + val purchaseRecord = PurchaseRecord(product = product, timestamp = now, id = UUID.randomUUID().toString(), msisdn = "") Neo4jStoreSingleton.addPurchaseRecord(EMAIL, purchaseRecord).bimap( { fail(it.message) }, { assertNotNull(it) } @@ -136,15 +136,15 @@ class GraphStoreTest { Neo4jStoreSingleton.createProduct(createProduct("3GB_349NOK", 34900)) Neo4jStoreSingleton.createProduct(createProduct("5GB_399NOK", 39900)) - val segment = Segment() - segment.id = "NEW_SEGMENT" - segment.subscribers = listOf(EMAIL) + val segment = Segment( + id = "NEW_SEGMENT", + subscribers = listOf(EMAIL)) Neo4jStoreSingleton.createSegment(segment) - val offer = Offer() - offer.id = "NEW_OFFER" - offer.segments = listOf("NEW_SEGMENT") - offer.products = listOf("3GB_349NOK") + val offer = Offer( + id = "NEW_OFFER", + segments = listOf("NEW_SEGMENT"), + products = listOf("3GB_349NOK")) Neo4jStoreSingleton.createOffer(offer) Neo4jStoreSingleton.getProducts(EMAIL).bimap( diff --git a/prime/script/wait.sh b/prime/script/wait.sh index 7693df266..f5450a2cb 100755 --- a/prime/script/wait.sh +++ b/prime/script/wait.sh @@ -1,13 +1,50 @@ #!/bin/bash +# exit if failed +set -e + echo "prime waiting for neo4j to launch on 7687..." while ! nc -z neo4j 7687; do - sleep 0.1 # wait for 1/10 of the second before check again + sleep 1 # wait for 1 second before check again done echo "neo4j launched" +echo "prime waiting Datastore emulator to launch on datastore-emulator:8081..." + +ds=$(curl --silent http://datastore-emulator:8081 | head -n1) +until [[ $ds == 'Ok' ]] ; do + printf 'prime waiting for Datastore emulator to launch...' + sleep 5 + ds=$(curl --silent http://datastore-emulator:8081 | head -n1) +done + +echo "Datastore emulator launched" + +echo "prime waiting pubsub emulator to launch on pubsub-emulator:8085..." + +ds=$(curl --silent http://pubsub-emulator:8085 | head -n1) +until [[ $ds == 'Ok' ]] ; do + printf 'prime waiting for pubsub emulator to launch...' + sleep 5 + ds=$(curl --silent http://pubsub-emulator:8085 | head -n1) +done + +echo "Pubsub emulator launched" + +echo "Creating topics and subscriptions...." + +curl -X PUT pubsub-emulator:8085/v1/projects/pantel-2decb/topics/data-traffic +curl -X PUT pubsub-emulator:8085/v1/projects/pantel-2decb/topics/pseudo-traffic +curl -X PUT -H "Content-Type: application/json" -d '{"topic":"projects/pantel-2decb/topics/data-traffic","ackDeadlineSeconds":10}' pubsub-emulator:8085/v1/projects/pantel-2decb/subscriptions/test-pseudo + +echo "Done creating topics and subscriptions" + +# Forward the local port 9090 to datastore-emulator:8081 +if [ -z $(type socat) ]; then echo "socat not installed."; exit 1; fi +socat TCP-LISTEN:9090,fork TCP:datastore-emulator:8081 & + # Start app exec java \ -Dfile.encoding=UTF-8 \ diff --git a/prime/src/integration-tests/kotlin/org/ostelco/prime/storage/firebase/FbStorageTest.kt b/prime/src/integration-tests/kotlin/org/ostelco/prime/storage/firebase/FbStorageTest.kt index 02a1b0384..1fbf5716c 100644 --- a/prime/src/integration-tests/kotlin/org/ostelco/prime/storage/firebase/FbStorageTest.kt +++ b/prime/src/integration-tests/kotlin/org/ostelco/prime/storage/firebase/FbStorageTest.kt @@ -29,21 +29,24 @@ class FbStorageTest { fun addApplicationNotificationTokenTest() { val token = "ThisIsTheToken" - val applicaitonId = "thisIsTheApplicationId" + val applicationId = "thisIsTheApplicationId" val tokenType = "FCM" - val applicationToken = ApplicationToken(token,applicaitonId,tokenType) + val applicationToken = ApplicationToken( + token = token, + applicationID = applicationId, + tokenType = tokenType) assertTrue(storage.addNotificationToken(MSISDN, applicationToken)) - val reply = storage.getNotificationToken(MSISDN, applicaitonId) + val reply = storage.getNotificationToken(MSISDN, applicationId) Assert.assertNotNull(reply) Assert.assertEquals(reply?.token, token) - Assert.assertEquals(reply?.applicationID, applicaitonId) + Assert.assertEquals(reply?.applicationID, applicationId) Assert.assertEquals(reply?.tokenType, tokenType) Assert.assertEquals(storage.getNotificationTokens(MSISDN).size, 1) - assertTrue(storage.removeNotificationToken(MSISDN, applicaitonId)) + assertTrue(storage.removeNotificationToken(MSISDN, applicationId)) Assert.assertEquals(storage.getNotificationTokens(MSISDN).size, 0) } diff --git a/prime/src/integration-tests/kotlin/org/ostelco/prime/storage/graph/Neo4jStorageTest.kt b/prime/src/integration-tests/kotlin/org/ostelco/prime/storage/graph/Neo4jStorageTest.kt index ff6012a73..9685a2420 100644 --- a/prime/src/integration-tests/kotlin/org/ostelco/prime/storage/graph/Neo4jStorageTest.kt +++ b/prime/src/integration-tests/kotlin/org/ostelco/prime/storage/graph/Neo4jStorageTest.kt @@ -24,6 +24,7 @@ import org.ostelco.prime.storage.firebase.initFirebaseConfigRegistry import org.ostelco.prime.storage.graph.Products.DATA_TOPUP_3GB import java.lang.Thread.sleep import java.time.Instant +import java.util.* class Neo4jStorageTest { @@ -79,9 +80,10 @@ class Neo4jStorageTest { val now = Instant.now().toEpochMilli() val purchase = PurchaseRecord( - msisdn = MSISDN, product = DATA_TOPUP_3GB, - timestamp = now) + timestamp = now, + id = UUID.randomUUID().toString(), + msisdn = "") storage.addPurchaseRecord(EPHERMERAL_EMAIL, purchase) } diff --git a/tools/neo4j-admin-tools/src/main/kotlin/org/ostelco/tools/migration/FirebaseExporter.kt b/tools/neo4j-admin-tools/src/main/kotlin/org/ostelco/tools/migration/FirebaseExporter.kt index afa32a6f4..eb2ae96f7 100644 --- a/tools/neo4j-admin-tools/src/main/kotlin/org/ostelco/tools/migration/FirebaseExporter.kt +++ b/tools/neo4j-admin-tools/src/main/kotlin/org/ostelco/tools/migration/FirebaseExporter.kt @@ -1,8 +1,11 @@ package org.ostelco.tools.migration +import com.google.firebase.database.FirebaseDatabase +import org.ostelco.prime.model.Subscriber +import org.ostelco.prime.storage.firebase.EntityStore +import org.ostelco.prime.storage.firebase.EntityType import org.ostelco.prime.storage.firebase.FirebaseConfig import org.ostelco.prime.storage.firebase.FirebaseConfigRegistry -import org.ostelco.prime.storage.firebase.FirebaseStorageSingleton fun initFirebase() { val config = FirebaseConfig() @@ -12,8 +15,19 @@ fun initFirebase() { FirebaseConfigRegistry.firebaseConfig = config } +// Code moved here from FirebaseStorageSingleton +private val balanceEntity = EntityType("balance", Long::class.java) +private val subscriptionEntity = EntityType("subscriptions", String::class.java) +private val subscriberEntity = EntityType("subscribers", Subscriber::class.java) + +// FirebaseDatabase.getInstance() will work only if FirebaseStorageSingleton.setupFirebaseInstance() is already executed. +private val balanceStore = EntityStore(FirebaseDatabase.getInstance(), balanceEntity) +private val subscriptionStore = EntityStore(FirebaseDatabase.getInstance(), subscriptionEntity) +private val subscriberStore = EntityStore(FirebaseDatabase.getInstance(), subscriberEntity) + + fun importFromFirebase(action: (String) -> Unit) { - val subscribers = FirebaseStorageSingleton.subscriberStore.getAll() + val subscribers = subscriberStore.getAll() println("// Create Subscriber") subscribers @@ -23,8 +37,7 @@ fun importFromFirebase(action: (String) -> Unit) { .forEach { action(it) } println("// Create Subscription") - FirebaseStorageSingleton - .balanceStore + balanceStore .getAll() .keys .stream() @@ -32,8 +45,7 @@ fun importFromFirebase(action: (String) -> Unit) { .forEach { action(it) } println("// Add Subscription to Subscriber") - FirebaseStorageSingleton - .subscriptionStore + subscriptionStore .getAll() .entries .stream() @@ -41,8 +53,7 @@ fun importFromFirebase(action: (String) -> Unit) { .forEach { action(it) } println("// Set balance") - FirebaseStorageSingleton - .balanceStore + balanceStore .getAll() .entries .stream() From 849c34578dc77287d7de5d1f88af4c196a8cd52b Mon Sep 17 00:00:00 2001 From: Martin Cederlof Date: Mon, 27 Aug 2018 10:21:56 +0200 Subject: [PATCH 16/30] Fix build after merge --- pseudonym-server/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pseudonym-server/build.gradle b/pseudonym-server/build.gradle index 2114e06f4..6aa518979 100644 --- a/pseudonym-server/build.gradle +++ b/pseudonym-server/build.gradle @@ -6,7 +6,7 @@ plugins { dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion" - implementation project(':ocs-api') + implementation project(':ocs-grpc-api') implementation project(':prime-api') implementation project(':model') implementation project(':analytics-grpc-api') From 96ec5e618935d368e91cccdbe10db6b6c73d93c3 Mon Sep 17 00:00:00 2001 From: Martin Cederlof Date: Mon, 27 Aug 2018 11:18:52 +0200 Subject: [PATCH 17/30] WIP adding cert for metrics.ostelco.org --- certs/metrics.ostelco.org/.gitignore | 2 + docker-compose.override.yaml | 21 ++++++ docker-compose.yaml | 2 +- docs/TEST.md | 8 ++- ocsgw/config/.gitignore | 2 +- .../java/org/ostelco/ocsgw/OcsServer.java | 4 +- .../ocsgw/data/grpc/GrpcDataSource.java | 67 +++++++------------ ...{OcsgwAnalytics.java => OcsgwMetrics.java} | 49 +++++++++++--- .../org/ostelco/ocsgw/utils/AppConfig.java | 11 +-- ocsgw/src/main/resources/config.properties | 7 +- ocsgw/src/test/resources/config.properties | 7 +- 11 files changed, 105 insertions(+), 75 deletions(-) create mode 100644 certs/metrics.ostelco.org/.gitignore rename ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/{OcsgwAnalytics.java => OcsgwMetrics.java} (56%) diff --git a/certs/metrics.ostelco.org/.gitignore b/certs/metrics.ostelco.org/.gitignore new file mode 100644 index 000000000..47c805dfe --- /dev/null +++ b/certs/metrics.ostelco.org/.gitignore @@ -0,0 +1,2 @@ +nginx.crt +nginx.key \ No newline at end of file diff --git a/docker-compose.override.yaml b/docker-compose.override.yaml index 75b71a471..0a35f4d89 100644 --- a/docker-compose.override.yaml +++ b/docker-compose.override.yaml @@ -50,6 +50,27 @@ services: ipv4_address: 172.16.238.4 default: + + metrics-esp: + container_name: metrics-esp + image: gcr.io/endpoints-release/endpoints-runtime:1 + volumes: + - "./prime/config:/esp" + - "./certs/metrics.ostelco.org:/etc/nginx/ssl" + command: > + --service=metrics.ostelco.org + --rollout_strategy=managed + --http2_port=80 + --ssl_port=443 + --backend=grpc://172.16.238.5:8082 + --service_account_key=/esp/pantel-prod.json + networks: + net: + aliases: + - "metrics.ostelco.org" + ipv4_address: 172.16.238.6 + default: + ocsgw: depends_on: - "prime" diff --git a/docker-compose.yaml b/docker-compose.yaml index 1340b62a9..d60d408eb 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -7,7 +7,7 @@ services: environment: - GOOGLE_APPLICATION_CREDENTIALS=/config/pantel-prod.json - GRPC_SERVER=ocs.ostelco.org - - GRPC_ENCRYPTION=true + - METRICS_SERVER=metrics.ostelco.org - GOOGLE_CLOUD_PROJECT=pantel-2decb auth-server: diff --git a/docs/TEST.md b/docs/TEST.md index 5f9243329..58df8eb31 100644 --- a/docs/TEST.md +++ b/docs/TEST.md @@ -20,7 +20,13 @@ grep -i pantel $(find . -name '.gitignore') | awk -F: '{print $1}' | sort | uniq ```bash cd certs/ocs.ostelco.org openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ./nginx.key -out ./nginx.crt -subj '/CN=ocs.ostelco.org' -cp nginx.crt ../../ocsgw/config +cp nginx.crt ../../ocsgw/config/ocs.crt +``` + +```bash +cd certs/metrics.ostelco.org +openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ./nginx.key -out ./nginx.crt -subj '/CN=metrics.ostelco.org' +cp nginx.crt ../../ocsgw/config/metrics.crt ``` ### Test ext-pgw -- ocsgw -- prime --firebase diff --git a/ocsgw/config/.gitignore b/ocsgw/config/.gitignore index e33eda7d4..893370e91 100644 --- a/ocsgw/config/.gitignore +++ b/ocsgw/config/.gitignore @@ -1,2 +1,2 @@ pantel-prod.json -nginx.crt \ No newline at end of file +*.crt \ No newline at end of file diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/OcsServer.java b/ocsgw/src/main/java/org/ostelco/ocsgw/OcsServer.java index 6c785f389..6a757aa29 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/OcsServer.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/OcsServer.java @@ -95,7 +95,7 @@ public void init(Stack stack, AppConfig appConfig) throws IOException { switch (appConfig.getDataStoreType()) { case DataSourceType.GRPC: LOG.info("Using GrpcDataSource"); - source = new GrpcDataSource(appConfig.getGrpcServer(), appConfig.encryptGrpc()); + source = new GrpcDataSource(appConfig.getGrpcServer(), appConfig.getMetricsServer()); break; case DataSourceType.LOCAL: LOG.info("Using LocalDataSource"); @@ -103,7 +103,7 @@ public void init(Stack stack, AppConfig appConfig) throws IOException { break; case DataSourceType.PROXY: LOG.info("Using ProxyDataSource"); - GrpcDataSource secondary = new GrpcDataSource(appConfig.getGrpcServer(), appConfig.encryptGrpc()); + GrpcDataSource secondary = new GrpcDataSource(appConfig.getGrpcServer(), appConfig.getMetricsServer()); secondary.init(); source = new ProxyDataSource(secondary); break; diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java index 7b965fb78..d41173176 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java @@ -68,7 +68,7 @@ public class GrpcDataSource implements DataSource { private StreamObserver creditControlRequest; - private OcsgwAnalytics ocsgwAnalytics; + private OcsgwMetrics ocsgwAnalytics; private ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); @@ -169,56 +169,39 @@ private void reconnectCreditControlRequest() { * Generate a new instande that connects to an endpoint, and * optionally also encrypts the connection. * - * @param target The gRPC endpoint to connect the client to. - * @param encrypted True iff transport level encryption is enabled. + * @param ocsServerHostname The gRPC endpoint to connect the client to. * @throws IOException */ - public GrpcDataSource(final String target, final boolean encrypted) throws IOException { + public GrpcDataSource(final String ocsServerHostname, final String metricsServerHostname) throws IOException { LOG.info("Created GrpcDataSource"); - LOG.info("target : {}", target); - LOG.info("encrypted : {}", encrypted); + LOG.info("ocsServerHostname : {}", ocsServerHostname); // Set up a channel to be used to communicate as an OCS instance, // to a gRPC instance. // Initialize the stub that will be used to actually // communicate from the client emulating being the OCS. - if (encrypted) { - final NettyChannelBuilder nettyChannelBuilder = NettyChannelBuilder - .forTarget(target) - .keepAliveWithoutCalls(true) - .keepAliveTimeout(KEEP_ALIVE_TIMEOUT_IN_MINUTES, TimeUnit.MINUTES) - .keepAliveTime(KEEP_ALIVE_TIME_IN_SECONDS, TimeUnit.SECONDS); - - final ManagedChannelBuilder channelBuilder = - Files.exists(Paths.get("/config/nginx.crt")) - ? nettyChannelBuilder.sslContext(GrpcSslContexts.forClient().trustManager(new File("/config/nginx.crt")).build()) - : nettyChannelBuilder; - - final String serviceAccountFile = System.getenv("GOOGLE_APPLICATION_CREDENTIALS"); - final ServiceAccountJwtAccessCredentials credentials = - ServiceAccountJwtAccessCredentials.fromStream(new FileInputStream(serviceAccountFile)); - final ManagedChannel channel = channelBuilder - .useTransportSecurity() - .build(); - ocsServiceStub = OcsServiceGrpc.newStub(channel) - .withCallCredentials(MoreCallCredentials.from(credentials)); - - ocsgwAnalytics = new OcsgwAnalytics(channel, credentials); - } else { - final ManagedChannelBuilder channelBuilder = ManagedChannelBuilder - .forTarget(target) - .keepAliveWithoutCalls(true) - .keepAliveTimeout(KEEP_ALIVE_TIMEOUT_IN_MINUTES, TimeUnit.MINUTES) - .keepAliveTime(KEEP_ALIVE_TIME_IN_SECONDS, TimeUnit.SECONDS); - - final ManagedChannel channel = channelBuilder - .usePlaintext() - .build(); - ocsServiceStub = OcsServiceGrpc.newStub(channel); - - ocsgwAnalytics = new OcsgwAnalytics(channel, null); - } + final NettyChannelBuilder nettyChannelBuilder = NettyChannelBuilder + .forTarget(ocsServerHostname) + .keepAliveWithoutCalls(true) + .keepAliveTimeout(KEEP_ALIVE_TIMEOUT_IN_MINUTES, TimeUnit.MINUTES) + .keepAliveTime(KEEP_ALIVE_TIME_IN_SECONDS, TimeUnit.SECONDS); + + final ManagedChannelBuilder channelBuilder = + Files.exists(Paths.get("/config/ocs.crt")) + ? nettyChannelBuilder.sslContext(GrpcSslContexts.forClient().trustManager(new File("/config/ocs.crt")).build()) + : nettyChannelBuilder; + + final String serviceAccountFile = System.getenv("GOOGLE_APPLICATION_CREDENTIALS"); + final ServiceAccountJwtAccessCredentials credentials = + ServiceAccountJwtAccessCredentials.fromStream(new FileInputStream(serviceAccountFile)); + final ManagedChannel channel = channelBuilder + .useTransportSecurity() + .build(); + ocsServiceStub = OcsServiceGrpc.newStub(channel) + .withCallCredentials(MoreCallCredentials.from(credentials)); + + ocsgwAnalytics = new OcsgwMetrics(metricsServerHostname, credentials); } @Override diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwAnalytics.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java similarity index 56% rename from ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwAnalytics.java rename to ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java index 8bd67490c..e156eced3 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwAnalytics.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java @@ -2,8 +2,11 @@ import com.google.auth.oauth2.ServiceAccountJwtAccessCredentials; import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; import io.grpc.StatusRuntimeException; import io.grpc.auth.MoreCallCredentials; +import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; import io.grpc.stub.StreamObserver; import org.ostelco.prime.analytics.api.OcsgwAnalyticsReply; import org.ostelco.prime.analytics.api.OcsgwAnalyticsReport; @@ -11,13 +14,21 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.net.ssl.SSLException; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.concurrent.*; -public class OcsgwAnalytics { +public class OcsgwMetrics { - private static final Logger LOG = LoggerFactory.getLogger(OcsgwAnalytics.class); + private static final Logger LOG = LoggerFactory.getLogger(OcsgwMetrics.class); - private final OcsgwAnalyticsServiceGrpc.OcsgwAnalyticsServiceStub ocsgwAnalyticsServiceStub; + private static final int KEEP_ALIVE_TIMEOUT_IN_MINUTES = 1; + + private static final int KEEP_ALIVE_TIME_IN_SECONDS = 50; + + private OcsgwAnalyticsServiceGrpc.OcsgwAnalyticsServiceStub ocsgwAnalyticsServiceStub; private ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); @@ -25,14 +36,30 @@ public class OcsgwAnalytics { private ScheduledFuture initAnalyticsFuture = null; - - // FixMe Martin : Can not reuse channel - public OcsgwAnalytics(ManagedChannel channel, ServiceAccountJwtAccessCredentials credentials) { - if (credentials != null) { - ocsgwAnalyticsServiceStub = OcsgwAnalyticsServiceGrpc.newStub(channel) - .withCallCredentials(MoreCallCredentials.from(credentials)); - } else { - ocsgwAnalyticsServiceStub = OcsgwAnalyticsServiceGrpc.newStub(channel); + public OcsgwMetrics(String metricsServerHostname, ServiceAccountJwtAccessCredentials credentials) { + + try { + final NettyChannelBuilder nettyChannelBuilder = NettyChannelBuilder + .forTarget(metricsServerHostname) + .keepAliveWithoutCalls(true) + .keepAliveTimeout(KEEP_ALIVE_TIMEOUT_IN_MINUTES, TimeUnit.MINUTES) + .keepAliveTime(KEEP_ALIVE_TIME_IN_SECONDS, TimeUnit.SECONDS); + + final ManagedChannelBuilder channelBuilder = Files.exists(Paths.get("/config/metrics.crt")) + ? nettyChannelBuilder.sslContext(GrpcSslContexts.forClient().trustManager(new File("/config/metrics.crt")).build()) + : nettyChannelBuilder; + + final ManagedChannel channel = channelBuilder + .useTransportSecurity() + .build(); + if (credentials != null) { + ocsgwAnalyticsServiceStub = OcsgwAnalyticsServiceGrpc.newStub(channel) + .withCallCredentials(MoreCallCredentials.from(credentials)); + } else { + ocsgwAnalyticsServiceStub = OcsgwAnalyticsServiceGrpc.newStub(channel); + } + } catch (SSLException e) { + LOG.warn("Failed to setup OcsMetrics", e); } } diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/utils/AppConfig.java b/ocsgw/src/main/java/org/ostelco/ocsgw/utils/AppConfig.java index ca1760137..c534dbe5f 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/utils/AppConfig.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/utils/AppConfig.java @@ -33,11 +33,12 @@ public String getGrpcServer() { return grpcServer; } - public boolean encryptGrpc() { - String encrypt = System.getenv("GRPC_ENCRYPTION"); - if (encrypt == null || encrypt.isEmpty()) { - encrypt = prop.getProperty("GrpcEncryption", "true"); + public String getMetricsServer() { + // GRPC_SERVER env has higher preference over config.properties + final String metricsServer = System.getenv("METRICS_SERVER"); + if (metricsServer == null || metricsServer.isEmpty()) { + LOG.warn("No metric server set in env"); } - return Boolean.valueOf(encrypt); + return metricsServer; } } diff --git a/ocsgw/src/main/resources/config.properties b/ocsgw/src/main/resources/config.properties index eb12db9ab..193710c77 100644 --- a/ocsgw/src/main/resources/config.properties +++ b/ocsgw/src/main/resources/config.properties @@ -1,7 +1,2 @@ # Type of data storage (Local | gRPC | Proxy) -DataStoreType=Proxy - -# The ip:port for the gRPC server that the ocsgw will connect to -GrpcServer=172.16.238.5:8082 - -GrpcEncryption=false \ No newline at end of file +DataStoreType=Proxy \ No newline at end of file diff --git a/ocsgw/src/test/resources/config.properties b/ocsgw/src/test/resources/config.properties index 97655d620..2718e9b61 100644 --- a/ocsgw/src/test/resources/config.properties +++ b/ocsgw/src/test/resources/config.properties @@ -1,7 +1,2 @@ # Type of data storage (Local | gRPC | Proxy) -DataStoreType=Local - -# The ip:port on which the gRPC service will be receiving incoming connections. -GrpcServer=172.16.238.5:8082 - -GrpcEncryption=false \ No newline at end of file +DataStoreType=Local \ No newline at end of file From 26e3d2dac72332402fa0c2a677fa97beee0e1bfb Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Mon, 27 Aug 2018 13:27:44 +0200 Subject: [PATCH 18/30] Add bigquery parameter to the init() --- .../service/PseudonymizerServiceSingleton.kt | 12 ++++++++---- .../org/ostelco/pseudonym/PseudonymResourceTest.kt | 4 +++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/service/PseudonymizerServiceSingleton.kt b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/service/PseudonymizerServiceSingleton.kt index b8eeb24d6..6533c1b16 100644 --- a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/service/PseudonymizerServiceSingleton.kt +++ b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/service/PseudonymizerServiceSingleton.kt @@ -67,12 +67,16 @@ object PseudonymizerServiceSingleton : PseudonymizerService { private val executor = Executors.newFixedThreadPool(3) - fun init() { + fun init(bq: BigQuery? = null) { datastore = getDatastore(ConfigRegistry.config) - if(System.getenv("LOCAL_TESTING") != "true") { - bigquery = BigQueryOptions.getDefaultInstance().service + if (bq != null) { + bigquery = bq } else { - logger.info("Local testing, BigQuery is not available...") + if (System.getenv("LOCAL_TESTING") != "true") { + bigquery = BigQueryOptions.getDefaultInstance().service + } else { + logger.info("Local testing, BigQuery is not available...") + } } } diff --git a/pseudonym-server/src/test/kotlin/org/ostelco/pseudonym/PseudonymResourceTest.kt b/pseudonym-server/src/test/kotlin/org/ostelco/pseudonym/PseudonymResourceTest.kt index cd8d5e973..46e8c68d0 100644 --- a/pseudonym-server/src/test/kotlin/org/ostelco/pseudonym/PseudonymResourceTest.kt +++ b/pseudonym-server/src/test/kotlin/org/ostelco/pseudonym/PseudonymResourceTest.kt @@ -2,9 +2,11 @@ package org.ostelco.pseudonym import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue +import com.google.cloud.bigquery.BigQuery import io.dropwizard.testing.junit.ResourceTestRule import org.junit.ClassRule import org.junit.Test +import org.mockito.Mockito.mock import org.ostelco.prime.model.ActivePseudonyms import org.ostelco.prime.model.PseudonymEntity import org.ostelco.pseudonym.resources.PseudonymResource @@ -32,7 +34,7 @@ class PseudonymResourceTest { init { ConfigRegistry.config = PseudonymServerConfig() .apply { this.datastoreType = "inmemory-emulator" } - PseudonymizerServiceSingleton.init() + PseudonymizerServiceSingleton.init(mock(BigQuery::class.java)) } @ClassRule From 3bdab19a2180bf2f9404bec49f863df9395b3512 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Mon, 27 Aug 2018 14:01:55 +0200 Subject: [PATCH 19/30] Added endpoint for metrics --- .gitignore | 3 +- .../src/main/proto/analytics.proto | 2 +- ...me_analytics.proto => prime_metrics.proto} | 7 +--- .../.gitignore | 0 docs/TEST.md | 4 +- .../StripePaymentProcessorTest.kt | 42 ++++++++++--------- prime/infra/README.md | 22 ++++++---- prime/infra/dev/metrics-api.yaml | 30 +++++++++++++ 8 files changed, 75 insertions(+), 35 deletions(-) rename analytics-grpc-api/src/main/proto/{prime_analytics.proto => prime_metrics.proto} (65%) rename certs/{metrics.ostelco.org => metrics.dev.endpoints.pantel-2decb.cloud.goog}/.gitignore (100%) create mode 100644 prime/infra/dev/metrics-api.yaml diff --git a/.gitignore b/.gitignore index efb094863..538ab953a 100644 --- a/.gitignore +++ b/.gitignore @@ -35,4 +35,5 @@ secrets/* .nb-gradle .swagger_gen_dir -api_descriptor.pb \ No newline at end of file +ocs_descriptor.pb +metrics_descriptor.pb \ No newline at end of file diff --git a/analytics-grpc-api/src/main/proto/analytics.proto b/analytics-grpc-api/src/main/proto/analytics.proto index f437c517c..6ba8085b9 100644 --- a/analytics-grpc-api/src/main/proto/analytics.proto +++ b/analytics-grpc-api/src/main/proto/analytics.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package org.ostelco.analytics.grpc.api; +package org.ostelco.analytics.api; option java_multiple_files = true; option java_package = "org.ostelco.analytics.api"; diff --git a/analytics-grpc-api/src/main/proto/prime_analytics.proto b/analytics-grpc-api/src/main/proto/prime_metrics.proto similarity index 65% rename from analytics-grpc-api/src/main/proto/prime_analytics.proto rename to analytics-grpc-api/src/main/proto/prime_metrics.proto index 327c1d62f..70dbf355a 100644 --- a/analytics-grpc-api/src/main/proto/prime_analytics.proto +++ b/analytics-grpc-api/src/main/proto/prime_metrics.proto @@ -1,11 +1,9 @@ syntax = "proto3"; -package org.ostelco.analytics.api; +package org.ostelco.prime.metrics.api; option java_multiple_files = true; -option java_package = "org.ostelco.prime.analytics.api"; - -import "google/protobuf/timestamp.proto"; +option java_package = "org.ostelco.prime.metrics.api"; // This is used to report Analytics events from OCSgw to Prime @@ -15,7 +13,6 @@ service OcsgwAnalyticsService { message OcsgwAnalyticsReport { uint32 activeSessions = 1; - google.protobuf.Timestamp timestamp = 2; } message OcsgwAnalyticsReply { diff --git a/certs/metrics.ostelco.org/.gitignore b/certs/metrics.dev.endpoints.pantel-2decb.cloud.goog/.gitignore similarity index 100% rename from certs/metrics.ostelco.org/.gitignore rename to certs/metrics.dev.endpoints.pantel-2decb.cloud.goog/.gitignore diff --git a/docs/TEST.md b/docs/TEST.md index 58df8eb31..1e62c0cb1 100644 --- a/docs/TEST.md +++ b/docs/TEST.md @@ -24,8 +24,8 @@ cp nginx.crt ../../ocsgw/config/ocs.crt ``` ```bash -cd certs/metrics.ostelco.org -openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ./nginx.key -out ./nginx.crt -subj '/CN=metrics.ostelco.org' +cd certs/metrics.dev.endpoints.pantel-2decb.cloud.goog +openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ./nginx.key -out ./nginx.crt -subj '/CN=metrics.dev.endpoints.pantel-2decb.cloud.goog' cp nginx.crt ../../ocsgw/config/metrics.crt ``` diff --git a/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt b/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt index a37997e23..074efa20b 100644 --- a/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt +++ b/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt @@ -7,6 +7,7 @@ import org.junit.Before import org.junit.Test import org.ostelco.prime.module.getResource import kotlin.test.assertEquals +import kotlin.test.fail class StripePaymentProcessorTest { @@ -33,7 +34,7 @@ class StripePaymentProcessorTest { val resultAdd = paymentProcessor.createPaymentProfile(testCustomer) assertEquals(true, resultAdd.isRight()) - stripeCustomerId = resultAdd.fold({""}, {it.id}) + stripeCustomerId = resultAdd.fold({ "" }, { it.id }) } @Before @@ -58,14 +59,17 @@ class StripePaymentProcessorTest { fun addSourceToCustomerAndRemove() { val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) - assertEquals(true, resultAddSource.isRight()) val resultStoredSources = paymentProcessor.getSavedSources(stripeCustomerId) - assertEquals(true, resultStoredSources.isRight()) - assertEquals(1, resultStoredSources.fold({0},{it.size})) - assertEquals(resultAddSource.get().id, resultStoredSources.get().first().id) + assertEquals(1, resultStoredSources.fold({ 0 }, { it.size })) + + resultAddSource.map { addedSource -> + resultStoredSources.map { storedSources -> + assertEquals(addedSource.id, storedSources.first().id) + }.mapLeft { fail() } + }.mapLeft { fail() } - val resultDeleteSource = paymentProcessor.removeSource(stripeCustomerId ,resultAddSource.fold({""}, {it.id})) + val resultDeleteSource = paymentProcessor.removeSource(stripeCustomerId, resultAddSource.fold({ "" }, { it.id })) assertEquals(true, resultDeleteSource.isRight()) } @@ -75,14 +79,14 @@ class StripePaymentProcessorTest { val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) assertEquals(true, resultAddSource.isRight()) - val resultAddDefault = paymentProcessor.setDefaultSource(stripeCustomerId, resultAddSource.fold({""}, {it.id})) + val resultAddDefault = paymentProcessor.setDefaultSource(stripeCustomerId, resultAddSource.fold({ "" }, { it.id })) assertEquals(true, resultAddDefault.isRight()) val resultGetDefault = paymentProcessor.getDefaultSource(stripeCustomerId) assertEquals(true, resultGetDefault.isRight()) - assertEquals(resultAddDefault.fold({""}, {it.id}), resultGetDefault.fold({""}, {it.id})) + assertEquals(resultAddDefault.fold({ "" }, { it.id }), resultGetDefault.fold({ "" }, { it.id })) - val resultRemoveDefault = paymentProcessor.removeSource(stripeCustomerId, resultAddDefault.fold({""}, {it.id})) + val resultRemoveDefault = paymentProcessor.removeSource(stripeCustomerId, resultAddDefault.fold({ "" }, { it.id })) assertEquals(true, resultRemoveDefault.isRight()) } @@ -91,7 +95,7 @@ class StripePaymentProcessorTest { val resultCreateProduct = paymentProcessor.createProduct("TestSku") assertEquals(true, resultCreateProduct.isRight()) - val resultRemoveProduct = paymentProcessor.removeProduct(resultCreateProduct.fold({""}, {it.id})) + val resultRemoveProduct = paymentProcessor.removeProduct(resultCreateProduct.fold({ "" }, { it.id })) assertEquals(true, resultRemoveProduct.isRight()) } @@ -105,25 +109,25 @@ class StripePaymentProcessorTest { val resultCreateProduct = paymentProcessor.createProduct("TestSku") assertEquals(true, resultCreateProduct.isRight()) - val resultCreatePlan = paymentProcessor.createPlan(resultCreateProduct.fold({""}, {it.id}), 1000, "NOK", PaymentProcessor.Interval.MONTH) + val resultCreatePlan = paymentProcessor.createPlan(resultCreateProduct.fold({ "" }, { it.id }), 1000, "NOK", PaymentProcessor.Interval.MONTH) assertEquals(true, resultCreatePlan.isRight()) - val resultSubscribePlan = paymentProcessor.subscribeToPlan(resultCreatePlan.fold({""}, {it.id}), stripeCustomerId) + val resultSubscribePlan = paymentProcessor.subscribeToPlan(resultCreatePlan.fold({ "" }, { it.id }), stripeCustomerId) assertEquals(true, resultSubscribePlan.isRight()) - val resultUnsubscribePlan = paymentProcessor.cancelSubscription(resultSubscribePlan.fold({""}, {it.id}), false) + val resultUnsubscribePlan = paymentProcessor.cancelSubscription(resultSubscribePlan.fold({ "" }, { it.id }), false) assertEquals(true, resultUnsubscribePlan.isRight()) - assertEquals(resultSubscribePlan.fold({""}, {it.id}), resultUnsubscribePlan.fold({""}, {it.id})) + assertEquals(resultSubscribePlan.fold({ "" }, { it.id }), resultUnsubscribePlan.fold({ "" }, { it.id })) - val resultDeletePlan = paymentProcessor.removePlan(resultCreatePlan.fold({""}, {it.id})) + val resultDeletePlan = paymentProcessor.removePlan(resultCreatePlan.fold({ "" }, { it.id })) assertEquals(true, resultDeletePlan.isRight()) - assertEquals(resultCreatePlan.fold({""}, {it.id}), resultDeletePlan.fold({""}, {it.id})) + assertEquals(resultCreatePlan.fold({ "" }, { it.id }), resultDeletePlan.fold({ "" }, { it.id })) - val resultRemoveProduct = paymentProcessor.removeProduct(resultCreateProduct.fold({""}, {it.id})) + val resultRemoveProduct = paymentProcessor.removeProduct(resultCreateProduct.fold({ "" }, { it.id })) assertEquals(true, resultRemoveProduct.isRight()) - assertEquals(resultCreateProduct.fold({""}, {it.id}), resultRemoveProduct.fold({""}, {it.id})) + assertEquals(resultCreateProduct.fold({ "" }, { it.id }), resultRemoveProduct.fold({ "" }, { it.id })) - val resultDeleteSource = paymentProcessor.removeSource(stripeCustomerId ,resultAddSource.fold({""}, {it.id})) + val resultDeleteSource = paymentProcessor.removeSource(stripeCustomerId, resultAddSource.fold({ "" }, { it.id })) assertEquals(true, resultDeleteSource.isRight()) } } \ No newline at end of file diff --git a/prime/infra/README.md b/prime/infra/README.md index eb6c136ff..2605930f3 100644 --- a/prime/infra/README.md +++ b/prime/infra/README.md @@ -80,7 +80,7 @@ Reference: ## Endpoint -Generate self-contained protobuf descriptor file - api_descriptor.pb +Generate self-contained protobuf descriptor file - `ocs_descriptor.pb` & `metrics_descriptor.pb` ```bash pip install grpcio grpcio-tools @@ -88,15 +88,23 @@ pip install grpcio grpcio-tools python -m grpc_tools.protoc \ --include_imports \ --include_source_info \ - --proto_path=ocs-api/src/main/proto \ - --descriptor_set_out=api_descriptor.pb \ + --proto_path=ocs-grpc-api/src/main/proto \ + --descriptor_set_out=ocs_descriptor.pb \ ocs.proto + +python -m grpc_tools.protoc \ + --include_imports \ + --include_source_info \ + --proto_path=analytics-grpc-api/src/main/proto \ + --descriptor_set_out=metrics_descriptor.pb \ + prime_metrics.proto ``` Deploy endpoints ```bash -gcloud endpoints services deploy api_descriptor.pb prime/infra/prod/ocs-api.yaml +gcloud endpoints services deploy ocs_descriptor.pb prime/infra/prod/ocs-api.yaml +gcloud endpoints services deploy metrics_descriptor.pb prime/infra/prod/metrics-api.yaml ``` ## Deployment & Service @@ -212,7 +220,7 @@ kubectl create secret generic api-ostelco-ssl \ * OCS gRPC endpoint -Generate self-contained protobuf descriptor file - api_descriptor.pb +Generate self-contained protobuf descriptor file - ocs_descriptor.pb ```bash pip install grpcio grpcio-tools @@ -221,14 +229,14 @@ python -m grpc_tools.protoc \ --include_imports \ --include_source_info \ --proto_path=ocs-api/src/main/proto \ - --descriptor_set_out=api_descriptor.pb \ + --descriptor_set_out=ocs_descriptor.pb \ ocs.proto ``` Deploy endpoints ```bash -gcloud endpoints services deploy api_descriptor.pb prime/infra/dev/ocs-api.yaml +gcloud endpoints services deploy ocs_descriptor.pb prime/infra/dev/ocs-api.yaml ``` * Client API HTTP endpoint diff --git a/prime/infra/dev/metrics-api.yaml b/prime/infra/dev/metrics-api.yaml new file mode 100644 index 000000000..85a47600f --- /dev/null +++ b/prime/infra/dev/metrics-api.yaml @@ -0,0 +1,30 @@ +type: google.api.Service + +config_version: 3 + +name: metrics.dev.endpoints.pantel-2decb.cloud.goog + +title: Prime Metrics Reporter Service gRPC API + +apis: + - name: org.ostelco.prime.metrics.api.OcsgwAnalyticsService + +usage: + rules: + # All methods can be called without an API Key. + - selector: "*" + allow_unregistered_calls: true + +authentication: + providers: + - id: google_service_account + issuer: prime-service-account@pantel-2decb.iam.gserviceaccount.com + jwks_uri: https://www.googleapis.com/robot/v1/metadata/x509/prime-service-account@pantel-2decb.iam.gserviceaccount.com + audiences: > + https://metrics.dev.endpoints.pantel-2decb.cloud.goog/org.ostelco.prime.metrics.api.OcsgwAnalyticsService, + metrics.dev.endpoints.pantel-2decb.cloud.goog/org.ostelco.prime.metrics.api.OcsgwAnalyticsService, + metrics.dev.endpoints.pantel-2decb.cloud.goog + rules: + - selector: "*" + requirements: + - provider_id: google_service_account From 1a168e14f984f21d0fa9e512044966a30f57202b Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Mon, 27 Aug 2018 14:10:09 +0200 Subject: [PATCH 20/30] Updated metrics uri in docker-compose files --- docker-compose.override.yaml | 6 +++--- docker-compose.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.override.yaml b/docker-compose.override.yaml index 0a35f4d89..af37e454c 100644 --- a/docker-compose.override.yaml +++ b/docker-compose.override.yaml @@ -56,9 +56,9 @@ services: image: gcr.io/endpoints-release/endpoints-runtime:1 volumes: - "./prime/config:/esp" - - "./certs/metrics.ostelco.org:/etc/nginx/ssl" + - "./certs/metrics.dev.endpoints.pantel-2decb.cloud.goog:/etc/nginx/ssl" command: > - --service=metrics.ostelco.org + --service=metrics.dev.endpoints.pantel-2decb.cloud.goog --rollout_strategy=managed --http2_port=80 --ssl_port=443 @@ -67,7 +67,7 @@ services: networks: net: aliases: - - "metrics.ostelco.org" + - "metrics.dev.endpoints.pantel-2decb.cloud.goog" ipv4_address: 172.16.238.6 default: diff --git a/docker-compose.yaml b/docker-compose.yaml index d60d408eb..fb44b676f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -7,7 +7,7 @@ services: environment: - GOOGLE_APPLICATION_CREDENTIALS=/config/pantel-prod.json - GRPC_SERVER=ocs.ostelco.org - - METRICS_SERVER=metrics.ostelco.org + - METRICS_SERVER=metrics.dev.endpoints.pantel-2decb.cloud.goog - GOOGLE_CLOUD_PROJECT=pantel-2decb auth-server: From a083a223665f5c0f0d942db2b40d2c538c930140 Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Mon, 27 Aug 2018 14:11:32 +0200 Subject: [PATCH 21/30] Use a fixed version of google-cloud-sdk docker image In the latest version of the docker image, there is a JAVA version mismatch which is preventing us from running datastore emulator. So for the time being lets use a tested one. --- docker-compose.override.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.override.yaml b/docker-compose.override.yaml index 75b71a471..b6f555dbf 100644 --- a/docker-compose.override.yaml +++ b/docker-compose.override.yaml @@ -91,7 +91,7 @@ services: datastore-emulator: container_name: datastore-emulator - image: google/cloud-sdk + image: google/cloud-sdk:206.0.0 expose: - "8081" environment: From 95b334bb695412d5cba965a98f0da2699c0ca4b8 Mon Sep 17 00:00:00 2001 From: Martin Cederlof Date: Mon, 27 Aug 2018 14:45:29 +0200 Subject: [PATCH 22/30] Use different port for grpc metrics --- docker-compose.override.yaml | 2 +- .../main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.override.yaml b/docker-compose.override.yaml index af37e454c..3ea22b5b4 100644 --- a/docker-compose.override.yaml +++ b/docker-compose.override.yaml @@ -62,7 +62,7 @@ services: --rollout_strategy=managed --http2_port=80 --ssl_port=443 - --backend=grpc://172.16.238.5:8082 + --backend=grpc://172.16.238.5:8083 --service_account_key=/esp/pantel-prod.json networks: net: diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java index d41173176..361021785 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java @@ -176,6 +176,7 @@ public GrpcDataSource(final String ocsServerHostname, final String metricsServer LOG.info("Created GrpcDataSource"); LOG.info("ocsServerHostname : {}", ocsServerHostname); + LOG.info("metricsServerHostname : {}", metricsServerHostname); // Set up a channel to be used to communicate as an OCS instance, // to a gRPC instance. From cd4b2a62ec8e1d7b0e0743e55b6934a541bbb6e7 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Mon, 27 Aug 2018 14:53:24 +0200 Subject: [PATCH 23/30] Working AT. --- .../ostelco/prime/analytics/AnalyticsGrpcService.kt | 8 ++++---- .../ostelco/prime/analytics/metrics/OcsgwMetrics.kt | 5 +++-- .../org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java | 12 ++++++++---- prime/Dockerfile | 1 + 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt index bd70847ff..a564dbdb1 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt @@ -1,11 +1,11 @@ package org.ostelco.prime.analytics import io.grpc.stub.StreamObserver -import org.ostelco.prime.analytics.api.OcsgwAnalyticsReply -import org.ostelco.prime.analytics.api.OcsgwAnalyticsReport -import org.ostelco.prime.analytics.api.OcsgwAnalyticsServiceGrpc import org.ostelco.prime.analytics.metrics.OcsgwMetrics import org.ostelco.prime.logger +import org.ostelco.prime.metrics.api.OcsgwAnalyticsReply +import org.ostelco.prime.metrics.api.OcsgwAnalyticsReport +import org.ostelco.prime.metrics.api.OcsgwAnalyticsServiceGrpc import java.util.* @@ -50,7 +50,7 @@ class AnalyticsGrpcService(private val metrics : OcsgwMetrics) : OcsgwAnalyticsS * @param request provides current active session as a counter with a timestamp */ override fun onNext(request: OcsgwAnalyticsReport) { - metrics.setActiveSessions(request.timestamp.seconds, request.activeSessions.toLong()) + metrics.setActiveSessions(request.activeSessions.toLong()) } override fun onError(t: Throwable) { diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt index bb0a317d3..52983f9ea 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt @@ -4,6 +4,7 @@ import com.codahale.metrics.Gauge import com.codahale.metrics.MetricRegistry import com.codahale.metrics.MetricRegistry.name import org.ostelco.prime.analytics.AnalyticsGrpcService +import java.time.Instant class OcsgwMetrics(private val registry: MetricRegistry) { @@ -27,8 +28,8 @@ class OcsgwMetrics(private val registry: MetricRegistry) { * @param count - active sessions */ @Synchronized - fun setActiveSessions(timestamp: Long, count: Long) { - currentActiveSessions.timestamp = timestamp + fun setActiveSessions(count: Long) { + currentActiveSessions.timestamp = Instant.now().toEpochMilli() currentActiveSessions.count = count } } diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java index e156eced3..c96f41b13 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java @@ -8,9 +8,9 @@ import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; import io.grpc.stub.StreamObserver; -import org.ostelco.prime.analytics.api.OcsgwAnalyticsReply; -import org.ostelco.prime.analytics.api.OcsgwAnalyticsReport; -import org.ostelco.prime.analytics.api.OcsgwAnalyticsServiceGrpc; +import org.ostelco.prime.metrics.api.OcsgwAnalyticsReply; +import org.ostelco.prime.metrics.api.OcsgwAnalyticsReport; +import org.ostelco.prime.metrics.api.OcsgwAnalyticsServiceGrpc; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,7 +18,11 @@ import java.io.File; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.concurrent.*; +import java.util.concurrent.Callable; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; public class OcsgwMetrics { diff --git a/prime/Dockerfile b/prime/Dockerfile index d38b20a3d..20573fb4f 100644 --- a/prime/Dockerfile +++ b/prime/Dockerfile @@ -12,5 +12,6 @@ EXPOSE 7687 EXPOSE 8080 EXPOSE 8081 EXPOSE 8082 +EXPOSE 8083 CMD ["/start.sh"] \ No newline at end of file From 376bcefa4be3b075c1c3297438a05c5e5417d998 Mon Sep 17 00:00:00 2001 From: Martin Cederlof Date: Mon, 27 Aug 2018 15:20:28 +0200 Subject: [PATCH 24/30] Resend last activeSessions on reconnect --- .../main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java index c96f41b13..6facc6699 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java @@ -40,6 +40,8 @@ public class OcsgwMetrics { private ScheduledFuture initAnalyticsFuture = null; + private int lastActiveSessions = 0; + public OcsgwMetrics(String metricsServerHostname, ServiceAccountJwtAccessCredentials credentials) { try { @@ -91,6 +93,7 @@ private void reconnectAnalyticsReport() { initAnalyticsFuture = executorService.schedule((Callable) () -> { LOG.info("Calling initAnalyticsRequest"); initAnalyticsRequest(); + sendAnalytics(lastActiveSessions); return "Called!"; }, 5, @@ -111,6 +114,7 @@ public void onNext(OcsgwAnalyticsReply value) { public void sendAnalytics(int size) { ocsgwAnalyticsReport.onNext(OcsgwAnalyticsReport.newBuilder().setActiveSessions(size).build()); + lastActiveSessions = size; } } \ No newline at end of file From f3cddeef8a99e43b4dea25092afed69c1acb6532 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Mon, 27 Aug 2018 16:29:30 +0200 Subject: [PATCH 25/30] Using default dropwizard metrics registry --- .../org/ostelco/prime/analytics/AnalyticsModule.kt | 6 ++---- .../org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt | 9 ++------- .../main/kotlin/org/ostelco/prime/PrimeApplication.kt | 2 +- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt index d9217ed46..44000d92a 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt @@ -3,12 +3,10 @@ package org.ostelco.prime.analytics import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.annotation.JsonTypeName import io.dropwizard.setup.Environment -import com.codahale.metrics.SharedMetricRegistries import org.hibernate.validator.constraints.NotEmpty -import org.ostelco.prime.analytics.ConfigRegistry.config -import org.ostelco.prime.module.PrimeModule import org.ostelco.prime.analytics.metrics.OcsgwMetrics import org.ostelco.prime.analytics.publishers.DataConsumptionInfoPublisher +import org.ostelco.prime.module.PrimeModule @JsonTypeName("analytics") class AnalyticsModule : PrimeModule { @@ -20,7 +18,7 @@ class AnalyticsModule : PrimeModule { override fun init(env: Environment) { - val ocsgwMetrics = OcsgwMetrics(SharedMetricRegistries.getOrCreate("ocsgw-metrics")) + val ocsgwMetrics = OcsgwMetrics(env.metrics()) val server = AnalyticsGrpcServer(8083, AnalyticsGrpcService(ocsgwMetrics)) diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt index 52983f9ea..9110d717c 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt @@ -2,23 +2,19 @@ package org.ostelco.prime.analytics.metrics import com.codahale.metrics.Gauge import com.codahale.metrics.MetricRegistry -import com.codahale.metrics.MetricRegistry.name -import org.ostelco.prime.analytics.AnalyticsGrpcService -import java.time.Instant -class OcsgwMetrics(private val registry: MetricRegistry) { +class OcsgwMetrics(registry: MetricRegistry) { private val activeSessions: Gauge private val currentActiveSessions = object { - var timestamp: Long = 0 var count: Long = 0 } init { this.activeSessions = registry.register( - name(AnalyticsGrpcService::class.java, "active-sessions"), + "active_sessions", Gauge { currentActiveSessions.count }) } @@ -29,7 +25,6 @@ class OcsgwMetrics(private val registry: MetricRegistry) { */ @Synchronized fun setActiveSessions(count: Long) { - currentActiveSessions.timestamp = Instant.now().toEpochMilli() currentActiveSessions.count = count } } diff --git a/prime/src/main/kotlin/org/ostelco/prime/PrimeApplication.kt b/prime/src/main/kotlin/org/ostelco/prime/PrimeApplication.kt index f8206a8ca..e8ca34def 100644 --- a/prime/src/main/kotlin/org/ostelco/prime/PrimeApplication.kt +++ b/prime/src/main/kotlin/org/ostelco/prime/PrimeApplication.kt @@ -19,7 +19,7 @@ class PrimeApplication : Application() { bootstrap.configurationSourceProvider, EnvironmentVariableSubstitutor()) bootstrap.objectMapper.registerModule(KotlinModule()) - bootstrap.addBundle(PrometheusBundle()); + bootstrap.addBundle(PrometheusBundle()) } override fun run( From 4c60dde8f83c433e9e5f7a5d0500df0e575e50d1 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Tue, 28 Aug 2018 08:03:19 +0200 Subject: [PATCH 26/30] Added more metrics --- .../org/ostelco/prime/admin/api/Resources.kt | 7 ++ .../prime/analytics/AnalyticsGrpcService.kt | 7 +- .../prime/analytics/AnalyticsModule.kt | 6 +- .../prime/analytics/AnalyticsServiceImpl.kt | 5 ++ .../metrics/CustomMetricsRegistry.kt | 74 +++++++++++++++++++ .../prime/analytics/metrics/OcsgwMetrics.kt | 30 -------- .../prime/client/api/ClientApiModule.kt | 9 +-- .../prime/client/api/metrics/Metrics.kt | 25 +++++++ .../client/api/store/SubscriberDAOImpl.kt | 25 +++++-- .../prime/client/api/auth/helpers/TestApp.kt | 6 +- .../ostelco/prime/storage/graph/Neo4jStore.kt | 38 ++++++++++ .../prime/analytics/DataConsumptionInfo.kt | 11 ++- .../ostelco/prime/disruptor/EventProducer.kt | 2 +- .../prime/handler/PurchaseRequestHandler.kt | 24 +++--- .../handler/PurchaseRequestHandlerTest.kt | 6 +- .../prime/analytics/AnalyticsService.kt | 23 ++++++ .../org/ostelco/prime/storage/Variants.kt | 4 + prime/config/config.yaml | 6 +- prime/config/test.yaml | 6 +- .../integration-tests/resources/config.yaml | 6 +- .../pseudonym/PseudonymResourceTest.kt | 2 +- 21 files changed, 244 insertions(+), 78 deletions(-) create mode 100644 analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/CustomMetricsRegistry.kt delete mode 100644 analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt create mode 100644 client-api/src/main/kotlin/org/ostelco/prime/client/api/metrics/Metrics.kt diff --git a/admin-api/src/main/kotlin/org/ostelco/prime/admin/api/Resources.kt b/admin-api/src/main/kotlin/org/ostelco/prime/admin/api/Resources.kt index cd4e48477..dd2477fc8 100644 --- a/admin-api/src/main/kotlin/org/ostelco/prime/admin/api/Resources.kt +++ b/admin-api/src/main/kotlin/org/ostelco/prime/admin/api/Resources.kt @@ -82,6 +82,13 @@ class SegmentResource { @PathParam("segment-id") segmentId: String, segment: Segment): Response { + if (segment.id != segmentId) { + return Response + .status(Response.Status.FORBIDDEN) + .entity("segment id in path and body do not match") + .build() + } + return adminDataSource.updateSegment(segment) .fold({ Response.status(Response.Status.NOT_MODIFIED).entity(it.message).build() }, { Response.ok().build() }) diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt index a564dbdb1..9cf9b7535 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt @@ -1,7 +1,8 @@ package org.ostelco.prime.analytics import io.grpc.stub.StreamObserver -import org.ostelco.prime.analytics.metrics.OcsgwMetrics +import org.ostelco.prime.analytics.PrimeMetric.ACTIVE_SESSIONS +import org.ostelco.prime.analytics.metrics.CustomMetricsRegistry import org.ostelco.prime.logger import org.ostelco.prime.metrics.api.OcsgwAnalyticsReply import org.ostelco.prime.metrics.api.OcsgwAnalyticsReport @@ -21,7 +22,7 @@ import java.util.* * } */ -class AnalyticsGrpcService(private val metrics : OcsgwMetrics) : OcsgwAnalyticsServiceGrpc.OcsgwAnalyticsServiceImplBase() { +class AnalyticsGrpcService : OcsgwAnalyticsServiceGrpc.OcsgwAnalyticsServiceImplBase() { private val logger by logger() @@ -50,7 +51,7 @@ class AnalyticsGrpcService(private val metrics : OcsgwMetrics) : OcsgwAnalyticsS * @param request provides current active session as a counter with a timestamp */ override fun onNext(request: OcsgwAnalyticsReport) { - metrics.setActiveSessions(request.activeSessions.toLong()) + CustomMetricsRegistry.updateMetricValue(ACTIVE_SESSIONS, request.activeSessions.toLong()) } override fun onError(t: Throwable) { diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt index 44000d92a..98b6a8ca8 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt @@ -4,7 +4,7 @@ import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.annotation.JsonTypeName import io.dropwizard.setup.Environment import org.hibernate.validator.constraints.NotEmpty -import org.ostelco.prime.analytics.metrics.OcsgwMetrics +import org.ostelco.prime.analytics.metrics.CustomMetricsRegistry import org.ostelco.prime.analytics.publishers.DataConsumptionInfoPublisher import org.ostelco.prime.module.PrimeModule @@ -18,9 +18,9 @@ class AnalyticsModule : PrimeModule { override fun init(env: Environment) { - val ocsgwMetrics = OcsgwMetrics(env.metrics()) + CustomMetricsRegistry.init(env.metrics()) - val server = AnalyticsGrpcServer(8083, AnalyticsGrpcService(ocsgwMetrics)) + val server = AnalyticsGrpcServer(8083, AnalyticsGrpcService()) env.lifecycle().manage(server) diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsServiceImpl.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsServiceImpl.kt index 6e2151fe8..99b9c34c5 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsServiceImpl.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsServiceImpl.kt @@ -1,5 +1,6 @@ package org.ostelco.prime.analytics +import org.ostelco.prime.analytics.metrics.CustomMetricsRegistry import org.ostelco.prime.analytics.publishers.DataConsumptionInfoPublisher import org.ostelco.prime.logger @@ -11,4 +12,8 @@ class AnalyticsServiceImpl : AnalyticsService { logger.info("reportTrafficInfo : msisdn {} usedBytes {} bundleBytes {}", msisdn, usedBytes, bundleBytes) DataConsumptionInfoPublisher.publish(msisdn, usedBytes, bundleBytes) } + + override fun reportMetric(primeMetric: PrimeMetric, value: Long) { + CustomMetricsRegistry.updateMetricValue(primeMetric, value) + } } \ No newline at end of file diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/CustomMetricsRegistry.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/CustomMetricsRegistry.kt new file mode 100644 index 000000000..0a1ae391f --- /dev/null +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/CustomMetricsRegistry.kt @@ -0,0 +1,74 @@ +package org.ostelco.prime.analytics.metrics + +import com.codahale.metrics.Counter +import com.codahale.metrics.Gauge +import com.codahale.metrics.MetricRegistry +import org.ostelco.prime.analytics.MetricType.COUNTER +import org.ostelco.prime.analytics.MetricType.GAUGE +import org.ostelco.prime.analytics.PrimeMetric + +/** + * Singleton wrapper dropwizard metrics. + */ +object CustomMetricsRegistry { + + private lateinit var registry: MetricRegistry + // boolean flag to avoid access to late init registry if it not yet initialized + private var isInitialized = false + + // map of long values which will act as cache for Gauge + private val gaugeValueMap: MutableMap = mutableMapOf() + + // map of counters + private val counterMap: MutableMap = mutableMapOf() + + @Synchronized + fun init(registry: MetricRegistry) { + this.registry = registry + isInitialized = true + counterMap.keys.forEach { registerCounter(it) } + gaugeValueMap.keys.forEach { registerGauge(it) } + } + + /** + * Update metric value. + * + * If metric is of type COUNTER, then the counter is increment by that value. + * If metric is of type GAUGE, then the gauge source is set to that value. + * + * @param primeMetric + * @param value + */ + @Synchronized + fun updateMetricValue(primeMetric: PrimeMetric, value: Long) { + when (primeMetric.metricType) { + COUNTER -> { + val counterExists = counterMap.containsKey(primeMetric) + counterMap.getOrPut(primeMetric) { Counter() }.inc(value) + if (isInitialized && !counterExists) { + registerCounter(primeMetric) + } + } + GAUGE -> { + val existingGaugeValue = gaugeValueMap.put(primeMetric, value) + if (isInitialized && existingGaugeValue == null) { + registerGauge(primeMetric) + } + } + } + } + + /** + * Register counter with value from counterMap + */ + private fun registerCounter(primeMetric: PrimeMetric) { + registry.register(primeMetric.metricName, counterMap[primeMetric]) + } + + /** + * Register gauge with value from gaugeValueMap as its source + */ + private fun registerGauge(primeMetric: PrimeMetric) { + registry.register(primeMetric.metricName, Gauge { gaugeValueMap[primeMetric] }) + } +} \ No newline at end of file diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt deleted file mode 100644 index 9110d717c..000000000 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/metrics/OcsgwMetrics.kt +++ /dev/null @@ -1,30 +0,0 @@ -package org.ostelco.prime.analytics.metrics - -import com.codahale.metrics.Gauge -import com.codahale.metrics.MetricRegistry - - -class OcsgwMetrics(registry: MetricRegistry) { - - private val activeSessions: Gauge - - private val currentActiveSessions = object { - var count: Long = 0 - } - - init { - this.activeSessions = registry.register( - "active_sessions", - Gauge { currentActiveSessions.count }) - } - - /** - * Records active user sessions. - * @param timestamp - seconds since epoch - * @param count - active sessions - */ - @Synchronized - fun setActiveSessions(count: Long) { - currentActiveSessions.count = count - } -} diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/ClientApiModule.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/ClientApiModule.kt index fc2b3ed6e..8795b4970 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/ClientApiModule.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/ClientApiModule.kt @@ -1,6 +1,5 @@ package org.ostelco.prime.client.api -import com.codahale.metrics.SharedMetricRegistries import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.annotation.JsonTypeName import com.fasterxml.jackson.databind.DeserializationFeature @@ -13,6 +12,7 @@ import io.dropwizard.client.JerseyClientBuilder import io.dropwizard.setup.Environment import org.ostelco.prime.client.api.auth.AccessTokenPrincipal import org.ostelco.prime.client.api.auth.OAuthAuthenticator +import org.ostelco.prime.client.api.metrics.reportMetricsAtStartUp import org.ostelco.prime.client.api.resources.AnalyticsResource import org.ostelco.prime.client.api.resources.ApplicationTokenResource import org.ostelco.prime.client.api.resources.ConsentsResource @@ -66,11 +66,8 @@ class ClientApiModule : PrimeModule { jerseyEnv.register(SubscriptionsResource(dao)) jerseyEnv.register(ApplicationTokenResource(dao)) - /* For reporting OAuth2 caching events. */ - val metrics = SharedMetricRegistries.getOrCreate(env.name) - /* OAuth2 with cache. */ - val authenticator = CachingAuthenticator(metrics, + val authenticator = CachingAuthenticator(env.metrics(), OAuthAuthenticator(client), config.authenticationCachePolicy) @@ -80,5 +77,7 @@ class ClientApiModule : PrimeModule { .setPrefix("Bearer") .buildAuthFilter())) jerseyEnv.register(AuthValueFactoryProvider.Binder(AccessTokenPrincipal::class.java)) + + reportMetricsAtStartUp() } } diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/metrics/Metrics.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/metrics/Metrics.kt new file mode 100644 index 000000000..51bd24737 --- /dev/null +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/metrics/Metrics.kt @@ -0,0 +1,25 @@ +package org.ostelco.prime.client.api.metrics + +import org.ostelco.prime.analytics.AnalyticsService +import org.ostelco.prime.analytics.PrimeMetric.TOTAL_USERS +import org.ostelco.prime.analytics.PrimeMetric.USERS_ACQUIRED_THROUGH_REFERRALS +import org.ostelco.prime.analytics.PrimeMetric.USERS_PAID_AT_LEAST_ONCE +import org.ostelco.prime.module.getResource +import org.ostelco.prime.storage.AdminDataSource + +val analyticsService: AnalyticsService = getResource() +val adminStore: AdminDataSource = getResource() + +fun reportMetricsAtStartUp() { + analyticsService.reportMetric(TOTAL_USERS, adminStore.getSubscriberCount()) + analyticsService.reportMetric(USERS_ACQUIRED_THROUGH_REFERRALS, adminStore.getReferredSubscriberCount()) + analyticsService.reportMetric(USERS_PAID_AT_LEAST_ONCE, adminStore.getPaidSubscriberCount()) +} + +fun updateMetricsOnNewSubscriber() { + analyticsService.reportMetric(TOTAL_USERS, adminStore.getSubscriberCount()) + analyticsService.reportMetric(USERS_ACQUIRED_THROUGH_REFERRALS, adminStore.getReferredSubscriberCount()) +} +fun updateMetricsOnPurchase() { + analyticsService.reportMetric(USERS_PAID_AT_LEAST_ONCE, adminStore.getPaidSubscriberCount()) +} \ No newline at end of file diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt index 153cd552f..1d9d2bca0 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt @@ -3,6 +3,10 @@ package org.ostelco.prime.client.api.store import arrow.core.Either import arrow.core.Tuple4 import arrow.core.flatMap +import org.ostelco.prime.analytics.AnalyticsService +import org.ostelco.prime.analytics.PrimeMetric.REVENUE +import org.ostelco.prime.client.api.metrics.updateMetricsOnNewSubscriber +import org.ostelco.prime.client.api.metrics.updateMetricsOnPurchase import org.ostelco.prime.client.api.model.Consent import org.ostelco.prime.client.api.model.Person import org.ostelco.prime.client.api.model.SubscriptionStatus @@ -40,6 +44,7 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu private val paymentProcessor by lazy { getResource() } private val pseudonymizer by lazy { getResource() } + private val analyticsReporter by lazy { getResource() } /* Table for 'profiles'. */ private val consentMap = ConcurrentHashMap>() @@ -63,7 +68,10 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu return try { storage.addSubscriber(profile, referredBy) .mapLeft { ForbiddenError("Failed to create profile. ${it.message}") } - .flatMap { getProfile(subscriberId) } + .flatMap { + updateMetricsOnNewSubscriber() + getProfile(subscriberId) + } } catch (e: Exception) { logger.error("Failed to create profile", e) Either.left(ForbiddenError("Failed to create profile")) @@ -212,6 +220,8 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu .flatMap { //TODO: Handle errors (when it becomes available) ocsSubscriberService.topup(subscriberId, sku) + // TODO vihang: handle currency conversion + analyticsReporter.reportMetric(REVENUE, product.price.amount.toLong()) Either.right(Unit) } } @@ -265,21 +275,26 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu .flatMap { //TODO: Handle errors (when it becomes available) ocsSubscriberService.topup(subscriberId, sku) - Either.right(Tuple4(profileInfo, savedSourceId, chargeId, ProductInfo(sku))) + Either.right(Tuple4(profileInfo, savedSourceId, chargeId, product)) } } - .flatMap { (profileInfo, savedSourceId, chargeId, productInfo) -> + .flatMap { (profileInfo, savedSourceId, chargeId, product) -> // Capture the charge, our database have been updated. paymentProcessor.captureCharge(chargeId, profileInfo.id, sourceId) .mapLeft { apiError -> logger.error("Capture failed for customerId ${profileInfo.id}, chargeId $chargeId, Fix this in Stripe Dashborad") apiError } - .map { Triple(profileInfo, savedSourceId, productInfo) } + .map { + // TODO vihang: handle currency conversion + analyticsReporter.reportMetric(REVENUE, product.price.amount.toLong()) + updateMetricsOnPurchase() + Triple(profileInfo, savedSourceId, ProductInfo(product.sku)) + } } .flatMap { (profileInfo, savedSourceId, productInfo) -> // Remove the payment source - if (saveCard == false && savedSourceId != null) { + if (!saveCard && savedSourceId != null) { paymentProcessor.removeSource(profileInfo.id, savedSourceId) .mapLeft { apiError -> logger.error("Failed to remove card, for customerId ${profileInfo.id}, sourceId $sourceId") diff --git a/client-api/src/test/kotlin/org/ostelco/prime/client/api/auth/helpers/TestApp.kt b/client-api/src/test/kotlin/org/ostelco/prime/client/api/auth/helpers/TestApp.kt index d744f8bd3..8af25a884 100644 --- a/client-api/src/test/kotlin/org/ostelco/prime/client/api/auth/helpers/TestApp.kt +++ b/client-api/src/test/kotlin/org/ostelco/prime/client/api/auth/helpers/TestApp.kt @@ -1,7 +1,6 @@ package org.ostelco.prime.client.api.auth.helpers import arrow.core.Either -import com.codahale.metrics.SharedMetricRegistries import com.fasterxml.jackson.databind.DeserializationFeature import com.fasterxml.jackson.databind.ObjectMapper import com.nhaarman.mockito_kotlin.argumentCaptor @@ -49,9 +48,6 @@ class TestApp : Application() { env.jersey().register(ProfileResource(DAO)) env.jersey().register(UserInfoResource()) - /* For reporting OAuth2 caching events. */ - val metrics = SharedMetricRegistries.getOrCreate(env.name) - val client = JerseyClientBuilder(env) .using(config.jerseyClientConfiguration) .using(ObjectMapper() @@ -59,7 +55,7 @@ class TestApp : Application() { .build(env.name) /* OAuth2 with cache. */ - val authenticator = CachingAuthenticator(metrics, + val authenticator = CachingAuthenticator(env.metrics(), OAuthAuthenticator(client), config.authenticationCachePolicy!!) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index 273dcd4e5..b2f109993 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -405,6 +405,44 @@ object Neo4jStoreSingleton : GraphStore { } } + // + // For metrics + // + override fun getSubscriberCount(): Long = readTransaction { + read(""" + MATCH (subscriber:${subscriberEntity.name}) + RETURN count(subscriber) AS count + """.trimIndent(), + transaction) { result -> + result.single().get("count").asLong() + } + } + + override fun getReferredSubscriberCount(): Long = readTransaction { + read(""" + MATCH (:${subscriberEntity.name})-[:${referredRelation.relation.name}]->(subscriber:${subscriberEntity.name}) + RETURN count(subscriber) AS count + """.trimIndent(), + transaction) { result -> + result.single().get("count").asLong() + } + } + + override fun getPaidSubscriberCount(): Long = readTransaction { + read(""" + MATCH (subscriber:${subscriberEntity.name})-[:${purchaseRecordRelation.relation.name}]->(product:${productEntity.name}) + WHERE product.`price/amount` > 0 + RETURN count(subscriber) AS count + """.trimIndent(), + transaction) { result -> + result.single().get("count").asLong() + } + } + + // + // Stores + // + private val offerEntity = EntityType(Offer::class.java) private val offerStore = EntityStore(offerEntity) diff --git a/ocs/src/main/kotlin/org/ostelco/prime/analytics/DataConsumptionInfo.kt b/ocs/src/main/kotlin/org/ostelco/prime/analytics/DataConsumptionInfo.kt index 576ab059e..09f3d6366 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/analytics/DataConsumptionInfo.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/analytics/DataConsumptionInfo.kt @@ -1,8 +1,9 @@ package org.ostelco.prime.analytics import com.lmax.disruptor.EventHandler -import org.ostelco.prime.disruptor.OcsEvent +import org.ostelco.prime.analytics.PrimeMetric.MEGABYTES_CONSUMED import org.ostelco.prime.disruptor.EventMessageType.CREDIT_CONTROL_REQUEST +import org.ostelco.prime.disruptor.OcsEvent import org.ostelco.prime.logger import org.ostelco.prime.module.getResource @@ -26,7 +27,13 @@ class DataConsumptionInfo() : EventHandler { if (event.msisdn != null) { logger.info("Sent DataConsumptionInfo event to analytics") - analyticsReporter.reportTrafficInfo(event.msisdn!!, event.usedBucketBytes, event.bundleBytes) + analyticsReporter.reportTrafficInfo( + msisdn = event.msisdn!!, + usedBytes = event.usedBucketBytes, + bundleBytes = event.bundleBytes) + analyticsReporter.reportMetric( + primeMetric = MEGABYTES_CONSUMED, + value = event.usedBucketBytes / 1_000_000) } } } diff --git a/ocs/src/main/kotlin/org/ostelco/prime/disruptor/EventProducer.kt b/ocs/src/main/kotlin/org/ostelco/prime/disruptor/EventProducer.kt index 8bc4c0526..a2e4880e8 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/disruptor/EventProducer.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/disruptor/EventProducer.kt @@ -5,7 +5,7 @@ import org.ostelco.prime.model.Bundle interface EventProducer { fun topupDataBundleBalanceEvent( - subscriberId: String, + bundleId: String, bytes: Long) fun releaseReservedDataBucketEvent( diff --git a/ocs/src/main/kotlin/org/ostelco/prime/handler/PurchaseRequestHandler.kt b/ocs/src/main/kotlin/org/ostelco/prime/handler/PurchaseRequestHandler.kt index c4a1c1490..b751d7dcc 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/handler/PurchaseRequestHandler.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/handler/PurchaseRequestHandler.kt @@ -19,21 +19,21 @@ class PurchaseRequestHandler( logger.info("Handling purchase request - subscriberId: {} sku = {}", subscriberId, productSku) // get Product by SKU - val product = storage.getProduct(subscriberId, productSku) + storage.getProduct(subscriberId, productSku) + .map { product -> + val noOfBytes = product.properties["noOfBytes"]?.replace("_", "")?.toLong() - if (product.isLeft()) { - throw Exception("Not a valid SKU: $productSku") - } + val bundleId = storage.getBundles(subscriberId).map { it?.first()?.id }.getOrElse { null } - val noOfBytes = product.get().properties["noOfBytes"]?.replace("_", "")?.toLong() + if (bundleId != null && noOfBytes != null && noOfBytes > 0) { - val bundleId = storage.getBundles(subscriberId).map { it?.first()?.id }.getOrElse { null } + logger.info("Handling topup product - bundleId: {} topup: {}", bundleId, noOfBytes) - if (bundleId != null && noOfBytes != null && noOfBytes > 0) { - - logger.info("Handling topup product - bundleId: {} topup: {}", bundleId, noOfBytes) - - producer.topupDataBundleBalanceEvent(bundleId, noOfBytes) - } + producer.topupDataBundleBalanceEvent(bundleId, noOfBytes) + } + }.mapLeft { + // TODO vihang: instead of throwing exception, return arrow.Either.left + throw Exception("Not a valid SKU: $productSku") + } } } diff --git a/ocs/src/test/kotlin/org/ostelco/prime/handler/PurchaseRequestHandlerTest.kt b/ocs/src/test/kotlin/org/ostelco/prime/handler/PurchaseRequestHandlerTest.kt index 03520f61c..837bfe450 100644 --- a/ocs/src/test/kotlin/org/ostelco/prime/handler/PurchaseRequestHandlerTest.kt +++ b/ocs/src/test/kotlin/org/ostelco/prime/handler/PurchaseRequestHandlerTest.kt @@ -37,7 +37,7 @@ class PurchaseRequestHandlerTest { @Before fun setUp() { - `when`< Either>(storage.getProduct("id", DATA_TOPUP_3GB.sku)) + `when`< Either>(storage.getProduct(SUBSCRIBER_ID, DATA_TOPUP_3GB.sku)) .thenReturn(Either.right(DATA_TOPUP_3GB)) this.purchaseRequestHandler = PurchaseRequestHandler(producer, storage) @@ -61,12 +61,14 @@ class PurchaseRequestHandlerTest { assertEquals(DATA_TOPUP_3GB, capturedPurchaseRecord.value.product) - verify(producer).topupDataBundleBalanceEvent(MSISDN, topupBytes) + verify(producer).topupDataBundleBalanceEvent(BUNDLE_ID, topupBytes) } companion object { private const val MSISDN = "12345678" + private const val SUBSCRIBER_ID = "foo@bar.com" + private const val BUNDLE_ID = "foo@bar.com" } diff --git a/prime-api/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt b/prime-api/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt index 325d83403..466f9f0be 100644 --- a/prime-api/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt +++ b/prime-api/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt @@ -1,5 +1,28 @@ package org.ostelco.prime.analytics +import org.ostelco.prime.analytics.MetricType.COUNTER +import org.ostelco.prime.analytics.MetricType.GAUGE + interface AnalyticsService { fun reportTrafficInfo(msisdn: String, usedBytes: Long, bundleBytes: Long) + fun reportMetric(primeMetric: PrimeMetric, value: Long) +} + +enum class PrimeMetric(val metricType: MetricType) { + + // sorted alphabetically + ACTIVE_SESSIONS(GAUGE), + MEGABYTES_CONSUMED(COUNTER), + REVENUE(COUNTER), + TOTAL_USERS(GAUGE), + USERS_ACQUIRED_THROUGH_REFERRALS(GAUGE), + USERS_PAID_AT_LEAST_ONCE(GAUGE); + + val metricName: String + get() = name.toLowerCase() } + +enum class MetricType { + COUNTER, + GAUGE, +} \ No newline at end of file diff --git a/prime-api/src/main/kotlin/org/ostelco/prime/storage/Variants.kt b/prime-api/src/main/kotlin/org/ostelco/prime/storage/Variants.kt index e07f3d693..a5fd68d89 100644 --- a/prime-api/src/main/kotlin/org/ostelco/prime/storage/Variants.kt +++ b/prime-api/src/main/kotlin/org/ostelco/prime/storage/Variants.kt @@ -137,6 +137,10 @@ interface AdminGraphStore { // updating an Offer and Product is not allowed fun updateSegment(segment: Segment): Either + fun getSubscriberCount(): Long + fun getReferredSubscriberCount(): Long + fun getPaidSubscriberCount(): Long + // simple getAll // fun getOffers(): Collection // fun getSegments(): Collection diff --git a/prime/config/config.yaml b/prime/config/config.yaml index aeaae4cb0..1f4bf8d61 100644 --- a/prime/config/config.yaml +++ b/prime/config/config.yaml @@ -8,13 +8,13 @@ modules: config: host: neo4j protocol: bolt+routing - - type: ocs - config: - lowBalanceThreshold: 100000000 - type: analytics config: projectId: pantel-2decb topicId: data-traffic + - type: ocs + config: + lowBalanceThreshold: 100000000 - type: pseudonymizer - type: api config: diff --git a/prime/config/test.yaml b/prime/config/test.yaml index 4b2db9944..68e56c2d9 100644 --- a/prime/config/test.yaml +++ b/prime/config/test.yaml @@ -10,13 +10,13 @@ modules: config: host: neo4j protocol: bolt - - type: ocs - config: - lowBalanceThreshold: 0 - type: analytics config: projectId: pantel-2decb topicId: data-traffic + - type: ocs + config: + lowBalanceThreshold: 0 - type: pseudonymizer - type: api config: diff --git a/prime/src/integration-tests/resources/config.yaml b/prime/src/integration-tests/resources/config.yaml index c814dc3b0..a8475ea4c 100644 --- a/prime/src/integration-tests/resources/config.yaml +++ b/prime/src/integration-tests/resources/config.yaml @@ -8,13 +8,13 @@ modules: config: host: 0.0.0.0 protocol: bolt - - type: ocs - config: - lowBalanceThreshold: 0 - type: analytics config: projectId: pantel-2decb topicId: data-traffic + - type: ocs + config: + lowBalanceThreshold: 0 - type: pseudonymizer - type: api config: diff --git a/pseudonym-server/src/test/kotlin/org/ostelco/pseudonym/PseudonymResourceTest.kt b/pseudonym-server/src/test/kotlin/org/ostelco/pseudonym/PseudonymResourceTest.kt index 46e8c68d0..a217f19ff 100644 --- a/pseudonym-server/src/test/kotlin/org/ostelco/pseudonym/PseudonymResourceTest.kt +++ b/pseudonym-server/src/test/kotlin/org/ostelco/pseudonym/PseudonymResourceTest.kt @@ -216,7 +216,7 @@ class PseudonymResourceTest { assertEquals(Status.OK.statusCode, result.status) val json = result.readEntity(String::class.java) val pseudonymEntity2 = mapper.readValue(json) - assertEquals(testMsisdn1, pseudonymEntity.msisdn) + assertEquals(testMsisdn1, pseudonymEntity2.msisdn) } } From 8e8774fc69121473bb79b138de9e16de5c3bc317 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Tue, 28 Aug 2018 14:35:01 +0200 Subject: [PATCH 27/30] Deploying prime with metrics to dev cluster --- build.gradle | 4 +- .../.gitignore | 0 docker-compose.dev.yaml | 12 +++++ docker-compose.override.yaml | 9 ++-- docker-compose.prod.yaml | 3 ++ docker-compose.yaml | 3 -- docs/TEST.md | 4 +- .../java/org/ostelco/ocsgw/OcsServer.java | 1 + .../org/ostelco/ocsgw/utils/AppConfig.java | 17 +++---- prime/infra/README.md | 16 +++++- prime/infra/dev/metrics-api.yaml | 8 +-- prime/infra/dev/prime.yaml | 51 +++++++++++++++++-- prime/script/deploy-dev-direct.sh | 2 + 13 files changed, 100 insertions(+), 30 deletions(-) rename certs/{metrics.dev.endpoints.pantel-2decb.cloud.goog => metrics.dev.ostelco.org}/.gitignore (100%) create mode 100644 docker-compose.dev.yaml diff --git a/build.gradle b/build.gradle index bb6b15cd6..04b352fea 100644 --- a/build.gradle +++ b/build.gradle @@ -41,8 +41,8 @@ task packDev(type: Zip, dependsOn: [':ocsgw:packDev', ':auth-server:pack']) { from zipTree('ocsgw/build/deploy/dev/ocsgw.zip') from zipTree('auth-server/build/deploy/auth-server.zip') from 'docker-compose.yaml' - from 'docker-compose.prod.yaml' - rename 'docker-compose.prod.yaml','docker-compose.override.yaml' + from 'docker-compose.dev.yaml' + rename 'docker-compose.dev.yaml','docker-compose.override.yaml' archiveName = 'ostelco-core-dev.zip' destinationDir = file('build/deploy/') } diff --git a/certs/metrics.dev.endpoints.pantel-2decb.cloud.goog/.gitignore b/certs/metrics.dev.ostelco.org/.gitignore similarity index 100% rename from certs/metrics.dev.endpoints.pantel-2decb.cloud.goog/.gitignore rename to certs/metrics.dev.ostelco.org/.gitignore diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml new file mode 100644 index 000000000..9377b7d23 --- /dev/null +++ b/docker-compose.dev.yaml @@ -0,0 +1,12 @@ +version: "3.3" + +services: + ocsgw: + environment: + - OCS_GRPC_SERVER=ocs.dev.ostelco.org + - METRICS_GRPC_SERVER=metrics.dev.ostelco.org + extra_hosts: + - "ocs.dev.ostelco.org:35.190.197.222" + - "api.dev.ostelco.org:35.190.192.27" + - "metrics.dev.ostelco.org:146.148.16.201" + network_mode: "host" \ No newline at end of file diff --git a/docker-compose.override.yaml b/docker-compose.override.yaml index 70e15cfa6..6b83015b3 100644 --- a/docker-compose.override.yaml +++ b/docker-compose.override.yaml @@ -56,9 +56,9 @@ services: image: gcr.io/endpoints-release/endpoints-runtime:1 volumes: - "./prime/config:/esp" - - "./certs/metrics.dev.endpoints.pantel-2decb.cloud.goog:/etc/nginx/ssl" + - "./certs/metrics.dev.ostelco.org:/etc/nginx/ssl" command: > - --service=metrics.dev.endpoints.pantel-2decb.cloud.goog + --service=metrics.dev.ostelco.org --rollout_strategy=managed --http2_port=80 --ssl_port=443 @@ -67,11 +67,14 @@ services: networks: net: aliases: - - "metrics.dev.endpoints.pantel-2decb.cloud.goog" + - "metrics.dev.ostelco.org" ipv4_address: 172.16.238.6 default: ocsgw: + environment: + - OCS_GRPC_SERVER=ocs.ostelco.org + - METRICS_GRPC_SERVER=metrics.dev.ostelco.org depends_on: - "prime" command: ["./wait.sh"] diff --git a/docker-compose.prod.yaml b/docker-compose.prod.yaml index 4e9b95958..5d5dd67fa 100644 --- a/docker-compose.prod.yaml +++ b/docker-compose.prod.yaml @@ -2,4 +2,7 @@ version: "3.3" services: ocsgw: + environment: + - OCS_GRPC_SERVER=ocs.ostelco.org + - METRICS_GRPC_SERVER=metrics.ostelco.org network_mode: "host" \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index fb44b676f..9895eeda0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -6,10 +6,7 @@ services: build: ocsgw environment: - GOOGLE_APPLICATION_CREDENTIALS=/config/pantel-prod.json - - GRPC_SERVER=ocs.ostelco.org - - METRICS_SERVER=metrics.dev.endpoints.pantel-2decb.cloud.goog - GOOGLE_CLOUD_PROJECT=pantel-2decb - auth-server: container_name: auth-server build: auth-server diff --git a/docs/TEST.md b/docs/TEST.md index 1e62c0cb1..8a8e18b3d 100644 --- a/docs/TEST.md +++ b/docs/TEST.md @@ -24,8 +24,8 @@ cp nginx.crt ../../ocsgw/config/ocs.crt ``` ```bash -cd certs/metrics.dev.endpoints.pantel-2decb.cloud.goog -openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ./nginx.key -out ./nginx.crt -subj '/CN=metrics.dev.endpoints.pantel-2decb.cloud.goog' +cd certs/metrics.dev.ostelco.org +openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ./nginx.key -out ./nginx.crt -subj '/CN=metrics.dev.ostelco.org' cp nginx.crt ../../ocsgw/config/metrics.crt ``` diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/OcsServer.java b/ocsgw/src/main/java/org/ostelco/ocsgw/OcsServer.java index 6a757aa29..e3b4c684a 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/OcsServer.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/OcsServer.java @@ -27,6 +27,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.naming.ConfigurationException; import java.io.IOException; diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/utils/AppConfig.java b/ocsgw/src/main/java/org/ostelco/ocsgw/utils/AppConfig.java index c534dbe5f..6695468e5 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/utils/AppConfig.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/utils/AppConfig.java @@ -1,16 +1,11 @@ package org.ostelco.ocsgw.utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.io.InputStream; import java.util.Properties; public class AppConfig { - private static final Logger LOG = LoggerFactory.getLogger(AppConfig.class); - private final Properties prop = new Properties(); public AppConfig() throws IOException { @@ -25,19 +20,19 @@ public String getDataStoreType () { } public String getGrpcServer() { - // GRPC_SERVER env has higher preference over config.properties - final String grpcServer = System.getenv("GRPC_SERVER"); + // OCS_GRPC_SERVER env has higher preference over config.properties + final String grpcServer = System.getenv("OCS_GRPC_SERVER"); if (grpcServer == null || grpcServer.isEmpty()) { - return prop.getProperty("GrpcServer", "127.0.0.1:8082"); + throw new Error("No OCS_GRPC_SERVER set in env"); } return grpcServer; } public String getMetricsServer() { - // GRPC_SERVER env has higher preference over config.properties - final String metricsServer = System.getenv("METRICS_SERVER"); + // METRICS_GRPC_SERVER env has higher preference over config.properties + final String metricsServer = System.getenv("METRICS_GRPC_SERVER"); if (metricsServer == null || metricsServer.isEmpty()) { - LOG.warn("No metric server set in env"); + throw new Error("No METRICS_GRPC_SERVER set in env"); } return metricsServer; } diff --git a/prime/infra/README.md b/prime/infra/README.md index 2605930f3..2582258fc 100644 --- a/prime/infra/README.md +++ b/prime/infra/README.md @@ -216,6 +216,12 @@ kubectl create secret generic api-ostelco-ssl \ --from-file=certs/dev.ostelco.org/nginx.key ``` +```bash +kubectl create secret generic metrics-ostelco-ssl \ + --from-file=certs/dev.ostelco.org/nginx.crt \ + --from-file=certs/dev.ostelco.org/nginx.key +``` + ### Endpoints * OCS gRPC endpoint @@ -231,12 +237,20 @@ python -m grpc_tools.protoc \ --proto_path=ocs-api/src/main/proto \ --descriptor_set_out=ocs_descriptor.pb \ ocs.proto + +python -m grpc_tools.protoc \ + --include_imports \ + --include_source_info \ + --proto_path=analytics-grpc-api/src/main/proto \ + --descriptor_set_out=metrics_descriptor.pb \ + prime_metrics.proto ``` Deploy endpoints ```bash gcloud endpoints services deploy ocs_descriptor.pb prime/infra/dev/ocs-api.yaml +gcloud endpoints services deploy metrics_descriptor.pb prime/infra/dev/metrics-api.yaml ``` * Client API HTTP endpoint @@ -258,7 +272,7 @@ Then, import initial data into neo4j using `tools/neo4j-admin-tools`. ### Deploy prime ```bash -prime/script/deploy-dev.sh +prime/script/deploy-dev-direct.sh ``` OR diff --git a/prime/infra/dev/metrics-api.yaml b/prime/infra/dev/metrics-api.yaml index 85a47600f..750adc869 100644 --- a/prime/infra/dev/metrics-api.yaml +++ b/prime/infra/dev/metrics-api.yaml @@ -2,7 +2,7 @@ type: google.api.Service config_version: 3 -name: metrics.dev.endpoints.pantel-2decb.cloud.goog +name: metrics.dev.ostelco.org title: Prime Metrics Reporter Service gRPC API @@ -21,9 +21,9 @@ authentication: issuer: prime-service-account@pantel-2decb.iam.gserviceaccount.com jwks_uri: https://www.googleapis.com/robot/v1/metadata/x509/prime-service-account@pantel-2decb.iam.gserviceaccount.com audiences: > - https://metrics.dev.endpoints.pantel-2decb.cloud.goog/org.ostelco.prime.metrics.api.OcsgwAnalyticsService, - metrics.dev.endpoints.pantel-2decb.cloud.goog/org.ostelco.prime.metrics.api.OcsgwAnalyticsService, - metrics.dev.endpoints.pantel-2decb.cloud.goog + https://metrics.dev.ostelco.org/org.ostelco.prime.metrics.api.OcsgwAnalyticsService, + metrics.dev.ostelco.org/org.ostelco.prime.metrics.api.OcsgwAnalyticsService, + metrics.dev.ostelco.org rules: - selector: "*" requirements: diff --git a/prime/infra/dev/prime.yaml b/prime/infra/dev/prime.yaml index f9b61f986..a387312fd 100644 --- a/prime/infra/dev/prime.yaml +++ b/prime/infra/dev/prime.yaml @@ -7,6 +7,7 @@ metadata: tier: backend spec: type: LoadBalancer + loadBalancerIP: 35.190.197.222 ports: - name: grpc port: 443 @@ -25,10 +26,30 @@ metadata: tier: backend spec: type: LoadBalancer + loadBalancerIP: 35.190.192.27 ports: - - port: 443 + - name: https + port: 443 + protocol: TCP + selector: + app: prime + tier: backend +--- +apiVersion: v1 +kind: Service +metadata: + name: prime-metrics + labels: + app: prime + tier: backend +spec: + type: LoadBalancer + loadBalancerIP: 146.148.16.201 + ports: + - name: grpc + port: 443 + targetPort: 9443 protocol: TCP - name: https selector: app: prime tier: backend @@ -49,11 +70,12 @@ spec: tier: backend spec: containers: - - name: esp + - name: ocs-esp image: gcr.io/endpoints-release/endpoints-runtime:1 args: [ "--http2_port=9000", "--ssl_port=8443", + "--status_port=8090", "--service=ocs.dev.ostelco.org", "--rollout_strategy=managed", "--backend=grpc://127.0.0.1:8082" @@ -82,6 +104,23 @@ spec: - mountPath: /etc/nginx/ssl name: api-ostelco-ssl readOnly: true + - name: metrics-esp + image: gcr.io/endpoints-release/endpoints-runtime:1 + args: [ + "--http2_port=9004", + "--ssl_port=9443", + "--status_port=8094", + "--service=metrics.dev.ostelco.org", + "--rollout_strategy=managed", + "--backend=grpc://127.0.0.1:8083" + ] + ports: + - containerPort: 9004 + - containerPort: 9443 + volumeMounts: + - mountPath: /etc/nginx/ssl + name: metrics-ostelco-ssl + readOnly: true - name: prime image: gcr.io/pantel-2decb/prime:PRIME_VERSION env: @@ -100,6 +139,7 @@ spec: - containerPort: 8080 - containerPort: 8081 - containerPort: 8082 + - containerPort: 8083 volumes: - name: secret-config secret: @@ -109,4 +149,7 @@ spec: secretName: api-ostelco-ssl - name: ocs-ostelco-ssl secret: - secretName: ocs-ostelco-ssl \ No newline at end of file + secretName: ocs-ostelco-ssl + - name: metrics-ostelco-ssl + secret: + secretName: metrics-ostelco-ssl \ No newline at end of file diff --git a/prime/script/deploy-dev-direct.sh b/prime/script/deploy-dev-direct.sh index 2abc40151..408df2c44 100755 --- a/prime/script/deploy-dev-direct.sh +++ b/prime/script/deploy-dev-direct.sh @@ -7,6 +7,8 @@ if [ ! -f prime/script/deploy.sh ]; then exit 1 fi +# TODO vihang: check if the kubectl config points to dev cluster + PROJECT_ID="$(gcloud config get-value project -q)" PRIME_VERSION="$(gradle prime:properties -q | grep "version:" | awk '{print $2}' | tr -d '[:space:]')" SHORT_SHA="$(git log -1 --pretty=format:%h)" From d05f904daba55f290bc3887add1911d7ad4d4c4f Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Wed, 29 Aug 2018 14:45:52 +0200 Subject: [PATCH 28/30] Deploying ocsgw dev instance --- acceptance-tests/build.gradle | 2 +- analytics-grpc-api/build.gradle | 8 ++++---- app-notifier/build.gradle | 4 ++-- auth-server/build.gradle | 2 +- build.gradle | 2 +- certs/metrics.ostelco.org/.gitignore | 2 ++ certs/ocs.dev.ostelco.org/.gitignore | 2 ++ docker-compose.dev.yaml | 4 ---- docker-compose.override.yaml | 8 ++++---- docs/TEST.md | 4 ++-- firebase-store/build.gradle | 2 +- gradle/wrapper/gradle-wrapper.properties | 2 +- model/build.gradle | 2 +- ocs-grpc-api/build.gradle | 8 ++++---- ocsgw/build.gradle | 16 +++++++++++++++- ostelco-lib/build.gradle | 2 +- payment-processor/build.gradle | 2 +- prime-api/build.gradle | 4 ++-- prime-client-api/build.gradle | 2 +- prime/infra/README.md | 8 +++++--- prime/infra/dev/prime.yaml | 4 ++++ prime/infra/prod/prime.yaml | 9 +++++++-- pseudonym-server/build.gradle | 4 ++-- 23 files changed, 64 insertions(+), 39 deletions(-) create mode 100644 certs/metrics.ostelco.org/.gitignore create mode 100644 certs/ocs.dev.ostelco.org/.gitignore diff --git a/acceptance-tests/build.gradle b/acceptance-tests/build.gradle index b6689a927..3b6a16248 100644 --- a/acceptance-tests/build.gradle +++ b/acceptance-tests/build.gradle @@ -14,7 +14,7 @@ dependencies { implementation project(":prime-client-api") implementation project(':diameter-test') - implementation "com.stripe:stripe-java:6.3.0" + implementation "com.stripe:stripe-java:6.8.0" implementation 'io.jsonwebtoken:jjwt:0.9.1' // tests fail when updated to 2.27 implementation "org.glassfish.jersey.media:jersey-media-json-jackson:2.25.1" diff --git a/analytics-grpc-api/build.gradle b/analytics-grpc-api/build.gradle index 08b28a100..308b1b22f 100644 --- a/analytics-grpc-api/build.gradle +++ b/analytics-grpc-api/build.gradle @@ -4,9 +4,9 @@ plugins { id "idea" } -// Keeping it version 1.13.1 to be consistent with grpc via PubSub client lib -// Keeping it version 1.13.1 to be consistent with netty via Firebase lib -ext.grpcVersion = "1.13.1" +// Keeping it version 1.14.0 to be consistent with grpc via PubSub client lib +// Keeping it version 1.14.0 to be consistent with netty via Firebase lib +ext.grpcVersion = "1.14.0" dependencies { api "io.grpc:grpc-netty-shaded:$grpcVersion" @@ -21,7 +21,7 @@ protobuf { artifact = "io.grpc:protoc-gen-grpc-java:$grpcVersion" } } - protoc { artifact = 'com.google.protobuf:protoc:3.6.0' } + protoc { artifact = 'com.google.protobuf:protoc:3.6.1' } generateProtoTasks { all()*.plugins { grpc {} diff --git a/app-notifier/build.gradle b/app-notifier/build.gradle index 4489a3fd8..4f8a2ff67 100644 --- a/app-notifier/build.gradle +++ b/app-notifier/build.gradle @@ -5,8 +5,8 @@ plugins { dependencies { implementation project(":prime-api") - // Keep it to 5.10.0 to match netty via ocs-api - implementation 'com.google.firebase:firebase-admin:6.2.0' + // Keep it to 6.4.0 to match netty via ocs-api + implementation 'com.google.firebase:firebase-admin:6.4.0' testImplementation "org.jetbrains.kotlin:kotlin-test:$kotlinVersion" testImplementation "org.jetbrains.kotlin:kotlin-test-junit:$kotlinVersion" diff --git a/auth-server/build.gradle b/auth-server/build.gradle index 0c5ed3979..9f4f2e9b2 100644 --- a/auth-server/build.gradle +++ b/auth-server/build.gradle @@ -8,7 +8,7 @@ plugins { dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion" implementation "io.dropwizard:dropwizard-core:$dropwizardVersion" - implementation 'com.google.firebase:firebase-admin:6.2.0' + implementation 'com.google.firebase:firebase-admin:6.4.0' testImplementation "io.dropwizard:dropwizard-testing:$dropwizardVersion" testImplementation "org.jetbrains.kotlin:kotlin-test-junit:$kotlinVersion" testRuntimeOnly 'org.hamcrest:hamcrest-all:1.3' diff --git a/build.gradle b/build.gradle index 04b352fea..e890c8522 100644 --- a/build.gradle +++ b/build.gradle @@ -30,7 +30,7 @@ subprojects { ext { kotlinVersion = "1.2.61" dropwizardVersion = "1.3.5" - googleCloudVersion = "1.35.0" + googleCloudVersion = "1.41.0" jacksonVersion = "2.9.6" } } diff --git a/certs/metrics.ostelco.org/.gitignore b/certs/metrics.ostelco.org/.gitignore new file mode 100644 index 000000000..47c805dfe --- /dev/null +++ b/certs/metrics.ostelco.org/.gitignore @@ -0,0 +1,2 @@ +nginx.crt +nginx.key \ No newline at end of file diff --git a/certs/ocs.dev.ostelco.org/.gitignore b/certs/ocs.dev.ostelco.org/.gitignore new file mode 100644 index 000000000..47c805dfe --- /dev/null +++ b/certs/ocs.dev.ostelco.org/.gitignore @@ -0,0 +1,2 @@ +nginx.crt +nginx.key \ No newline at end of file diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 9377b7d23..e75bd7443 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -5,8 +5,4 @@ services: environment: - OCS_GRPC_SERVER=ocs.dev.ostelco.org - METRICS_GRPC_SERVER=metrics.dev.ostelco.org - extra_hosts: - - "ocs.dev.ostelco.org:35.190.197.222" - - "api.dev.ostelco.org:35.190.192.27" - - "metrics.dev.ostelco.org:146.148.16.201" network_mode: "host" \ No newline at end of file diff --git a/docker-compose.override.yaml b/docker-compose.override.yaml index 6b83015b3..2923d1a8d 100644 --- a/docker-compose.override.yaml +++ b/docker-compose.override.yaml @@ -35,9 +35,9 @@ services: image: gcr.io/endpoints-release/endpoints-runtime:1 volumes: - "./prime/config:/esp" - - "./certs/ocs.ostelco.org:/etc/nginx/ssl" + - "./certs/ocs.dev.ostelco.org:/etc/nginx/ssl" command: > - --service=ocs.ostelco.org + --service=ocs.dev.ostelco.org --rollout_strategy=managed --http2_port=80 --ssl_port=443 @@ -46,7 +46,7 @@ services: networks: net: aliases: - - "ocs.ostelco.org" + - "ocs.dev.ostelco.org" ipv4_address: 172.16.238.4 default: @@ -73,7 +73,7 @@ services: ocsgw: environment: - - OCS_GRPC_SERVER=ocs.ostelco.org + - OCS_GRPC_SERVER=ocs.dev.ostelco.org - METRICS_GRPC_SERVER=metrics.dev.ostelco.org depends_on: - "prime" diff --git a/docs/TEST.md b/docs/TEST.md index 8a8e18b3d..4b93d4df8 100644 --- a/docs/TEST.md +++ b/docs/TEST.md @@ -18,8 +18,8 @@ grep -i pantel $(find . -name '.gitignore') | awk -F: '{print $1}' | sort | uniq * In `ocsgw/config`, keep `nginx.cert`. ```bash -cd certs/ocs.ostelco.org -openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ./nginx.key -out ./nginx.crt -subj '/CN=ocs.ostelco.org' +cd certs/ocs.dev.ostelco.org +openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ./nginx.key -out ./nginx.crt -subj '/CN=ocs.dev.ostelco.org' cp nginx.crt ../../ocsgw/config/ocs.crt ``` diff --git a/firebase-store/build.gradle b/firebase-store/build.gradle index 8d3dc3fe9..88fc3cd52 100644 --- a/firebase-store/build.gradle +++ b/firebase-store/build.gradle @@ -6,7 +6,7 @@ plugins { dependencies { implementation project(":prime-api") // Match netty via ocs-api - api 'com.google.firebase:firebase-admin:6.2.0' + api 'com.google.firebase:firebase-admin:6.4.0' implementation "com.fasterxml.jackson.module:jackson-module-kotlin:$jacksonVersion" testImplementation "org.jetbrains.kotlin:kotlin-test:$kotlinVersion" diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index a95009c3b..115e6ac0a 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.9-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-4.10-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/model/build.gradle b/model/build.gradle index b465dbcef..081b9c8f6 100644 --- a/model/build.gradle +++ b/model/build.gradle @@ -8,6 +8,6 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-annotations:$jacksonVersion" // TODO vihang: this dependency is added only for @Exclude annotation for firebase - implementation 'com.google.firebase:firebase-admin:6.2.0' + implementation 'com.google.firebase:firebase-admin:6.4.0' implementation "org.slf4j:slf4j-api:1.7.25" } \ No newline at end of file diff --git a/ocs-grpc-api/build.gradle b/ocs-grpc-api/build.gradle index 08b28a100..308b1b22f 100644 --- a/ocs-grpc-api/build.gradle +++ b/ocs-grpc-api/build.gradle @@ -4,9 +4,9 @@ plugins { id "idea" } -// Keeping it version 1.13.1 to be consistent with grpc via PubSub client lib -// Keeping it version 1.13.1 to be consistent with netty via Firebase lib -ext.grpcVersion = "1.13.1" +// Keeping it version 1.14.0 to be consistent with grpc via PubSub client lib +// Keeping it version 1.14.0 to be consistent with netty via Firebase lib +ext.grpcVersion = "1.14.0" dependencies { api "io.grpc:grpc-netty-shaded:$grpcVersion" @@ -21,7 +21,7 @@ protobuf { artifact = "io.grpc:protoc-gen-grpc-java:$grpcVersion" } } - protoc { artifact = 'com.google.protobuf:protoc:3.6.0' } + protoc { artifact = 'com.google.protobuf:protoc:3.6.1' } generateProtoTasks { all()*.plugins { grpc {} diff --git a/ocsgw/build.gradle b/ocsgw/build.gradle index b8ac38f13..3a307fed5 100644 --- a/ocsgw/build.gradle +++ b/ocsgw/build.gradle @@ -13,7 +13,7 @@ dependencies { implementation 'ch.qos.logback:logback-classic:1.2.3' // log to gcp stack-driver - implementation 'com.google.cloud:google-cloud-logging-logback:0.53.0-alpha' + implementation 'com.google.cloud:google-cloud-logging-logback:0.59.0-alpha' testImplementation project(':diameter-test') testImplementation 'org.junit.jupiter:junit-jupiter-api:5.2.0' @@ -85,6 +85,20 @@ task packDev(type: Zip, dependsOn: 'shadowJar') { fileName.replace('dev.', '') } } + // TODO vihang: figure out why wild-card certs fail to verify + from ('../certs/dev.ostelco.org/nginx.crt') { + into (project.name + '/config/') + rename { String fileName -> + fileName.replace('nginx', 'ocs') + } + } + from ('../certs/dev.ostelco.org/nginx.crt') { + into (project.name + '/config/') + rename { String fileName -> + fileName.replace('nginx', 'metrics') + } + } + // END of certs from ('config/pantel-prod.json') { into (project.name + '/config/') } diff --git a/ostelco-lib/build.gradle b/ostelco-lib/build.gradle index b85c065fb..a320261f4 100644 --- a/ostelco-lib/build.gradle +++ b/ostelco-lib/build.gradle @@ -7,7 +7,7 @@ dependencies { implementation "io.dropwizard:dropwizard-core:$dropwizardVersion" implementation "io.dropwizard:dropwizard-auth:$dropwizardVersion" // Match netty via ocs-api - implementation 'com.google.firebase:firebase-admin:6.2.0' + implementation 'com.google.firebase:firebase-admin:6.4.0' implementation 'com.lmax:disruptor:3.4.2' implementation 'com.google.guava:guava:25.1-jre' diff --git a/payment-processor/build.gradle b/payment-processor/build.gradle index 7d322d658..727e5e3e0 100644 --- a/payment-processor/build.gradle +++ b/payment-processor/build.gradle @@ -21,7 +21,7 @@ dependencies { implementation project(":prime-api") - implementation "com.stripe:stripe-java:5.51.0" + implementation "com.stripe:stripe-java:6.8.0" testImplementation "org.jetbrains.kotlin:kotlin-test:$kotlinVersion" testImplementation "org.jetbrains.kotlin:kotlin-test-junit:$kotlinVersion" diff --git a/prime-api/build.gradle b/prime-api/build.gradle index 5ffc69148..d678ce5f9 100644 --- a/prime-api/build.gradle +++ b/prime-api/build.gradle @@ -7,12 +7,12 @@ dependencies { api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion" api "org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion" - api "io.jsonwebtoken:jjwt:0.9.0" + api "io.jsonwebtoken:jjwt:0.9.1" api project(':ocs-grpc-api') api project(':analytics-grpc-api') api project(':model') api "io.dropwizard:dropwizard-core:$dropwizardVersion" - api "io.arrow-kt:arrow-core:0.7.2" + api "io.arrow-kt:arrow-core:0.7.3" } \ No newline at end of file diff --git a/prime-client-api/build.gradle b/prime-client-api/build.gradle index ad07f35e9..9bbc9c03d 100644 --- a/prime-client-api/build.gradle +++ b/prime-client-api/build.gradle @@ -45,7 +45,7 @@ dependencies { implementation 'com.squareup.okhttp:okhttp:2.7.5' implementation 'com.squareup.okhttp:logging-interceptor:2.7.5' implementation 'io.gsonfire:gson-fire:1.8.3' - implementation 'org.threeten:threetenbp:1.3.6' + implementation 'org.threeten:threetenbp:1.3.7' testImplementation 'junit:junit:4.12' } diff --git a/prime/infra/README.md b/prime/infra/README.md index 2582258fc..11d55723c 100644 --- a/prime/infra/README.md +++ b/prime/infra/README.md @@ -150,7 +150,7 @@ kubectl describe service prime-service gcloud endpoints services deploy prime/infra/prod/prime-client-api.yaml ``` -## SSL secrets for api.ostelco.org & ocs.ostelco.org +## SSL secrets for api.ostelco.org, ocs.ostelco.org & metrics.ostelco.org The endpoints runtime expects the SSL configuration to be named as `nginx.crt` and `nginx.key`. Sample command to create the secret: ```bash @@ -158,8 +158,10 @@ kubectl create secret generic api-ostelco-ssl \ --from-file=./nginx.crt \ --from-file=./nginx.key ``` -The secret for api.ostelco.org is in `api-ostelco-ssl` & the one for -ocs.ostelco.org is in `ocs-ostelco-ssl` +The secret for ... + * `api.ostelco.org` is in `api-ostelco-ssl` + * `ocs.ostelco.org` is in `ocs-ostelco-ssl` + * `metrics.ostelco.org` is in `metrics-ostelco-ssl` # For Dev cluster diff --git a/prime/infra/dev/prime.yaml b/prime/infra/dev/prime.yaml index a387312fd..e78ea3e16 100644 --- a/prime/infra/dev/prime.yaml +++ b/prime/infra/dev/prime.yaml @@ -68,6 +68,10 @@ spec: labels: app: prime tier: backend + annotations: + prometheus.io/scrape: 'true' + prometheus.io/path: '/prometheus-metrics' + prometheus.io/port: '8081' spec: containers: - name: ocs-esp diff --git a/prime/infra/prod/prime.yaml b/prime/infra/prod/prime.yaml index e36b0b0ee..de74f8fc2 100644 --- a/prime/infra/prod/prime.yaml +++ b/prime/infra/prod/prime.yaml @@ -28,9 +28,9 @@ spec: type: LoadBalancer loadBalancerIP: 35.233.36.235 ports: - - port: 443 + - name: https + port: 443 protocol: TCP - name: https selector: app: prime tier: backend @@ -49,6 +49,10 @@ spec: labels: app: prime tier: backend + annotations: + prometheus.io/scrape: 'true' + prometheus.io/path: '/prometheus-metrics' + prometheus.io/port: '8081' spec: containers: - name: esp @@ -56,6 +60,7 @@ spec: args: [ "--http2_port=9000", "--ssl_port=8443", + "--status_port=8090", "--service=ocs.ostelco.org", "--rollout_strategy=managed", "--backend=grpc://127.0.0.1:8082" diff --git a/pseudonym-server/build.gradle b/pseudonym-server/build.gradle index 6aa518979..6f5af1d2f 100644 --- a/pseudonym-server/build.gradle +++ b/pseudonym-server/build.gradle @@ -14,8 +14,8 @@ dependencies { implementation "io.dropwizard:dropwizard-client:$dropwizardVersion" implementation 'com.google.guava:guava:25.1-jre' // Match with grpc-netty-shaded via PubSub - // removing io.grpc:grpc-netty-shaded:1.13.1 causes ALPN error - implementation 'io.grpc:grpc-netty-shaded:1.13.1' + // removing io.grpc:grpc-netty-shaded:1.14.0 causes ALPN error + implementation 'io.grpc:grpc-netty-shaded:1.14.0' implementation "com.google.cloud:google-cloud-bigquery:$googleCloudVersion" implementation "com.google.cloud:google-cloud-datastore:$googleCloudVersion" implementation "com.google.cloud:google-cloud-pubsub:$googleCloudVersion" From ca523a7f1b5cf64dcc9ce3ac7abe354b856e4a28 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Thu, 30 Aug 2018 09:15:19 +0200 Subject: [PATCH 29/30] Renamed analytics to dataflow-pipelines and refactored to support adding more pipelines into it --- .../analytics/DataConsumptionPipeline.kt | 156 ------------------ {analytics => dataflow-pipelines}/Dockerfile | 2 +- {analytics => dataflow-pipelines}/README.md | 2 +- .../build.gradle | 2 +- .../config/.gitignore | 0 .../config/logback.xml | 0 .../docker-compose.yaml | 4 +- .../script/start.sh | 0 .../dataflow/pipelines/DeployPipeline.kt | 63 +++++++ .../DataConsumptionPipelineDefinition.kt | 123 ++++++++++++++ .../definitions/DummyPipelineDefinition.kt | 9 + .../definitions/PipelineDefinition.kt | 7 + .../dataflow/pipelines/dsl}/ParDoFn.kt | 2 +- .../ostelco/dataflow/pipelines/io/BigQuery.kt | 56 +++++-- .../ostelco/dataflow/pipelines/io/PubSub.kt | 9 + .../pipelines/ConsumptionPerMsisdnTest.kt | 9 +- .../src/test/resources/logback-test.xml | 0 settings.gradle | 4 +- .../src/main/resources/docker-compose.yaml | 2 +- 19 files changed, 271 insertions(+), 179 deletions(-) delete mode 100644 analytics/src/main/kotlin/org/ostelco/analytics/DataConsumptionPipeline.kt rename {analytics => dataflow-pipelines}/Dockerfile (71%) rename {analytics => dataflow-pipelines}/README.md (89%) rename {analytics => dataflow-pipelines}/build.gradle (92%) rename {analytics => dataflow-pipelines}/config/.gitignore (100%) rename {analytics => dataflow-pipelines}/config/logback.xml (100%) rename {analytics => dataflow-pipelines}/docker-compose.yaml (68%) rename {analytics => dataflow-pipelines}/script/start.sh (100%) create mode 100644 dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/DeployPipeline.kt create mode 100644 dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/definitions/DataConsumptionPipelineDefinition.kt create mode 100644 dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/definitions/DummyPipelineDefinition.kt create mode 100644 dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/definitions/PipelineDefinition.kt rename {analytics/src/main/kotlin/org/ostelco/analytics => dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/dsl}/ParDoFn.kt (91%) rename analytics/src/main/kotlin/org/ostelco/analytics/TableSchemas.kt => dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/io/BigQuery.kt (59%) create mode 100644 dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/io/PubSub.kt rename analytics/src/test/kotlin/org/ostelco/analytics/PipelineTest.kt => dataflow-pipelines/src/test/kotlin/org/ostelco/dataflow/pipelines/ConsumptionPerMsisdnTest.kt (93%) rename {analytics => dataflow-pipelines}/src/test/resources/logback-test.xml (100%) diff --git a/analytics/src/main/kotlin/org/ostelco/analytics/DataConsumptionPipeline.kt b/analytics/src/main/kotlin/org/ostelco/analytics/DataConsumptionPipeline.kt deleted file mode 100644 index 62fff2a56..000000000 --- a/analytics/src/main/kotlin/org/ostelco/analytics/DataConsumptionPipeline.kt +++ /dev/null @@ -1,156 +0,0 @@ -package org.ostelco.analytics - -import ch.qos.logback.classic.util.ContextInitializer -import com.google.api.services.bigquery.model.TableRow -import com.google.protobuf.util.Timestamps -import org.apache.beam.runners.dataflow.DataflowRunner -import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions -import org.apache.beam.sdk.Pipeline -import org.apache.beam.sdk.coders.KvCoder -import org.apache.beam.sdk.coders.VarLongCoder -import org.apache.beam.sdk.extensions.protobuf.ProtoCoder -import org.apache.beam.sdk.io.gcp.pubsub.PubsubIO -import org.apache.beam.sdk.options.PipelineOptionsFactory -import org.apache.beam.sdk.transforms.Combine -import org.apache.beam.sdk.transforms.Filter -import org.apache.beam.sdk.transforms.GroupByKey -import org.apache.beam.sdk.transforms.SerializableFunction -import org.apache.beam.sdk.transforms.Sum -import org.apache.beam.sdk.transforms.WithTimestamps -import org.apache.beam.sdk.transforms.windowing.FixedWindows -import org.apache.beam.sdk.transforms.windowing.Window -import org.apache.beam.sdk.values.KV -import org.apache.beam.sdk.values.PCollection -import org.joda.time.Duration -import org.joda.time.Instant -import org.ostelco.analytics.ParDoFn.transform -import org.ostelco.analytics.Table.HOURLY_CONSUMPTION -import org.ostelco.analytics.Table.RAW_CONSUMPTION -import org.ostelco.analytics.api.AggregatedDataTrafficInfo -import org.ostelco.analytics.api.DataTrafficInfo -import java.time.ZoneOffset -import java.time.ZonedDateTime -import java.time.format.DateTimeFormatter - - -fun main(args: Array) { - - System.setProperty(ContextInitializer.CONFIG_FILE_PROPERTY, "config/logback.xml") - - // may be we need to pass options via command-line args - /* - val options = PipelineOptionsFactory - .fromArgs( - "--project=pantel-2decb", - "--runner=DataflowRunner", - "--stagingLocation=gs://data-traffic/staging/", - "--jobName=data-traffic") - .withValidation() - .create() - */ - - val options = PipelineOptionsFactory.`as`(DataflowPipelineOptions::class.java) - options.jobName = "data-traffic" - options.project = "pantel-2decb" - options.stagingLocation = "gs://data-traffic/staging/" - options.region = "europe-west1" - options.runner = DataflowRunner::class.java - options.isUpdate = true - - val pipeline = Pipeline.create(options) - - val dataTrafficInfoEvents = pipeline - .apply(PubsubIO.readProtos(DataTrafficInfo::class.java) - .fromSubscription("projects/pantel-2decb/subscriptions/data-traffic")) - .apply>(Filter.by(SerializableFunction { it.bucketBytes > 0 })) - - val saveRawEventsToBigQuery = BigQueryIOUtils().writeTo(RAW_CONSUMPTION) - - val saveToBigQueryGroupedByHour = BigQueryIOUtils().writeTo(HOURLY_CONSUMPTION) - - val convertToRawTableRows = transform { - TableRow() - .set("msisdn", it.msisdn) - .set("bucketBytes", it.bucketBytes) - .set("bundleBytes", it.bundleBytes) - .set("timestamp", ZonedDateTime.ofInstant( - java.time.Instant.ofEpochMilli(Timestamps.toMillis(it.timestamp)), - ZoneOffset.UTC).toString()) - } - - val convertToHourlyTableRows = transform { - TableRow() - .set("msisdn", it.msisdn) - .set("bytes", it.dataBytes) - .set("timestamp", it.dateTime) - } - - // PubSubEvents -> raw_consumption big-query - dataTrafficInfoEvents - .apply(convertToRawTableRows) - .apply(saveRawEventsToBigQuery) - - // PubSubEvents -> aggregate by hour -> hourly_consumption big-query - appendTransformations(dataTrafficInfoEvents) - .apply(convertToHourlyTableRows) - .apply(saveToBigQueryGroupedByHour) - - pipeline.run() - .waitUntilFinish() -} - -// This method has a part of pipeline which is independent of GCP PubSubIO and BigQueryIO. -// So, this part of the pipeline can be run locally and does not need GCP. -// This separation is done so that it can be tested using JUnit. -fun appendTransformations(inCollection: PCollection): PCollection { - - val linkTimestamps = WithTimestamps - .of { Instant(Timestamps.toMillis(it.timestamp)) } - .withAllowedTimestampSkew(Duration.standardMinutes(1L)) - - val groupByHour: Window = Window - .into(FixedWindows.of(Duration.standardHours(1L))) - .withAllowedLateness(Duration.standardMinutes(1L)) - .discardingFiredPanes() - - val toKeyValuePair = transform> { - val zonedDateTime = ZonedDateTime - .ofInstant(java.time.Instant.ofEpochMilli(Timestamps.toMillis(it.timestamp)), ZoneOffset.UTC) - .withMinute(0) - .withSecond(0) - .withNano(0) - val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:SS") - KV.of( - AggregatedDataTrafficInfo.newBuilder() - .setMsisdn(it.msisdn) - .setDateTime(formatter.format(zonedDateTime)) - .setDataBytes(0) - .build(), - it.bucketBytes) - } - - val reduceToSumOfBucketBytes = Combine.groupedValues(Sum.ofLongs()) - - val kvToSingleObject = transform, AggregatedDataTrafficInfo> { - AggregatedDataTrafficInfo.newBuilder() - .setMsisdn(it.key?.msisdn) - .setDateTime(it.key?.dateTime) - .setDataBytes(it.value) - .build() - } - - // In this method, the code above is declaring all transformations. - // Whereas the code below is chaining them into a pipeline. - - return inCollection - // In order to use timestamp in the event object instead of timestamp when event was registered to PubSub - .apply("linkTimestamps", linkTimestamps) - .apply("groupByHour", groupByHour) - // change to KV and then group by Key - .apply("toKeyValuePair", toKeyValuePair) - .setCoder(KvCoder.of(ProtoCoder.of(AggregatedDataTrafficInfo::class.java), VarLongCoder.of())) - .apply("groupByKey", GroupByKey.create()) - // sum for each group - .apply("reduceToSumOfBucketBytes", reduceToSumOfBucketBytes) - .apply("kvToSingleObject", kvToSingleObject) -} \ No newline at end of file diff --git a/analytics/Dockerfile b/dataflow-pipelines/Dockerfile similarity index 71% rename from analytics/Dockerfile rename to dataflow-pipelines/Dockerfile index a6604c571..6c8a80698 100644 --- a/analytics/Dockerfile +++ b/dataflow-pipelines/Dockerfile @@ -6,7 +6,7 @@ COPY script/start.sh /start.sh COPY config /config -COPY build/libs/analytics-uber.jar /analytics.jar +COPY build/libs/dataflow-pipelines-uber.jar /dataflow-pipelines.jar EXPOSE 8080 EXPOSE 8081 diff --git a/analytics/README.md b/dataflow-pipelines/README.md similarity index 89% rename from analytics/README.md rename to dataflow-pipelines/README.md index bf172e103..87c729277 100644 --- a/analytics/README.md +++ b/dataflow-pipelines/README.md @@ -15,4 +15,4 @@ With unit testing: ## Deploy to GCP - sudo docker-compose up --build \ No newline at end of file + docker-compose up --build \ No newline at end of file diff --git a/analytics/build.gradle b/dataflow-pipelines/build.gradle similarity index 92% rename from analytics/build.gradle rename to dataflow-pipelines/build.gradle index 00117c145..ea2eb9480 100644 --- a/analytics/build.gradle +++ b/dataflow-pipelines/build.gradle @@ -17,7 +17,7 @@ dependencies { } shadowJar { - mainClassName = 'org.ostelco.analytics.DataConsumptionPipelineKt' + mainClassName = 'org.ostelco.dataflow.pipelines.DeployPipelineKt' mergeServiceFiles() classifier = "uber" version = null diff --git a/analytics/config/.gitignore b/dataflow-pipelines/config/.gitignore similarity index 100% rename from analytics/config/.gitignore rename to dataflow-pipelines/config/.gitignore diff --git a/analytics/config/logback.xml b/dataflow-pipelines/config/logback.xml similarity index 100% rename from analytics/config/logback.xml rename to dataflow-pipelines/config/logback.xml diff --git a/analytics/docker-compose.yaml b/dataflow-pipelines/docker-compose.yaml similarity index 68% rename from analytics/docker-compose.yaml rename to dataflow-pipelines/docker-compose.yaml index 25e773775..c892a375e 100644 --- a/analytics/docker-compose.yaml +++ b/dataflow-pipelines/docker-compose.yaml @@ -1,8 +1,8 @@ -version: "3.6" +version: "3.7" services: analytics: - container_name: analytics + container_name: dataflow-pipelines build: . environment: - GOOGLE_APPLICATION_CREDENTIALS=/config/pantel-prod.json \ No newline at end of file diff --git a/analytics/script/start.sh b/dataflow-pipelines/script/start.sh similarity index 100% rename from analytics/script/start.sh rename to dataflow-pipelines/script/start.sh diff --git a/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/DeployPipeline.kt b/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/DeployPipeline.kt new file mode 100644 index 000000000..39491d5fa --- /dev/null +++ b/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/DeployPipeline.kt @@ -0,0 +1,63 @@ +package org.ostelco.dataflow.pipelines + +import ch.qos.logback.classic.util.ContextInitializer +import org.apache.beam.runners.dataflow.DataflowRunner +import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions +import org.apache.beam.sdk.Pipeline +import org.apache.beam.sdk.options.PipelineOptions +import org.apache.beam.sdk.options.PipelineOptionsFactory +import org.ostelco.dataflow.pipelines.definitions.DataConsumptionPipelineDefinition +import org.ostelco.dataflow.pipelines.definitions.DummyPipelineDefinition +import org.ostelco.dataflow.pipelines.definitions.PipelineDefinition + +enum class PipelineDefinitionRegistry(val pipelineDefinition: PipelineDefinition) { + DATA_CONSUMPTION(DataConsumptionPipelineDefinition), + DUMMY(DummyPipelineDefinition), +} + +fun main(args: Array) { + System.setProperty(ContextInitializer.CONFIG_FILE_PROPERTY, "config/logback.xml") + DeployPipeline().deploy(pipelineName = "DATA_CONSUMPTION") +} + +class DeployPipeline { + + private fun parseOptions(): PipelineOptions { + + // may be we need to pass options via command-line args + /* + val options = PipelineOptionsFactory + .fromArgs( + "--project=pantel-2decb", + "--runner=DataflowRunner", + "--stagingLocation=gs://data-traffic/staging/", + "--jobName=data-traffic") + .withValidation() + .create() + */ + + val options = PipelineOptionsFactory.`as`(DataflowPipelineOptions::class.java) + options.jobName = "data-traffic" + options.project = "pantel-2decb" + options.stagingLocation = "gs://data-traffic/staging/" + options.region = "europe-west1" + options.runner = DataflowRunner::class.java + options.isUpdate = true + + return options + } + + fun deploy(pipelineName: String) { + + val options = parseOptions() + + PipelineDefinitionRegistry + .valueOf(pipelineName) + .apply { + Pipeline.create(options) + .apply { pipelineDefinition.define(this) } + .run() + .waitUntilFinish() + } + } +} \ No newline at end of file diff --git a/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/definitions/DataConsumptionPipelineDefinition.kt b/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/definitions/DataConsumptionPipelineDefinition.kt new file mode 100644 index 000000000..34b0abf3c --- /dev/null +++ b/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/definitions/DataConsumptionPipelineDefinition.kt @@ -0,0 +1,123 @@ +package org.ostelco.dataflow.pipelines.definitions + +import com.google.protobuf.util.Timestamps +import org.apache.beam.sdk.Pipeline +import org.apache.beam.sdk.coders.KvCoder +import org.apache.beam.sdk.coders.VarLongCoder +import org.apache.beam.sdk.extensions.protobuf.ProtoCoder +import org.apache.beam.sdk.transforms.Combine +import org.apache.beam.sdk.transforms.Filter +import org.apache.beam.sdk.transforms.GroupByKey +import org.apache.beam.sdk.transforms.PTransform +import org.apache.beam.sdk.transforms.SerializableFunction +import org.apache.beam.sdk.transforms.Sum +import org.apache.beam.sdk.transforms.WithTimestamps +import org.apache.beam.sdk.transforms.windowing.FixedWindows +import org.apache.beam.sdk.transforms.windowing.Window +import org.apache.beam.sdk.values.KV +import org.apache.beam.sdk.values.PCollection +import org.joda.time.Duration +import org.joda.time.Instant +import org.ostelco.analytics.api.AggregatedDataTrafficInfo +import org.ostelco.analytics.api.DataTrafficInfo +import org.ostelco.dataflow.pipelines.dsl.ParDoFn +import org.ostelco.dataflow.pipelines.io.BigQueryIOUtils.saveToBigQuery +import org.ostelco.dataflow.pipelines.io.Table.HOURLY_CONSUMPTION +import org.ostelco.dataflow.pipelines.io.Table.RAW_CONSUMPTION +import org.ostelco.dataflow.pipelines.io.convertToHourlyTableRows +import org.ostelco.dataflow.pipelines.io.convertToRawTableRows +import org.ostelco.dataflow.pipelines.io.readFromPubSub +import java.time.ZoneOffset +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter + +object DataConsumptionPipelineDefinition : PipelineDefinition { + + override fun define(pipeline: Pipeline) { + + // Filter events with empty buckets + val filterEmptyBucketEvents = Filter.by(SerializableFunction { dataTrafficInfo: DataTrafficInfo -> + dataTrafficInfo.bucketBytes > 0 + }) + + // + // Construct pipeline chain + // + + + // First two common steps of pipeline, before it gets forked. + val dataTrafficInfoEvents = pipeline + .apply("readFromPubSub", readFromPubSub("data-traffic")) + .apply("filterEmptyBucketEvents", filterEmptyBucketEvents) + + // PubSubEvents -> raw_consumption big-query + dataTrafficInfoEvents + .apply("convertToRawTableRows", convertToRawTableRows) + .apply("saveRawEventsToBigQuery", saveToBigQuery(RAW_CONSUMPTION)) + + // PubSubEvents -> aggregate by hour -> hourly_consumption big-query + dataTrafficInfoEvents + .apply("TotalDataConsumptionGroupByMsisdn", consumptionPerMsisdn) + .apply("convertToHourlyTableRows", convertToHourlyTableRows) + .apply("saveToBigQueryGroupedByHour", saveToBigQuery(HOURLY_CONSUMPTION)) + } +} + +// This method has a part of pipeline which is independent of GCP PubSubIO and BigQueryIO. +// So, this part of the pipeline can be run locally and does not need GCP. +// This separation is done so that it can be tested using JUnit. +val consumptionPerMsisdn = object : PTransform, PCollection>() { + + override fun expand(inCollection: PCollection): PCollection { + + val linkTimestamps = WithTimestamps + .of { Instant(Timestamps.toMillis(it.timestamp)) } + .withAllowedTimestampSkew(Duration.standardMinutes(1L)) + + val groupByHour: Window = Window + .into(FixedWindows.of(Duration.standardHours(1L))) + .withAllowedLateness(Duration.standardMinutes(1L)) + .discardingFiredPanes() + + val toKeyValuePair = ParDoFn.transform> { + val zonedDateTime = ZonedDateTime + .ofInstant(java.time.Instant.ofEpochMilli(Timestamps.toMillis(it.timestamp)), ZoneOffset.UTC) + .withMinute(0) + .withSecond(0) + .withNano(0) + val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:SS") + KV.of( + AggregatedDataTrafficInfo.newBuilder() + .setMsisdn(it.msisdn) + .setDateTime(formatter.format(zonedDateTime)) + .setDataBytes(0) + .build(), + it.bucketBytes) + } + + val reduceToSumOfBucketBytes = Combine.groupedValues(Sum.ofLongs()) + + val kvToSingleObject = ParDoFn.transform, AggregatedDataTrafficInfo> { + AggregatedDataTrafficInfo.newBuilder() + .setMsisdn(it.key?.msisdn) + .setDateTime(it.key?.dateTime) + .setDataBytes(it.value) + .build() + } + + // In this method, the code above is declaring all transformations. + // Whereas the code below is chaining them into a pipeline. + + return inCollection + // In order to use timestamp in the event object instead of timestamp when event was registered to PubSub + .apply("linkTimestamps", linkTimestamps) + .apply("groupByHour", groupByHour) + // change to KV and then group by Key + .apply("toKeyValuePair", toKeyValuePair) + .setCoder(KvCoder.of(ProtoCoder.of(AggregatedDataTrafficInfo::class.java), VarLongCoder.of())) + .apply("groupByKey", GroupByKey.create()) + // sum for each group + .apply("reduceToSumOfBucketBytes", reduceToSumOfBucketBytes) + .apply("kvToSingleObject", kvToSingleObject) + } +} \ No newline at end of file diff --git a/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/definitions/DummyPipelineDefinition.kt b/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/definitions/DummyPipelineDefinition.kt new file mode 100644 index 000000000..bba3e0627 --- /dev/null +++ b/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/definitions/DummyPipelineDefinition.kt @@ -0,0 +1,9 @@ +package org.ostelco.dataflow.pipelines.definitions + +import org.apache.beam.sdk.Pipeline + +object DummyPipelineDefinition : PipelineDefinition { + override fun define(pipeline: Pipeline) { + TODO("not implemented") //To change body of created functions use File | Settings | File Templates. + } +} \ No newline at end of file diff --git a/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/definitions/PipelineDefinition.kt b/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/definitions/PipelineDefinition.kt new file mode 100644 index 000000000..2a69de643 --- /dev/null +++ b/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/definitions/PipelineDefinition.kt @@ -0,0 +1,7 @@ +package org.ostelco.dataflow.pipelines.definitions + +import org.apache.beam.sdk.Pipeline + +interface PipelineDefinition { + fun define(pipeline: Pipeline) +} \ No newline at end of file diff --git a/analytics/src/main/kotlin/org/ostelco/analytics/ParDoFn.kt b/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/dsl/ParDoFn.kt similarity index 91% rename from analytics/src/main/kotlin/org/ostelco/analytics/ParDoFn.kt rename to dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/dsl/ParDoFn.kt index 62bd22948..9cc0621bc 100644 --- a/analytics/src/main/kotlin/org/ostelco/analytics/ParDoFn.kt +++ b/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/dsl/ParDoFn.kt @@ -1,4 +1,4 @@ -package org.ostelco.analytics +package org.ostelco.dataflow.pipelines.dsl import org.apache.beam.sdk.transforms.DoFn import org.apache.beam.sdk.transforms.ParDo diff --git a/analytics/src/main/kotlin/org/ostelco/analytics/TableSchemas.kt b/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/io/BigQuery.kt similarity index 59% rename from analytics/src/main/kotlin/org/ostelco/analytics/TableSchemas.kt rename to dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/io/BigQuery.kt index d4bc19182..b65e8e4c7 100644 --- a/analytics/src/main/kotlin/org/ostelco/analytics/TableSchemas.kt +++ b/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/io/BigQuery.kt @@ -1,19 +1,26 @@ -package org.ostelco.analytics +package org.ostelco.dataflow.pipelines.io import com.google.api.services.bigquery.model.TableFieldSchema import com.google.api.services.bigquery.model.TableRow import com.google.api.services.bigquery.model.TableSchema +import com.google.protobuf.util.Timestamps import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO -import org.ostelco.analytics.Table.DAILY_CONSUMPTION -import org.ostelco.analytics.Table.HOURLY_CONSUMPTION -import org.ostelco.analytics.Table.RAW_CONSUMPTION +import org.ostelco.analytics.api.AggregatedDataTrafficInfo +import org.ostelco.analytics.api.DataTrafficInfo +import org.ostelco.dataflow.pipelines.dsl.ParDoFn +import org.ostelco.dataflow.pipelines.io.Table.DAILY_CONSUMPTION +import org.ostelco.dataflow.pipelines.io.Table.HOURLY_CONSUMPTION +import org.ostelco.dataflow.pipelines.io.Table.RAW_CONSUMPTION +import java.time.Instant +import java.time.ZoneOffset +import java.time.ZonedDateTime import java.util.* // This code is an attempt to keep all database schema in one place. // This may be moved to config. -const val project = "pantel-2decb" -const val dataset = "data_consumption" +private const val project = "pantel-2decb" +private const val dataset = "data_consumption" /** @@ -31,7 +38,7 @@ enum class Table { /** * Schemas for tables. */ -class TableSchemas { +private object TableSchemas { /** * Getting a table schema for the tables @@ -58,20 +65,47 @@ class TableSchemas { } } +// +// convert to BigQuery table rows +// +val convertToRawTableRows = ParDoFn.transform { + TableRow() + .set("msisdn", it.msisdn) + .set("bucketBytes", it.bucketBytes) + .set("bundleBytes", it.bundleBytes) + .set("timestamp", ZonedDateTime.ofInstant( + Instant.ofEpochMilli(Timestamps.toMillis(it.timestamp)), + ZoneOffset.UTC).toString()) +} + +val convertToHourlyTableRows = ParDoFn.transform { + TableRow() + .set("msisdn", it.msisdn) + .set("bytes", it.dataBytes) + .set("timestamp", it.dateTime) +} + +// +// Save to BigQuery Table +// + /** * Helpers for accessing BigTable */ -class BigQueryIOUtils { +object BigQueryIOUtils { /** * Create a [BigQueryIO.Write] query for writing all the * rows in a [Table] - denoted table. */ - fun writeTo(table: Table) : BigQueryIO.Write { + fun saveToBigQuery(table: Table): BigQueryIO.Write { return BigQueryIO.writeTableRows() .to("$project:$dataset.${table.name.toLowerCase()}") - .withSchema(TableSchemas().getTableSchema(table)) + .withSchema(TableSchemas.getTableSchema(table)) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_APPEND) } -} \ No newline at end of file +} + + + diff --git a/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/io/PubSub.kt b/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/io/PubSub.kt new file mode 100644 index 000000000..8353134e3 --- /dev/null +++ b/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/io/PubSub.kt @@ -0,0 +1,9 @@ +package org.ostelco.dataflow.pipelines.io + +import org.apache.beam.sdk.io.gcp.pubsub.PubsubIO +import org.ostelco.analytics.api.DataTrafficInfo + +// Read from PubSub +fun readFromPubSub(topic: String) = PubsubIO + .readProtos(DataTrafficInfo::class.java) + .fromSubscription("projects/pantel-2decb/subscriptions/$topic") diff --git a/analytics/src/test/kotlin/org/ostelco/analytics/PipelineTest.kt b/dataflow-pipelines/src/test/kotlin/org/ostelco/dataflow/pipelines/ConsumptionPerMsisdnTest.kt similarity index 93% rename from analytics/src/test/kotlin/org/ostelco/analytics/PipelineTest.kt rename to dataflow-pipelines/src/test/kotlin/org/ostelco/dataflow/pipelines/ConsumptionPerMsisdnTest.kt index 7f9787e5d..0b26d7bc7 100644 --- a/analytics/src/test/kotlin/org/ostelco/analytics/PipelineTest.kt +++ b/dataflow-pipelines/src/test/kotlin/org/ostelco/dataflow/pipelines/ConsumptionPerMsisdnTest.kt @@ -1,4 +1,4 @@ -package org.ostelco.analytics +package org.ostelco.dataflow.pipelines import com.google.protobuf.util.Timestamps import org.apache.beam.sdk.extensions.protobuf.ProtoCoder @@ -13,11 +13,12 @@ import org.junit.Test import org.junit.experimental.categories.Category import org.ostelco.analytics.api.AggregatedDataTrafficInfo import org.ostelco.analytics.api.DataTrafficInfo +import org.ostelco.dataflow.pipelines.definitions.consumptionPerMsisdn import java.time.ZoneOffset import java.time.ZonedDateTime import java.time.format.DateTimeFormatter -class PipelineTest { +class ConsumptionPerMsisdnTest { @Rule @Transient @@ -73,7 +74,9 @@ class PipelineTest { if (pipeline != null) { val currentHourDateTime = getCurrentHourDateTime() - val out: PCollection = appendTransformations(pipeline.apply(testStream)) + val out: PCollection = pipeline + .apply(testStream) + .apply(consumptionPerMsisdn) .setCoder(ProtoCoder.of(AggregatedDataTrafficInfo::class.java)) PAssert.that(out).containsInAnyOrder( diff --git a/analytics/src/test/resources/logback-test.xml b/dataflow-pipelines/src/test/resources/logback-test.xml similarity index 100% rename from analytics/src/test/resources/logback-test.xml rename to dataflow-pipelines/src/test/resources/logback-test.xml diff --git a/settings.gradle b/settings.gradle index b00b5c3ed..cc44e9b47 100644 --- a/settings.gradle +++ b/settings.gradle @@ -5,9 +5,9 @@ include ':app-notifier' include ':admin-api' include ':analytics-grpc-api' include ':analytics-module' -include ':analytics' include ':auth-server' include ':client-api' +include ':dataflow-pipelines' include ':diameter-stack' include ':diameter-test' include ':ext-auth-provider' @@ -30,9 +30,9 @@ project(':app-notifier').projectDir = "$rootDir/app-notifier" as File project(':admin-api').projectDir = "$rootDir/admin-api" as File project(':analytics-grpc-api').projectDir = "$rootDir/analytics-grpc-api" as File project(':analytics-module').projectDir = "$rootDir/analytics-module" as File -project(':analytics').projectDir = "$rootDir/analytics" as File project(':auth-server').projectDir = "$rootDir/auth-server" as File project(':client-api').projectDir = "$rootDir/client-api" as File +project(':dataflow-pipelines').projectDir = "$rootDir/dataflow-pipelines" as File project(':diameter-stack').projectDir = "$rootDir/diameter-stack" as File project(':diameter-test').projectDir = "$rootDir/diameter-test" as File project(':ext-auth-provider').projectDir = "$rootDir/ext-auth-provider" as File diff --git a/tools/neo4j-admin-tools/src/main/resources/docker-compose.yaml b/tools/neo4j-admin-tools/src/main/resources/docker-compose.yaml index b56fd3555..c3b46e20f 100644 --- a/tools/neo4j-admin-tools/src/main/resources/docker-compose.yaml +++ b/tools/neo4j-admin-tools/src/main/resources/docker-compose.yaml @@ -1,4 +1,4 @@ -version: "3.6" +version: "3.7" services: neo4j: From 89aaa26cb2c7f4ca711d3b93967ab83cd0bfd3c9 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Fri, 31 Aug 2018 05:30:49 +0200 Subject: [PATCH 30/30] Improvement: Pseudonymized msisdn before pushing to PubSub. Fix: Connect to datastore emulator when running in GCP via CI/CD. --- .../DataConsumptionInfoPublisher.kt | 15 ++- docker-compose.override.yaml | 1 - .../pseudonymizer/PseudonymizerService.kt | 4 + prime/build.gradle | 2 +- prime/config/test.yaml | 2 + .../org/ostelco/pseudonym/PseudonymModule.kt | 2 +- .../pseudonym/resources/PseudonymResource.kt | 6 +- .../{managed => service}/PseudonymExport.kt | 2 +- .../service/PseudonymizerServiceSingleton.kt | 125 +++++++++++------- .../pseudonym/PseudonymResourceTest.kt | 2 +- 10 files changed, 101 insertions(+), 60 deletions(-) rename pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/{managed => service}/PseudonymExport.kt (99%) diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DataConsumptionInfoPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DataConsumptionInfoPublisher.kt index feec66edc..674750198 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DataConsumptionInfoPublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DataConsumptionInfoPublisher.kt @@ -11,6 +11,8 @@ import io.dropwizard.lifecycle.Managed import org.ostelco.analytics.api.DataTrafficInfo import org.ostelco.prime.analytics.ConfigRegistry.config import org.ostelco.prime.logger +import org.ostelco.prime.module.getResource +import org.ostelco.prime.pseudonymizer.PseudonymizerService import java.io.IOException import java.time.Instant @@ -21,6 +23,8 @@ object DataConsumptionInfoPublisher : Managed { private val logger by logger() + private val pseudonymizerService by lazy { getResource() } + private lateinit var publisher: Publisher @Throws(IOException::class) @@ -40,11 +44,18 @@ object DataConsumptionInfoPublisher : Managed { fun publish(msisdn: String, usedBucketBytes: Long, bundleBytes: Long) { + if (usedBucketBytes == 0L) { + return + } + + val now = Instant.now().toEpochMilli() + val pseudonym = pseudonymizerService.getPseudonymEntityFor(msisdn, now).pseudonym + val data = DataTrafficInfo.newBuilder() - .setMsisdn(msisdn) + .setMsisdn(pseudonym) .setBucketBytes(usedBucketBytes) .setBundleBytes(bundleBytes) - .setTimestamp(Timestamps.fromMillis(Instant.now().toEpochMilli())) + .setTimestamp(Timestamps.fromMillis(now)) .build() .toByteString() diff --git a/docker-compose.override.yaml b/docker-compose.override.yaml index 2923d1a8d..9d810bf56 100644 --- a/docker-compose.override.yaml +++ b/docker-compose.override.yaml @@ -50,7 +50,6 @@ services: ipv4_address: 172.16.238.4 default: - metrics-esp: container_name: metrics-esp image: gcr.io/endpoints-release/endpoints-runtime:1 diff --git a/prime-api/src/main/kotlin/org/ostelco/prime/pseudonymizer/PseudonymizerService.kt b/prime-api/src/main/kotlin/org/ostelco/prime/pseudonymizer/PseudonymizerService.kt index 721d50c6f..f48b214c7 100644 --- a/prime-api/src/main/kotlin/org/ostelco/prime/pseudonymizer/PseudonymizerService.kt +++ b/prime-api/src/main/kotlin/org/ostelco/prime/pseudonymizer/PseudonymizerService.kt @@ -1,7 +1,11 @@ package org.ostelco.prime.pseudonymizer import org.ostelco.prime.model.ActivePseudonyms +import org.ostelco.prime.model.PseudonymEntity interface PseudonymizerService { + fun getActivePseudonymsForMsisdn(msisdn: String): ActivePseudonyms + + fun getPseudonymEntityFor(msisdn: String, timestamp: Long): PseudonymEntity } \ No newline at end of file diff --git a/prime/build.gradle b/prime/build.gradle index dd7b71803..c295eb5a3 100644 --- a/prime/build.gradle +++ b/prime/build.gradle @@ -18,7 +18,7 @@ sourceSets { } } -version = "1.12.0" +version = "1.13.0" repositories { maven { diff --git a/prime/config/test.yaml b/prime/config/test.yaml index 68e56c2d9..11def9a99 100644 --- a/prime/config/test.yaml +++ b/prime/config/test.yaml @@ -18,6 +18,8 @@ modules: config: lowBalanceThreshold: 0 - type: pseudonymizer + config: + datastoreType: emulator - type: api config: authenticationCachePolicy: maximumSize=10000, expireAfterAccess=10m diff --git a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/PseudonymModule.kt b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/PseudonymModule.kt index 762e270c7..2a79b6e80 100644 --- a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/PseudonymModule.kt +++ b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/PseudonymModule.kt @@ -17,7 +17,7 @@ class PseudonymModule : PrimeModule { } override fun init(env: Environment) { - PseudonymizerServiceSingleton.init() + PseudonymizerServiceSingleton.init(env = env) env.jersey().register(PseudonymResource()) } } diff --git a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/resources/PseudonymResource.kt b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/resources/PseudonymResource.kt index b32be3f0c..054bdabf2 100644 --- a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/resources/PseudonymResource.kt +++ b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/resources/PseudonymResource.kt @@ -2,8 +2,6 @@ package org.ostelco.pseudonym.resources import org.hibernate.validator.constraints.NotBlank import org.ostelco.pseudonym.service.PseudonymizerServiceSingleton -import org.ostelco.pseudonym.service.PseudonymizerServiceSingleton.getExportTask -import org.ostelco.pseudonym.service.PseudonymizerServiceSingleton.getPseudonymEntityFor import org.slf4j.LoggerFactory import java.time.Instant import javax.ws.rs.DELETE @@ -53,7 +51,7 @@ class PseudonymResource { fun getPseudonym(@NotBlank @PathParam("msisdn") msisdn: String): Response { val timestamp = Instant.now().toEpochMilli() logger.info("GET pseudonym for Msisdn = $msisdn at current time, timestamp = $timestamp") - val entity = getPseudonymEntityFor(msisdn, timestamp) + val entity = PseudonymizerServiceSingleton.getPseudonymEntityFor(msisdn, timestamp) return Response.ok(entity, MediaType.APPLICATION_JSON).build() } @@ -119,7 +117,7 @@ class PseudonymResource { @Path("/exportstatus/{exportId}") fun getExportStatus(@NotBlank @PathParam("exportId") exportId: String): Response { logger.info("GET status of export $exportId") - return getExportTask(exportId) + return PseudonymizerServiceSingleton.getExportTask(exportId) ?.let { Response.ok(it, MediaType.APPLICATION_JSON).build() } ?: Response.status(Status.NOT_FOUND).build() } diff --git a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/managed/PseudonymExport.kt b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/service/PseudonymExport.kt similarity index 99% rename from pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/managed/PseudonymExport.kt rename to pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/service/PseudonymExport.kt index 856464e80..e94c9872d 100644 --- a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/managed/PseudonymExport.kt +++ b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/service/PseudonymExport.kt @@ -1,4 +1,4 @@ -package org.ostelco.pseudonym.managed +package org.ostelco.pseudonym.service import com.google.cloud.bigquery.BigQuery import com.google.cloud.bigquery.Field diff --git a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/service/PseudonymizerServiceSingleton.kt b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/service/PseudonymizerServiceSingleton.kt index 6533c1b16..44509bb39 100644 --- a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/service/PseudonymizerServiceSingleton.kt +++ b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/service/PseudonymizerServiceSingleton.kt @@ -1,5 +1,6 @@ package org.ostelco.pseudonym.service +import com.codahale.metrics.health.HealthCheck import com.google.cloud.bigquery.BigQuery import com.google.cloud.bigquery.BigQueryOptions import com.google.cloud.datastore.Datastore @@ -9,7 +10,10 @@ import com.google.cloud.datastore.Key import com.google.cloud.datastore.Query import com.google.cloud.datastore.StructuredQuery.PropertyFilter import com.google.cloud.datastore.testing.LocalDatastoreHelper -import org.hibernate.validator.constraints.NotBlank +import com.google.cloud.http.HttpTransportOptions +import com.google.common.cache.Cache +import com.google.common.cache.CacheBuilder +import io.dropwizard.setup.Environment import org.ostelco.prime.logger import org.ostelco.prime.model.ActivePseudonyms import org.ostelco.prime.model.PseudonymEntity @@ -17,11 +21,9 @@ import org.ostelco.prime.pseudonymizer.PseudonymizerService import org.ostelco.pseudonym.ConfigRegistry import org.ostelco.pseudonym.ExportTaskKind import org.ostelco.pseudonym.PseudonymEntityKind -import org.ostelco.pseudonym.PseudonymServerConfig import org.ostelco.pseudonym.endPropertyName import org.ostelco.pseudonym.errorPropertyName import org.ostelco.pseudonym.exportIdPropertyName -import org.ostelco.pseudonym.managed.PseudonymExport import org.ostelco.pseudonym.msisdnPropertyName import org.ostelco.pseudonym.pseudonymPropertyName import org.ostelco.pseudonym.resources.ExportTask @@ -62,21 +64,24 @@ object PseudonymizerServiceSingleton : PseudonymizerService { private val logger by logger() private lateinit var datastore: Datastore - private lateinit var bigquery: BigQuery + private var bigQuery: BigQuery? = null private val dateBounds: DateBounds = WeeklyBounds() private val executor = Executors.newFixedThreadPool(3) - fun init(bq: BigQuery? = null) { - datastore = getDatastore(ConfigRegistry.config) - if (bq != null) { - bigquery = bq + val pseudonymCache: Cache = CacheBuilder.newBuilder() + .maximumSize(5000) + .build() + + fun init(env: Environment?, bq: BigQuery? = null) { + + initDatastore(env) + + bigQuery = bq ?: if (System.getenv("LOCAL_TESTING") != "true") { + BigQueryOptions.getDefaultInstance().service } else { - if (System.getenv("LOCAL_TESTING") != "true") { - bigquery = BigQueryOptions.getDefaultInstance().service - } else { - logger.info("Local testing, BigQuery is not available...") - } + logger.info("Local testing, BigQuery is not available...") + null } } @@ -89,6 +94,14 @@ object PseudonymizerServiceSingleton : PseudonymizerService { return ActivePseudonyms(current, next) } + override fun getPseudonymEntityFor(msisdn: String, timestamp: Long): PseudonymEntity { + val (bounds, keyPrefix) = dateBounds.getBoundsNKeyPrefix(msisdn, timestamp) + // Retrieves the element from cache. + return pseudonymCache.get(keyPrefix) { + getPseudonymEntity(keyPrefix) ?: createPseudonym(msisdn, bounds, keyPrefix) + } + } + fun findPseudonym(pseudonym: String): PseudonymEntity? { val query = Query.newEntityQueryBuilder() .setKind(PseudonymEntityKind) @@ -121,37 +134,60 @@ object PseudonymizerServiceSingleton : PseudonymizerService { } fun exportPseudonyms(exportId: String) { - logger.info("GET export all pseudonyms to the table $exportId") - val exporter = PseudonymExport(exportId, bigquery, datastore) - executor.execute(exporter.getRunnable()) + bigQuery?.apply { + logger.info("GET export all pseudonyms to the table $exportId") + val exporter = PseudonymExport(exportId = exportId, bigquery = this, datastore = datastore) + executor.execute(exporter.getRunnable()) + } } // Integration testing helper for Datastore. - private fun getDatastore(config: PseudonymServerConfig): Datastore { - val datastore: Datastore? - if (config.datastoreType == "inmemory-emulator") { - logger.info("Starting with in-memory datastore emulator...") - val helper: LocalDatastoreHelper = LocalDatastoreHelper.create(1.0) - helper.start() - datastore = helper.options.service - } else { - datastore = DatastoreOptions.getDefaultInstance().service - logger.info("Created default instance of datastore client") - - // TODO vihang: make this part of health-check - val testKey = datastore.newKeyFactory().setKind("TestKind").newKey("testKey") - val testPropertyKey = "testPropertyKey" - val testPropertyValue = "testPropertyValue" - val testEntity = Entity.newBuilder(testKey).set(testPropertyKey, testPropertyValue).build() - datastore.put(testEntity) - val value = datastore.get(testKey).getString(testPropertyKey) - if (testPropertyValue != value) { - logger.warn("Unable to fetch test property value from datastore") + private fun initDatastore(env: Environment?) { + datastore = when (ConfigRegistry.config.datastoreType) { + "inmemory-emulator" -> { + logger.info("Starting with in-memory datastore emulator") + val helper: LocalDatastoreHelper = LocalDatastoreHelper.create(1.0) + helper.start() + helper.options } - datastore.delete(testKey) - // END - } - return datastore + "emulator" -> { + // When prime running in GCP by hosted CI/CD, Datastore client library assumes it is running in + // production and ignore our instruction to connect to the datastore emulator. So, we are explicitly + // connecting to emulator + logger.info("Connecting to datastore emulator") + DatastoreOptions + .newBuilder() + .setHost("localhost:9090") + .setTransportOptions(HttpTransportOptions.newBuilder().build()) + .build() + } + else -> { + logger.info("Created default instance of datastore client") + DatastoreOptions.getDefaultInstance() + } + }.service + + // health-check for datastore + env?.healthChecks()?.register("datastore", object : HealthCheck() { + override fun check(): Result { + try { + val testKey = datastore.newKeyFactory().setKind("TestKind").newKey("testKey") + val testPropertyKey = "testPropertyKey" + val testPropertyValue = "testPropertyValue" + val testEntity = Entity.newBuilder(testKey).set(testPropertyKey, testPropertyValue).build() + datastore.put(testEntity) + val value = datastore.get(testKey).getString(testPropertyKey) + datastore.delete(testKey) + if (testPropertyValue != value) { + logger.warn("Unable to fetch test property value from datastore") + return Result.builder().unhealthy().build() + } + return Result.builder().healthy().build() + } catch (e: Exception) { + return Result.builder().unhealthy(e).build() + } + } + }) } fun getExportTask(exportId: String): ExportTask? { @@ -181,15 +217,6 @@ object PseudonymizerServiceSingleton : PseudonymizerService { return null } - fun getPseudonymEntityFor(@NotBlank msisdn: String, timestamp: Long): PseudonymEntity { - val (bounds, keyPrefix) = dateBounds.getBoundsNKeyPrefix(msisdn, timestamp) - var entity = getPseudonymEntity(keyPrefix) - if (entity == null) { - entity = createPseudonym(msisdn, bounds, keyPrefix) - } - return entity - } - private fun createPseudonym(msisdn: String, bounds: Bounds, keyPrefix: String): PseudonymEntity { val uuid = UUID.randomUUID().toString() var entity = PseudonymEntity(msisdn, uuid, bounds.start, bounds.end) diff --git a/pseudonym-server/src/test/kotlin/org/ostelco/pseudonym/PseudonymResourceTest.kt b/pseudonym-server/src/test/kotlin/org/ostelco/pseudonym/PseudonymResourceTest.kt index a217f19ff..7c9d6ce98 100644 --- a/pseudonym-server/src/test/kotlin/org/ostelco/pseudonym/PseudonymResourceTest.kt +++ b/pseudonym-server/src/test/kotlin/org/ostelco/pseudonym/PseudonymResourceTest.kt @@ -34,7 +34,7 @@ class PseudonymResourceTest { init { ConfigRegistry.config = PseudonymServerConfig() .apply { this.datastoreType = "inmemory-emulator" } - PseudonymizerServiceSingleton.init(mock(BigQuery::class.java)) + PseudonymizerServiceSingleton.init(env = null, bq = mock(BigQuery::class.java)) } @ClassRule