From 7e012f6296a6311e930329cf548ae517a529e2d6 Mon Sep 17 00:00:00 2001 From: Martin Cederlof Date: Thu, 20 Sep 2018 10:58:28 +0200 Subject: [PATCH 01/85] WIP new analytics events for active users --- .../src/main/proto/prime_metrics.proto | 7 ++ .../prime/analytics/AnalyticsGrpcService.kt | 4 +- .../prime/analytics/AnalyticsModule.kt | 4 + .../publishers/ActiveUsersPublisher.kt | 91 +++++++++++++++++++ .../java/org/ostelco/ocsgw/OcsServer.java | 1 - .../ocsgw/data/grpc/GrpcDataSource.java | 15 ++- .../ostelco/ocsgw/data/grpc/OcsgwMetrics.java | 8 +- prime/config/config.yaml | 1 + prime/config/test.yaml | 1 + .../integration-tests/resources/config.yaml | 1 + 10 files changed, 126 insertions(+), 7 deletions(-) create mode 100644 analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt diff --git a/analytics-grpc-api/src/main/proto/prime_metrics.proto b/analytics-grpc-api/src/main/proto/prime_metrics.proto index 70dbf355a..f7c8ac851 100644 --- a/analytics-grpc-api/src/main/proto/prime_metrics.proto +++ b/analytics-grpc-api/src/main/proto/prime_metrics.proto @@ -13,6 +13,13 @@ service OcsgwAnalyticsService { message OcsgwAnalyticsReport { uint32 activeSessions = 1; + repeated User users = 2; +} + +message User { + string msisdn = 1; + string apn = 2; + string mnc_mcc = 3; } message OcsgwAnalyticsReply { diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt index 9cf9b7535..d6a574f01 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt @@ -3,6 +3,7 @@ package org.ostelco.prime.analytics import io.grpc.stub.StreamObserver import org.ostelco.prime.analytics.PrimeMetric.ACTIVE_SESSIONS import org.ostelco.prime.analytics.metrics.CustomMetricsRegistry +import org.ostelco.prime.analytics.publishers.ActiveUsersPublisher import org.ostelco.prime.logger import org.ostelco.prime.metrics.api.OcsgwAnalyticsReply import org.ostelco.prime.metrics.api.OcsgwAnalyticsReport @@ -11,7 +12,7 @@ import java.util.* /** - * Serves incoming GRPC analytcs requests. + * Serves incoming GRPC analytics requests. * * It's implemented as a subclass of [OcsServiceGrpc.OcsServiceImplBase] overriding * methods that together implements the protocol described in the analytics protobuf @@ -52,6 +53,7 @@ class AnalyticsGrpcService : OcsgwAnalyticsServiceGrpc.OcsgwAnalyticsServiceImpl */ override fun onNext(request: OcsgwAnalyticsReport) { CustomMetricsRegistry.updateMetricValue(ACTIVE_SESSIONS, request.activeSessions.toLong()) + ActiveUsersPublisher.publish(request.usersList) } override fun onError(t: Throwable) { diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt index 1f0c7d0b8..7ffa00c00 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt @@ -43,6 +43,10 @@ class AnalyticsConfig { @NotEmpty @JsonProperty("purchaseInfoTopicId") lateinit var purchaseInfoTopicId: String + + @NotEmpty + @JsonProperty("activeUsersTopicId") + lateinit var activeUsersTopicId: String } object ConfigRegistry { diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt new file mode 100644 index 000000000..0e07f68c5 --- /dev/null +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt @@ -0,0 +1,91 @@ +package org.ostelco.prime.analytics.publishers + +import com.google.api.core.ApiFutureCallback +import com.google.api.core.ApiFutures +import com.google.api.gax.rpc.ApiException +import com.google.gson.Gson +import com.google.gson.GsonBuilder +import com.google.gson.JsonArray +import com.google.gson.JsonObject +import com.google.gson.JsonSerializer +import com.google.gson.reflect.TypeToken +import com.google.protobuf.ByteString +import com.google.pubsub.v1.PubsubMessage +import org.ostelco.prime.analytics.ConfigRegistry +import org.ostelco.prime.logger +import org.ostelco.prime.metrics.api.User +import org.ostelco.prime.model.PurchaseRecord +import org.ostelco.prime.model.PurchaseRecordInfo +import org.ostelco.prime.module.getResource +import org.ostelco.prime.pseudonymizer.PseudonymizerService +import java.net.URLEncoder + + +/** + * This class publishes the active users information events to the Google Cloud Pub/Sub. + */ +object ActiveUsersPublisher : + PubSubPublisher by DelegatePubSubPublisher(topicId = ConfigRegistry.config.activeUsersTopicId) { + + private val logger by logger() + + private val pseudonymizerService by lazy { getResource() } + + private var gson: Gson = createGson() + + private fun createGson(): Gson { + val builder = GsonBuilder() + // Type for this conversion is explicitly set to java.util.Map + // This is needed because of kotlin's own Map interface + val mapType = object : TypeToken>() {}.type + val serializer = JsonSerializer> { src, _, _ -> + val array = JsonArray() + src.forEach { k, v -> + val property = JsonObject() + property.addProperty("key", k) + property.addProperty("value", v) + array.add(property) + } + array + } + builder.registerTypeAdapter(mapType, serializer) + return builder.create() + } + + private fun convertToJson(purchaseRecordInfo: PurchaseRecordInfo): ByteString = + ByteString.copyFromUtf8(gson.toJson(purchaseRecordInfo)) + + + fun publish(userList: List) { + for (user in userList) { + val encodedSubscriberId = URLEncoder.encode(user.msisdn, "UTF-8") + val pseudonym = pseudonymizerService.getSubscriberIdPseudonym(encodedSubscriberId, purchaseRecord.timestamp).pseudonym + } + + + val pubsubMessage = PubsubMessage.newBuilder() + .setData(convertToJson(PurchaseRecordInfo(purchaseRecord, pseudonym, status))) + .build() + + //schedule a message to be published, messages are automatically batched + val future = publishPubSubMessage(pubsubMessage) + + // add an asynchronous callback to handle success / failure + ApiFutures.addCallback(future, object : ApiFutureCallback { + + override fun onFailure(throwable: Throwable) { + if (throwable is ApiException) { + // details on the API exception + logger.warn("Status code: {}", throwable.statusCode.code) + logger.warn("Retrying: {}", throwable.isRetryable) + } + logger.warn("Error publishing purchase record for msisdn: {}", purchaseRecord.msisdn) + } + + override fun onSuccess(messageId: String) { + // Once published, returns server-assigned message ids (unique within the topic) + logger.debug(messageId) + } + }, singleThreadScheduledExecutor) + } +} diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/OcsServer.java b/ocsgw/src/main/java/org/ostelco/ocsgw/OcsServer.java index e3b4c684a..6a757aa29 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/OcsServer.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/OcsServer.java @@ -27,7 +27,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.naming.ConfigurationException; import java.io.IOException; diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java index 1b615aa1b..e8154eb65 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java @@ -25,6 +25,8 @@ import org.ostelco.ocs.api.*; import org.ostelco.ocsgw.OcsServer; import org.ostelco.ocsgw.data.DataSource; +import org.ostelco.prime.metrics.api.OcsgwAnalyticsReport; +import org.ostelco.prime.metrics.api.User; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -281,7 +283,18 @@ private void removeFromSessionMap(CreditControlContext creditControlContext) { private void updateAnalytics() { LOG.info("Number of active sesssions is {}", sessionIdMap.size()); - ocsgwAnalytics.sendAnalytics(sessionIdMap.size()); + + OcsgwAnalyticsReport.Builder builder = OcsgwAnalyticsReport.newBuilder().setActiveSessions(sessionIdMap.size()); + sessionIdMap.forEach((msisdn, sessionContext) -> { + try { + String apn = ccrMap.get(msisdn).getCreditControlRequest().getServiceInformation().get(0).getPsInformation().get(0).getCalledStationId(); + String mncMcc = ccrMap.get(msisdn).getCreditControlRequest().getServiceInformation().get(0).getPsInformation().get(0).getSgsnMncMcc(); + builder.addUsers(User.newBuilder().setApn(apn).setMncMcc(mncMcc).build()); + } catch (Exception e) { + LOG.info("Failed to match session info to ccr map"); + } + }); + ocsgwAnalytics.sendAnalytics(builder.build()); } private void initActivate() { diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java index 3b863e8dd..0e3790626 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java @@ -42,7 +42,7 @@ class OcsgwMetrics { private ScheduledFuture keepAliveFuture = null; - private int lastActiveSessions = 0; + private OcsgwAnalyticsReport lastActiveSessions = OcsgwAnalyticsReport.newBuilder().build(); OcsgwMetrics(String metricsServerHostname, ServiceAccountJwtAccessCredentials credentials) { @@ -133,8 +133,8 @@ private void initKeepAlive() { TimeUnit.SECONDS); } - void sendAnalytics(int size) { - ocsgwAnalyticsReport.onNext(OcsgwAnalyticsReport.newBuilder().setActiveSessions(size).build()); - lastActiveSessions = size; + void sendAnalytics(OcsgwAnalyticsReport report) { + ocsgwAnalyticsReport.onNext(report); + lastActiveSessions = report; } } \ No newline at end of file diff --git a/prime/config/config.yaml b/prime/config/config.yaml index 40c463c04..13c0bb8a0 100644 --- a/prime/config/config.yaml +++ b/prime/config/config.yaml @@ -12,6 +12,7 @@ modules: projectId: pantel-2decb dataTrafficTopicId: data-traffic purchaseInfoTopicId: purchase-info + activeUsersTopicId: active-users - type: ocs config: lowBalanceThreshold: 100000000 diff --git a/prime/config/test.yaml b/prime/config/test.yaml index a178eb66f..57ffe6db3 100644 --- a/prime/config/test.yaml +++ b/prime/config/test.yaml @@ -14,6 +14,7 @@ modules: projectId: pantel-2decb dataTrafficTopicId: data-traffic purchaseInfoTopicId: purchase-info + activeUsersTopicId: active-users - type: ocs config: lowBalanceThreshold: 0 diff --git a/prime/src/integration-tests/resources/config.yaml b/prime/src/integration-tests/resources/config.yaml index 123175946..dce9318e8 100644 --- a/prime/src/integration-tests/resources/config.yaml +++ b/prime/src/integration-tests/resources/config.yaml @@ -12,6 +12,7 @@ modules: projectId: pantel-2decb dataTrafficTopicId: data-traffic purchaseInfoTopicId: purchase-info + activeUsersTopicId: active-users - type: ocs config: lowBalanceThreshold: 0 From 23dda27a42e7d404773414f20a383e3917efd7c3 Mon Sep 17 00:00:00 2001 From: Martin Cederlof Date: Thu, 20 Sep 2018 14:17:35 +0200 Subject: [PATCH 02/85] Wip Active users --- .../src/main/proto/analytics.proto | 11 ++++++++++ .../publishers/ActiveUsersPublisher.kt | 20 +++++++++++-------- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/analytics-grpc-api/src/main/proto/analytics.proto b/analytics-grpc-api/src/main/proto/analytics.proto index f2d68aa83..a67cdf479 100644 --- a/analytics-grpc-api/src/main/proto/analytics.proto +++ b/analytics-grpc-api/src/main/proto/analytics.proto @@ -20,4 +20,15 @@ message AggregatedDataTrafficInfo { string msisdn = 1; uint64 dataBytes = 2; google.protobuf.Timestamp timestamp = 3; +} + +message User { + string msisdn = 1; + string apn = 2; + string mnc_mcc = 3; +} + +message ActiveUsersInfo { + repeated User users = 1; + google.protobuf.Timestamp timestamp = 2; } \ No newline at end of file diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt index 0e07f68c5..cc9a98258 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt @@ -10,16 +10,16 @@ import com.google.gson.JsonObject import com.google.gson.JsonSerializer import com.google.gson.reflect.TypeToken import com.google.protobuf.ByteString +import com.google.protobuf.util.Timestamps import com.google.pubsub.v1.PubsubMessage +import org.ostelco.analytics.api.ActiveUsersInfo import org.ostelco.prime.analytics.ConfigRegistry import org.ostelco.prime.logger import org.ostelco.prime.metrics.api.User -import org.ostelco.prime.model.PurchaseRecord -import org.ostelco.prime.model.PurchaseRecordInfo import org.ostelco.prime.module.getResource import org.ostelco.prime.pseudonymizer.PseudonymizerService import java.net.URLEncoder - +import java.time.Instant /** * This class publishes the active users information events to the Google Cloud Pub/Sub. @@ -52,19 +52,23 @@ object ActiveUsersPublisher : return builder.create() } - private fun convertToJson(purchaseRecordInfo: PurchaseRecordInfo): ByteString = - ByteString.copyFromUtf8(gson.toJson(purchaseRecordInfo)) + private fun convertToJson(activeUsersInfo: ActiveUsersInfo): ByteString = + ByteString.copyFromUtf8(gson.toJson(activeUsersInfo)) fun publish(userList: List) { + val activeUsersInfoBuilder = ActiveUsersInfo.newBuilder().setTimestamp(Timestamps.fromMillis(Instant.now().toEpochMilli())) + val timestamp = Instant.now().toEpochMilli() for (user in userList) { + val userBuilder = org.ostelco.analytics.api.User.newBuilder() val encodedSubscriberId = URLEncoder.encode(user.msisdn, "UTF-8") - val pseudonym = pseudonymizerService.getSubscriberIdPseudonym(encodedSubscriberId, purchaseRecord.timestamp).pseudonym + val pseudonym = pseudonymizerService.getSubscriberIdPseudonym(encodedSubscriberId, timestamp).pseudonym + activeUsersInfoBuilder.addUsers(userBuilder.setApn(user.apn).setMncMcc(user.mncMcc).setMsisdn(pseudonym).build()) } val pubsubMessage = PubsubMessage.newBuilder() - .setData(convertToJson(PurchaseRecordInfo(purchaseRecord, pseudonym, status))) + .setData(convertToJson(activeUsersInfoBuilder.build())) .build() //schedule a message to be published, messages are automatically batched @@ -79,7 +83,7 @@ object ActiveUsersPublisher : logger.warn("Status code: {}", throwable.statusCode.code) logger.warn("Retrying: {}", throwable.isRetryable) } - logger.warn("Error publishing purchase record for msisdn: {}", purchaseRecord.msisdn) + logger.warn("Error publishing active users list") } override fun onSuccess(messageId: String) { From 5af74a29d45e19eac209b11fc343b086725f4aae Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Thu, 20 Sep 2018 15:41:11 +0200 Subject: [PATCH 03/85] Adds support for default segment per country (WIP) --- .../kotlin/org/ostelco/at/common/TestUser.kt | 2 +- .../kotlin/org/ostelco/at/jersey/Tests.kt | 25 ++++++++++++------- .../kotlin/org/ostelco/at/okhttp/Tests.kt | 17 +++++++------ .../prime/client/api/store/SubscriberDAO.kt | 3 ++- .../prime/storage/graph/Neo4jModule.kt | 9 ++++--- .../ostelco/prime/storage/graph/Neo4jStore.kt | 2 +- .../prime/storage/graph/GraphStoreTest.kt | 15 +++++------ .../prime/storage/graph/Neo4jStorageTest.kt | 3 ++- 8 files changed, 46 insertions(+), 30 deletions(-) diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/common/TestUser.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/common/TestUser.kt index d55a70598..3a278d77b 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/common/TestUser.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/common/TestUser.kt @@ -12,7 +12,7 @@ fun createProfile(name: String, email: String) { .name(name) .address("") .city("") - .country("") + .country("NO") .postCode("") .referralId("") diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt index 8f1c28711..c96fe09fe 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt @@ -19,14 +19,10 @@ import org.ostelco.prime.client.model.Profile import org.ostelco.prime.client.model.PurchaseRecordList import org.ostelco.prime.client.model.Subscription import org.ostelco.prime.client.model.SubscriptionStatus +import java.lang.AssertionError import java.time.Instant import java.util.* -import kotlin.test.assertEquals -import kotlin.test.assertFails -import kotlin.test.assertFalse -import kotlin.test.assertNotNull -import kotlin.test.assertNull -import kotlin.test.assertTrue +import kotlin.test.* class ProfileTest { @@ -41,7 +37,7 @@ class ProfileTest { .name("Test Profile User") .address("") .city("") - .country("") + .country("NO") .postCode("") .referralId("") @@ -87,7 +83,6 @@ class ProfileTest { .address("") .postCode("") .city("") - .country("") val clearedProfile: Profile = put { path = "/profile" @@ -100,7 +95,19 @@ class ProfileTest { assertEquals("", clearedProfile.address, "Incorrect 'address' in response after clearing profile") assertEquals("", clearedProfile.postCode, "Incorrect 'postcode' in response after clearing profile") assertEquals("", clearedProfile.city, "Incorrect 'city' in response after clearing profile") - assertEquals("", clearedProfile.country, "Incorrect 'country' in response after clearing profile") + + updatedProfile + .country("") + + // A test in 'HttpClientUtil' checks for status code 200 while the + // expected status code is actually 400. + assertFailsWith(AssertionError::class, "Incorrectly accepts that 'country' is cleared/not set") { + put { + path = "/profile" + body = updatedProfile + subscriberId = email + } + } } @Test diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt index f2a84491b..a7c54a5b4 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt @@ -9,6 +9,7 @@ import org.ostelco.at.common.logger import org.ostelco.at.common.randomInt import org.ostelco.at.okhttp.ClientFactory.clientForSubject import org.ostelco.prime.client.api.DefaultApi +import org.ostelco.prime.client.ApiException import org.ostelco.prime.client.model.ApplicationToken import org.ostelco.prime.client.model.Consent import org.ostelco.prime.client.model.PaymentSource @@ -20,10 +21,7 @@ import org.ostelco.prime.client.model.Profile import org.ostelco.prime.client.model.SubscriptionStatus import java.time.Instant import java.util.* -import kotlin.test.assertEquals -import kotlin.test.assertFails -import kotlin.test.assertNotNull -import kotlin.test.assertNull +import kotlin.test.* class ProfileTest { @@ -39,7 +37,7 @@ class ProfileTest { .name("Test Profile User") .address("") .city("") - .country("") + .country("NO") .postCode("") .referralId("") @@ -70,7 +68,6 @@ class ProfileTest { .address("") .postCode("") .city("") - .country("") val clearedProfile: Profile = client.updateProfile(updatedProfile) @@ -79,7 +76,13 @@ class ProfileTest { assertEquals("", clearedProfile.address, "Incorrect 'address' in response after clearing profile") assertEquals("", clearedProfile.postCode, "Incorrect 'postcode' in response after clearing profile") assertEquals("", clearedProfile.city, "Incorrect 'city' in response after clearing profile") - assertEquals("", clearedProfile.country, "Incorrect 'country' in response after clearing profile") + + updatedProfile + .country("") + + assertFailsWith(ApiException::class, "Incorrectly accepts that 'country' is cleared/not set") { + client.updateProfile(updatedProfile) + } } @Test diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAO.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAO.kt index ba98ee75c..89349346f 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAO.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAO.kt @@ -70,7 +70,8 @@ interface SubscriberDAO { fun isValidProfile(profile: Subscriber?): Boolean { return (profile != null && !profile.name.isEmpty() - && !profile.email.isEmpty()) + && !profile.email.isEmpty() + && !profile.country.isEmpty()) } /** diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt index d9c3ea8ce..b731e6189 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt @@ -51,12 +51,15 @@ fun initDatabase() { price = Price(0, "NOK"), properties = mapOf("noOfBytes" to "1_000_000_000"))) - val segment = Segment(id = "all") - Neo4jStoreSingleton.createSegment(segment) + val segments = listOf( + Segment(id = "no"), + Segment(id = "sg") + ) + segments.map { Neo4jStoreSingleton.createSegment(it) } val offer = Offer( id = "default_offer", - segments = listOf("all"), + segments = listOf("no"), products = listOf("1GB_249NOK", "2GB_299NOK", "3GB_349NOK", "5GB_399NOK")) Neo4jStoreSingleton.createOffer(offer) } diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index d669d596e..dcc8a36f3 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -155,6 +155,7 @@ object Neo4jStoreSingleton : GraphStore { val bundleId = subscriber.id val either = subscriberStore.create(subscriber, transaction) + .flatMap { subscriberToSegmentStore.create(subscriber.id, subscriber.country.toLowerCase(), transaction) } if (referredBy != null) { // Give 1 GB if subscriber is referred either @@ -193,7 +194,6 @@ object Neo4jStoreSingleton : GraphStore { Either.right(Unit) } }.flatMap { subscriberToBundleStore.create(subscriber.id, bundleId, transaction) } - .flatMap { subscriberToSegmentStore.create(subscriber.id, "all", transaction) } .ifFailedThenRollback(transaction) } // << END diff --git a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt index f7c5d439f..89e8d6dd9 100644 --- a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt +++ b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt @@ -48,19 +48,19 @@ class GraphStoreTest { price = Price(0, "NOK"), properties = mapOf("noOfBytes" to "1_000_000_000"))) - val allSegment = Segment(id = "all") + val allSegment = Segment(id = "no") Neo4jStoreSingleton.createSegment(allSegment) } @Test fun `add subscriber`() { - Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME), referredBy = null) + Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME, country = COUNTRY), referredBy = null) .mapLeft { fail(it.message) } Neo4jStoreSingleton.getSubscriber(EMAIL).bimap( { fail(it.message) }, - { assertEquals(Subscriber(email = EMAIL, name = NAME, referralId = EMAIL), it) }) + { assertEquals(Subscriber(email = EMAIL, name = NAME, referralId = EMAIL, country = COUNTRY), it) }) // TODO vihang: fix argument captor for neo4j-store tests // val bundleArgCaptor: ArgumentCaptor = ArgumentCaptor.forClass(Bundle::class.java) @@ -71,7 +71,7 @@ class GraphStoreTest { @Test fun `fail to add subscriber with invalid referred by`() { - Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME), referredBy = "blah") + Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME, country = COUNTRY), referredBy = "blah") .fold({ assertEquals( expected = "Failed to create REFERRED - blah -> foo@bar.com", @@ -83,7 +83,7 @@ class GraphStoreTest { @Test fun `add subscription`() { - Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME), referredBy = null) + Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME, country = COUNTRY), referredBy = null) .mapLeft { fail(it.message) } Neo4jStoreSingleton.addSubscription(EMAIL, MSISDN) @@ -107,7 +107,7 @@ class GraphStoreTest { @Test fun `set and get Purchase record`() { - assert(Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME), referredBy = null).isRight()) + assert(Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME, country = COUNTRY), referredBy = null).isRight()) val product = createProduct("1GB_249NOK", 24900) val now = Instant.now().toEpochMilli() @@ -129,7 +129,7 @@ class GraphStoreTest { @Test fun `create products, offer, segment and then get products for a subscriber`() { - assert(Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME), referredBy = null).isRight()) + assert(Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME, country = COUNTRY), referredBy = null).isRight()) Neo4jStoreSingleton.createProduct(createProduct("1GB_249NOK", 24900)) Neo4jStoreSingleton.createProduct(createProduct("2GB_299NOK", 29900)) @@ -162,6 +162,7 @@ class GraphStoreTest { companion object { const val EMAIL = "foo@bar.com" const val NAME = "Test User" + const val COUNTRY = "NO" const val MSISDN = "4712345678" @ClassRule diff --git a/prime/src/integration-tests/kotlin/org/ostelco/prime/storage/graph/Neo4jStorageTest.kt b/prime/src/integration-tests/kotlin/org/ostelco/prime/storage/graph/Neo4jStorageTest.kt index 9685a2420..62e771103 100644 --- a/prime/src/integration-tests/kotlin/org/ostelco/prime/storage/graph/Neo4jStorageTest.kt +++ b/prime/src/integration-tests/kotlin/org/ostelco/prime/storage/graph/Neo4jStorageTest.kt @@ -37,7 +37,7 @@ class Neo4jStorageTest { sleep(MILLIS_TO_WAIT_WHEN_STARTING_UP.toLong()) storage.removeSubscriber(EPHERMERAL_EMAIL) - storage.addSubscriber(Subscriber(EPHERMERAL_EMAIL), referredBy = null) + storage.addSubscriber(Subscriber(EPHERMERAL_EMAIL, country = COUNTRY), referredBy = null) .mapLeft { fail(it.message) } storage.addSubscription(EPHERMERAL_EMAIL, MSISDN) .mapLeft { fail(it.message) } @@ -91,6 +91,7 @@ class Neo4jStorageTest { private const val EPHERMERAL_EMAIL = "attherate@dotcom.com" private const val MSISDN = "4747116996" + private const val COUNTRY = "NO" private const val MILLIS_TO_WAIT_WHEN_STARTING_UP = 3000 From c04cea4884f07a545995bbf658b2004a26f71fcf Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Fri, 21 Sep 2018 14:21:21 +0200 Subject: [PATCH 04/85] Fixes to 'add support for default segment per country' (from code review) --- .../kotlin/org/ostelco/at/jersey/Tests.kt | 3 +- .../kotlin/org/ostelco/at/okhttp/Tests.kt | 3 +- .../prime/storage/graph/Neo4jModule.kt | 6 ++-- .../ostelco/prime/storage/graph/Neo4jStore.kt | 34 ++++++++++++++++--- .../prime/storage/graph/GraphStoreTest.kt | 7 ++-- 5 files changed, 38 insertions(+), 15 deletions(-) diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt index c96fe09fe..a54017625 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt @@ -96,8 +96,7 @@ class ProfileTest { assertEquals("", clearedProfile.postCode, "Incorrect 'postcode' in response after clearing profile") assertEquals("", clearedProfile.city, "Incorrect 'city' in response after clearing profile") - updatedProfile - .country("") + updatedProfile.country("") // A test in 'HttpClientUtil' checks for status code 200 while the // expected status code is actually 400. diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt index a7c54a5b4..7ec06526f 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt @@ -77,8 +77,7 @@ class ProfileTest { assertEquals("", clearedProfile.postCode, "Incorrect 'postcode' in response after clearing profile") assertEquals("", clearedProfile.city, "Incorrect 'city' in response after clearing profile") - updatedProfile - .country("") + updatedProfile.country("") assertFailsWith(ApiException::class, "Incorrectly accepts that 'country' is cleared/not set") { client.updateProfile(updatedProfile) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt index b731e6189..e640fe8a3 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt @@ -52,14 +52,14 @@ fun initDatabase() { properties = mapOf("noOfBytes" to "1_000_000_000"))) val segments = listOf( - Segment(id = "no"), - Segment(id = "sg") + Segment(id = Neo4jStoreSingleton.getSegmentNameFromCountryCode("NO")), + Segment(id = Neo4jStoreSingleton.getSegmentNameFromCountryCode("SG")) ) segments.map { Neo4jStoreSingleton.createSegment(it) } val offer = Offer( id = "default_offer", - segments = listOf("no"), + segments = listOf(Neo4jStoreSingleton.getSegmentNameFromCountryCode("NO")), products = listOf("1GB_249NOK", "2GB_299NOK", "3GB_349NOK", "5GB_399NOK")) Neo4jStoreSingleton.createOffer(offer) } diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index dcc8a36f3..758ae5bb9 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -7,7 +7,6 @@ import org.neo4j.driver.v1.Transaction import org.ostelco.prime.analytics.AnalyticsService import org.ostelco.prime.analytics.PrimeMetric.REVENUE import org.ostelco.prime.analytics.PrimeMetric.USERS_PAID_AT_LEAST_ONCE -import org.ostelco.prime.core.ApiError import org.ostelco.prime.logger import org.ostelco.prime.model.Bundle import org.ostelco.prime.model.Offer @@ -24,8 +23,6 @@ import org.ostelco.prime.paymentprocessor.PaymentProcessor import org.ostelco.prime.paymentprocessor.core.BadGatewayError import org.ostelco.prime.paymentprocessor.core.PaymentError import org.ostelco.prime.paymentprocessor.core.ProductInfo -import org.ostelco.prime.paymentprocessor.core.ProfileInfo -import org.ostelco.prime.storage.DocumentStore import org.ostelco.prime.storage.GraphStore import org.ostelco.prime.storage.NotFoundError import org.ostelco.prime.storage.StoreError @@ -155,7 +152,23 @@ object Neo4jStoreSingleton : GraphStore { val bundleId = subscriber.id val either = subscriberStore.create(subscriber, transaction) - .flatMap { subscriberToSegmentStore.create(subscriber.id, subscriber.country.toLowerCase(), transaction) } + .flatMap { + subscriberToSegmentStore + .create(subscriber.id, + getSegmentNameFromCountryCode(subscriber.country), + transaction) + .mapLeft { storeError -> + if (storeError is NotFoundError && storeError.type == segmentEntity.name) { + ValidationError( + type = subscriberEntity.name, + id = subscriber.id, + message = "Unsupported country: ${subscriber.country}") + } else { + storeError + } + } + } + if (referredBy != null) { // Give 1 GB if subscriber is referred either @@ -197,7 +210,18 @@ object Neo4jStoreSingleton : GraphStore { .ifFailedThenRollback(transaction) } // << END - + + // Helper for naming of default segments based on country code. + + fun getSegmentNameFromCountryCode(country: String) : String { + val segmentName = when (country.toUpperCase()) { + "NO" -> "Norway_${country}" + "SG" -> "Singapore_${country}" + else -> "" + } + return segmentName.toLowerCase() + } + override fun updateSubscriber(subscriber: Subscriber): Either = writeTransaction { subscriberStore.update(subscriber, transaction) .ifFailedThenRollback(transaction) diff --git a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt index 89e8d6dd9..7258454ed 100644 --- a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt +++ b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt @@ -40,15 +40,15 @@ class GraphStoreTest { Neo4jStoreSingleton.createProduct( Product(sku = "100MB_FREE_ON_JOINING", - price = Price(0, "NOK"), + price = Price(0, CURRENCY), properties = mapOf("noOfBytes" to "100_000_000"))) Neo4jStoreSingleton.createProduct( Product(sku = "1GB_FREE_ON_REFERRED", - price = Price(0, "NOK"), + price = Price(0, CURRENCY), properties = mapOf("noOfBytes" to "1_000_000_000"))) - val allSegment = Segment(id = "no") + val allSegment = Segment(id = Neo4jStoreSingleton.getSegmentNameFromCountryCode(COUNTRY)) Neo4jStoreSingleton.createSegment(allSegment) } @@ -162,6 +162,7 @@ class GraphStoreTest { companion object { const val EMAIL = "foo@bar.com" const val NAME = "Test User" + const val CURRENCY = "NOK" const val COUNTRY = "NO" const val MSISDN = "4712345678" From c7cc26ac235b82b9f6b5cffc72eb170947f516a5 Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Fri, 21 Sep 2018 17:32:03 +0200 Subject: [PATCH 05/85] Adds sorting of list of sources on 'created' timestamp (WIP) --- .../paymentprocessor/StripePaymentProcessor.kt | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt index 27bb66d03..d5829832e 100644 --- a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt +++ b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt @@ -24,6 +24,13 @@ class StripePaymentProcessor : PaymentProcessor { sources.add(SourceDetailsInfo(it.id, getAccountType(details), details)) } sources +// sources.sortWith(Comparator { +// override fun compare(p1: SourceDetailsInfo, p2: SourceDetailsInfo) : Int when { +// p1.id > p2.id -> 1 +// p1.id == p2.id -> 0 +// else -> -1 +// } +// }) } private fun getAccountType(details: Map) : String { @@ -33,6 +40,7 @@ class StripePaymentProcessor : PaymentProcessor { /* Returns detailed 'account details' for the given Stripe source/account. Note that including the fields 'id' and 'type' are manadatory. */ private fun getAccountDetails(accountInfo: ExternalAccount) : Map { + logger.info(">>> details: {}", accountInfo) when (accountInfo) { is Card -> { return mapOf("id" to accountInfo.id, @@ -46,6 +54,9 @@ class StripePaymentProcessor : PaymentProcessor { "country" to accountInfo.country, "currency" to accountInfo.currency, "cvcCheck" to accountInfo.cvcCheck, + "created" to accountInfo.metadata.getOrElse("created") { + "${System.currentTimeMillis() / 1000}" + }, "expMonth" to accountInfo.expMonth, "expYear" to accountInfo.expYear, "fingerprint" to accountInfo.fingerprint, @@ -122,7 +133,8 @@ class StripePaymentProcessor : PaymentProcessor { override fun addSource(customerId: String, sourceId: String): Either = either("Failed to add source $sourceId to customer $customerId") { val customer = Customer.retrieve(customerId) - val params = mapOf("source" to sourceId) + val params = mapOf("source" to sourceId, + "metadata" to mapOf("created" to "${System.currentTimeMillis() / 1000}")) SourceInfo(customer.sources.create(params).id) } From 606e2b0e53436ef2bfdee78d19f8de9279a90f71 Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Fri, 21 Sep 2018 17:32:03 +0200 Subject: [PATCH 06/85] Adds sorting of list of sources on 'created' timestamp (WIP) --- .../paymentprocessor/StripePaymentProcessor.kt | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt index 27bb66d03..d5829832e 100644 --- a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt +++ b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt @@ -24,6 +24,13 @@ class StripePaymentProcessor : PaymentProcessor { sources.add(SourceDetailsInfo(it.id, getAccountType(details), details)) } sources +// sources.sortWith(Comparator { +// override fun compare(p1: SourceDetailsInfo, p2: SourceDetailsInfo) : Int when { +// p1.id > p2.id -> 1 +// p1.id == p2.id -> 0 +// else -> -1 +// } +// }) } private fun getAccountType(details: Map) : String { @@ -33,6 +40,7 @@ class StripePaymentProcessor : PaymentProcessor { /* Returns detailed 'account details' for the given Stripe source/account. Note that including the fields 'id' and 'type' are manadatory. */ private fun getAccountDetails(accountInfo: ExternalAccount) : Map { + logger.info(">>> details: {}", accountInfo) when (accountInfo) { is Card -> { return mapOf("id" to accountInfo.id, @@ -46,6 +54,9 @@ class StripePaymentProcessor : PaymentProcessor { "country" to accountInfo.country, "currency" to accountInfo.currency, "cvcCheck" to accountInfo.cvcCheck, + "created" to accountInfo.metadata.getOrElse("created") { + "${System.currentTimeMillis() / 1000}" + }, "expMonth" to accountInfo.expMonth, "expYear" to accountInfo.expYear, "fingerprint" to accountInfo.fingerprint, @@ -122,7 +133,8 @@ class StripePaymentProcessor : PaymentProcessor { override fun addSource(customerId: String, sourceId: String): Either = either("Failed to add source $sourceId to customer $customerId") { val customer = Customer.retrieve(customerId) - val params = mapOf("source" to sourceId) + val params = mapOf("source" to sourceId, + "metadata" to mapOf("created" to "${System.currentTimeMillis() / 1000}")) SourceInfo(customer.sources.create(params).id) } From 6bbf657c5c6522fdea3412588e1840cfd04e183d Mon Sep 17 00:00:00 2001 From: Martin Cederlof Date: Mon, 24 Sep 2018 14:25:41 +0200 Subject: [PATCH 07/85] Fixed configuration files for new pubsub topic --- .circleci/prime-dev-values.yaml | 1 + .circleci/prime-prod-values-template.yaml | 1 + .circleci/prime-prod-values.yaml | 1 + .../prime/analytics/publishers/ActiveUsersPublisher.kt | 4 ++-- .../main/kotlin/org/ostelco/prime/consumption/OcsService.kt | 1 - .../main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java | 2 +- prime/config/config.yaml | 2 +- prime/infra/dev/prime.yaml | 2 ++ prime/infra/prod/prime.yaml | 2 ++ 9 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.circleci/prime-dev-values.yaml b/.circleci/prime-dev-values.yaml index af7ba5042..7d9c67148 100644 --- a/.circleci/prime-dev-values.yaml +++ b/.circleci/prime-dev-values.yaml @@ -16,6 +16,7 @@ prime: STRIPE_API_KEY: "" DATA_TRAFFIC_TOPIC: "data-traffic" PURCHASE_INFO_TOPIC: "purchase-info" + ACTIVE_USERS_TOPIC: "active-users" ports: - 8080 - 8081 diff --git a/.circleci/prime-prod-values-template.yaml b/.circleci/prime-prod-values-template.yaml index 1664a9e9c..59615a8c3 100644 --- a/.circleci/prime-prod-values-template.yaml +++ b/.circleci/prime-prod-values-template.yaml @@ -15,6 +15,7 @@ prime: STRIPE_API_KEY: "" DATA_TRAFFIC_TOPIC: "data-traffic" PURCHASE_INFO_TOPIC: "purchase-info" + ACTIVE_USERS_TOPIC: "active-users" ports: - 8080 - 8081 diff --git a/.circleci/prime-prod-values.yaml b/.circleci/prime-prod-values.yaml index 12a906c71..19bc9752e 100644 --- a/.circleci/prime-prod-values.yaml +++ b/.circleci/prime-prod-values.yaml @@ -15,6 +15,7 @@ prime: STRIPE_API_KEY: "" DATA_TRAFFIC_TOPIC: "data-traffic" PURCHASE_INFO_TOPIC: "purchase-info" + ACTIVE_USERS_TOPIC: "active-users" ports: - 8080 diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt index cc9a98258..dce332dfe 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt @@ -14,7 +14,7 @@ import com.google.protobuf.util.Timestamps import com.google.pubsub.v1.PubsubMessage import org.ostelco.analytics.api.ActiveUsersInfo import org.ostelco.prime.analytics.ConfigRegistry -import org.ostelco.prime.logger +import org.ostelco.prime.getLogger import org.ostelco.prime.metrics.api.User import org.ostelco.prime.module.getResource import org.ostelco.prime.pseudonymizer.PseudonymizerService @@ -27,7 +27,7 @@ import java.time.Instant object ActiveUsersPublisher : PubSubPublisher by DelegatePubSubPublisher(topicId = ConfigRegistry.config.activeUsersTopicId) { - private val logger by logger() + private val logger by getLogger() private val pseudonymizerService by lazy { getResource() } diff --git a/ocs/src/main/kotlin/org/ostelco/prime/consumption/OcsService.kt b/ocs/src/main/kotlin/org/ostelco/prime/consumption/OcsService.kt index a529586a6..41a4588a8 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/consumption/OcsService.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/consumption/OcsService.kt @@ -72,7 +72,6 @@ class OcsService(private val producer: EventProducer) : OcsAsyncRequestConsumer, } override fun activateOnNextResponse(response: ActivateResponse) { - // TODO martin: send activate MSISDN to selective ocsgw instead of all this.activateMsisdnClientMap.forEach { _ , responseStream -> responseStream.onNext(response) } diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java index 6315abe93..6c755f101 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java @@ -288,7 +288,7 @@ private void updateAnalytics() { sessionIdMap.forEach((msisdn, sessionContext) -> { try { String apn = ccrMap.get(msisdn).getCreditControlRequest().getServiceInformation().get(0).getPsInformation().get(0).getCalledStationId(); - String mncMcc = ccrMap.get(msisdn).getCreditControlRequest().getServiceInformation().get(0).getPsInformation().get(0).getSgsnMncMcc(); + String mncMcc = ccrMap.get(msisdn).getCreditControlRequest().getServiceInformation().get(0).getPsInformation().get(0).getSgsnMccMnc(); builder.addUsers(User.newBuilder().setApn(apn).setMncMcc(mncMcc).build()); } catch (Exception e) { LOG.info("Failed to match session info to ccr map"); diff --git a/prime/config/config.yaml b/prime/config/config.yaml index 0aed4556b..f96a3fcc4 100644 --- a/prime/config/config.yaml +++ b/prime/config/config.yaml @@ -12,7 +12,7 @@ modules: projectId: pantel-2decb dataTrafficTopicId: ${DATA_TRAFFIC_TOPIC} purchaseInfoTopicId: ${PURCHASE_INFO_TOPIC} - activeUsersTopicId: active-users + activeUsersTopicId: ${ACTIVE_USERS_TOPIC} - type: ocs config: lowBalanceThreshold: 100000000 diff --git a/prime/infra/dev/prime.yaml b/prime/infra/dev/prime.yaml index ec00224e2..05a80140b 100644 --- a/prime/infra/dev/prime.yaml +++ b/prime/infra/dev/prime.yaml @@ -155,6 +155,8 @@ spec: value: data-traffic-dev - name: PURCHASE_INFO_TOPIC value: purchase-info-dev + - name: ACTIVE_USERS_TOPIC + value: active-users-dev - name: STRIPE_API_KEY valueFrom: secretKeyRef: diff --git a/prime/infra/prod/prime.yaml b/prime/infra/prod/prime.yaml index 56189a91c..2dfc8a11a 100644 --- a/prime/infra/prod/prime.yaml +++ b/prime/infra/prod/prime.yaml @@ -117,6 +117,8 @@ spec: value: data-traffic - name: PURCHASE_INFO_TOPIC value: purchase-info + - name: ACTIVE_USERS_TOPIC + value: active-users - name: STRIPE_API_KEY valueFrom: secretKeyRef: From b4ccfb4d3269f4e08c8e619f2d7b2b43ba6a6c98 Mon Sep 17 00:00:00 2001 From: Martin Cederlof Date: Mon, 24 Sep 2018 14:40:13 +0200 Subject: [PATCH 08/85] Actually setting the msisdn in the grpc message --- .../prime/analytics/publishers/ActiveUsersPublisher.kt | 3 +-- .../main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt index dce332dfe..502f87fd2 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt @@ -57,8 +57,8 @@ object ActiveUsersPublisher : fun publish(userList: List) { - val activeUsersInfoBuilder = ActiveUsersInfo.newBuilder().setTimestamp(Timestamps.fromMillis(Instant.now().toEpochMilli())) val timestamp = Instant.now().toEpochMilli() + val activeUsersInfoBuilder = ActiveUsersInfo.newBuilder().setTimestamp(Timestamps.fromMillis(timestamp)) for (user in userList) { val userBuilder = org.ostelco.analytics.api.User.newBuilder() val encodedSubscriberId = URLEncoder.encode(user.msisdn, "UTF-8") @@ -66,7 +66,6 @@ object ActiveUsersPublisher : activeUsersInfoBuilder.addUsers(userBuilder.setApn(user.apn).setMncMcc(user.mncMcc).setMsisdn(pseudonym).build()) } - val pubsubMessage = PubsubMessage.newBuilder() .setData(convertToJson(activeUsersInfoBuilder.build())) .build() diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java index 6c755f101..8fd2b68f7 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java @@ -281,7 +281,7 @@ private void removeFromSessionMap(CreditControlContext creditControlContext) { } private void updateAnalytics() { - LOG.info("Number of active sesssions is {}", sessionIdMap.size()); + LOG.info("Number of active sessions is {}", sessionIdMap.size()); OcsgwAnalyticsReport.Builder builder = OcsgwAnalyticsReport.newBuilder().setActiveSessions(sessionIdMap.size()); @@ -289,7 +289,7 @@ private void updateAnalytics() { try { String apn = ccrMap.get(msisdn).getCreditControlRequest().getServiceInformation().get(0).getPsInformation().get(0).getCalledStationId(); String mncMcc = ccrMap.get(msisdn).getCreditControlRequest().getServiceInformation().get(0).getPsInformation().get(0).getSgsnMccMnc(); - builder.addUsers(User.newBuilder().setApn(apn).setMncMcc(mncMcc).build()); + builder.addUsers(User.newBuilder().setApn(apn).setMncMcc(mncMcc).setMsisdn(msisdn).build()); } catch (Exception e) { LOG.info("Failed to match session info to ccr map"); } From c6280e5a41811675b3717da5c1717cdfa21488d3 Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Mon, 24 Sep 2018 14:41:42 +0200 Subject: [PATCH 09/85] Adds sorting in desceding order of list of sources based on the 'created' timestamp (WIP) --- .../StripePaymentProcessorTest.kt | 19 +++++++++++++++ .../StripePaymentProcessor.kt | 24 +++++++------------ .../prime/paymentprocessor/core/Model.kt | 12 +++++----- 3 files changed, 34 insertions(+), 21 deletions(-) diff --git a/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt b/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt index 11e9421f7..826c39bed 100644 --- a/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt +++ b/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt @@ -1,5 +1,7 @@ package org.ostelco.prime.paymentprocessor +import arrow.core.flatMap +import arrow.core.right import com.stripe.Stripe import com.stripe.model.Token import org.junit.After @@ -68,6 +70,23 @@ class StripePaymentProcessorTest { assertEquals(false, result.isRight()) } + @Test + fun ensureSortedSourceList() { + + val addedSources = listOf( + paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()), + paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()), + paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + ) + + // Should now be sorted descending by the "created" timestamp. + val sortedSources = paymentProcessor.getSavedSources(stripeCustomerId) + + assertEquals(true, sortedSources.isRight()) + + println(">>> stored sources: ${sortedSources}") + } + @Test fun addSourceToCustomerAndRemove() { diff --git a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt index d5829832e..a7da852a6 100644 --- a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt +++ b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt @@ -9,28 +9,18 @@ import org.ostelco.prime.paymentprocessor.core.* import com.stripe.model.Customer - - class StripePaymentProcessor : PaymentProcessor { private val logger by logger() override fun getSavedSources(customerId: String): Either> = either("Failed to retrieve sources for customer $customerId") { - val sources = mutableListOf() val customer = Customer.retrieve(customerId) - customer.sources.data.forEach { + val sources: List = customer.sources.data.map { val details = getAccountDetails(it) - sources.add(SourceDetailsInfo(it.id, getAccountType(details), details)) + SourceDetailsInfo(it.id, getAccountType(details), details) } - sources -// sources.sortWith(Comparator { -// override fun compare(p1: SourceDetailsInfo, p2: SourceDetailsInfo) : Int when { -// p1.id > p2.id -> 1 -// p1.id == p2.id -> 0 -// else -> -1 -// } -// }) + sources.sortedByDescending { it.details.get("created") as String } } private fun getAccountType(details: Map) : String { @@ -55,7 +45,7 @@ class StripePaymentProcessor : PaymentProcessor { "currency" to accountInfo.currency, "cvcCheck" to accountInfo.cvcCheck, "created" to accountInfo.metadata.getOrElse("created") { - "${System.currentTimeMillis() / 1000}" + getSecondsSinceEpoch() }, "expMonth" to accountInfo.expMonth, "expYear" to accountInfo.expYear, @@ -80,6 +70,10 @@ class StripePaymentProcessor : PaymentProcessor { } } + private fun getSecondsSinceEpoch() : Long { + return System.currentTimeMillis() / 1000L + } + override fun createPaymentProfile(userEmail: String): Either = either("Failed to create profile for user $userEmail") { val customerParams = mapOf("email" to userEmail) @@ -134,7 +128,7 @@ class StripePaymentProcessor : PaymentProcessor { either("Failed to add source $sourceId to customer $customerId") { val customer = Customer.retrieve(customerId) val params = mapOf("source" to sourceId, - "metadata" to mapOf("created" to "${System.currentTimeMillis() / 1000}")) + "metadata" to mapOf("created" to getSecondsSinceEpoch())) SourceInfo(customer.sources.create(params).id) } diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/core/Model.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/core/Model.kt index bf608316c..21c0b9301 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/core/Model.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/core/Model.kt @@ -1,13 +1,13 @@ package org.ostelco.prime.paymentprocessor.core -class PlanInfo(val id: String) +data class PlanInfo(val id: String) -class ProductInfo(val id: String) +data class ProductInfo(val id: String) -class ProfileInfo(val id: String) +data class ProfileInfo(val id: String) -class SourceInfo(val id: String) +data class SourceInfo(val id: String) -class SourceDetailsInfo(val id: String, val type: String, val details: Map) +data class SourceDetailsInfo(val id: String, val type: String, val details: Map) -class SubscriptionInfo(val id: String) +data class SubscriptionInfo(val id: String) From 0d704945002a8f71615664dc760ab45cd61e5cf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 24 Sep 2018 15:14:41 +0200 Subject: [PATCH 10/85] Initial checkin of branch --- sample-agent/README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 sample-agent/README.md diff --git a/sample-agent/README.md b/sample-agent/README.md new file mode 100644 index 000000000..a3582e70c --- /dev/null +++ b/sample-agent/README.md @@ -0,0 +1,19 @@ + +This is a sample of how an analytic agent could be made +=== + +The objective of an anlytic agent is to take input from the exporter +module, do something with it, and then import into the admin-API's +"importer" interface with the result that new offers are made +available to whomever the agent decides it should be made available +to. + +The code in this directory is intended as a proof of concept that +it is indeed possible to export customer data, use it for something, +and use it to produce segments and offers. + +The functions being tested are the pseudo-anonymization happening in +the exporter and importers, and also the general "ergonomics" of the +agent. If it's not something we believe can be used by a competent +person of an external organization after a two-minute setup time, +then it's not good enugh. From b74bdfbe14a06967f0cb195343da856bc172b568 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 24 Sep 2018 15:17:03 +0200 Subject: [PATCH 11/85] Update readme with a tbd --- exporter/README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/exporter/README.md b/exporter/README.md index cd8e306bc..05e7614ce 100644 --- a/exporter/README.md +++ b/exporter/README.md @@ -1,4 +1,4 @@ -# Exporter +## Exporter This contains a set of scripts to generate the data for analayis. The export script `export_data.sh` creates a new big query table with a new uuid which maps the pseudonyms to @@ -35,4 +35,9 @@ kubectl exec -it -- /bin/bash # Delete deployment kubectl delete deployment exporter -``` \ No newline at end of file +``` + + +## How to get data from the exporter + + .... tbd \ No newline at end of file From d4b8cdf90ce3ee205fba751bc5bf29b58713b443 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 24 Sep 2018 15:29:27 +0200 Subject: [PATCH 12/85] First stab at sample agent --- sample-agent/sample-agent.sh | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 sample-agent/sample-agent.sh diff --git a/sample-agent/sample-agent.sh b/sample-agent/sample-agent.sh new file mode 100644 index 000000000..13611c83c --- /dev/null +++ b/sample-agent/sample-agent.sh @@ -0,0 +1,37 @@ +#!/bin/bash +IMPORTER_URL=http://127.00.1:8080/importer + + +## Check that credentials are set up + + # tbd + +## Fetch the input data form the exporter + +# tbd + +## Calculate the output, and put into a file + + +TMPFILE=tmpfile.yml + +cat > $TMPFILE < Date: Mon, 24 Sep 2018 15:35:16 +0200 Subject: [PATCH 13/85] Publishers are managed by dropwizard --- .../main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt index 7ffa00c00..0b8e35dec 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt @@ -5,6 +5,7 @@ import com.fasterxml.jackson.annotation.JsonTypeName import io.dropwizard.setup.Environment import org.hibernate.validator.constraints.NotEmpty import org.ostelco.prime.analytics.metrics.CustomMetricsRegistry +import org.ostelco.prime.analytics.publishers.ActiveUsersPublisher import org.ostelco.prime.analytics.publishers.DataConsumptionInfoPublisher import org.ostelco.prime.analytics.publishers.PurchaseInfoPublisher import org.ostelco.prime.module.PrimeModule @@ -28,6 +29,7 @@ class AnalyticsModule : PrimeModule { // dropwizard starts Analytics events publisher env.lifecycle().manage(DataConsumptionInfoPublisher) env.lifecycle().manage(PurchaseInfoPublisher) + env.lifecycle().manage(ActiveUsersPublisher) } } From 2ef8210da684364162fd4a5c4b2cf7e9a85fb479 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 24 Sep 2018 15:35:41 +0200 Subject: [PATCH 14/85] Chmod +x on sample-agent.sh --- sample-agent/sample-agent.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 sample-agent/sample-agent.sh diff --git a/sample-agent/sample-agent.sh b/sample-agent/sample-agent.sh old mode 100644 new mode 100755 From ca174dd56e9a0e28e7bf4492660927075f12aa4f Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Thu, 20 Sep 2018 15:41:11 +0200 Subject: [PATCH 15/85] Adds support for default segment per country (WIP) --- .../kotlin/org/ostelco/at/common/TestUser.kt | 2 +- .../kotlin/org/ostelco/at/jersey/Tests.kt | 25 ++++++++++++------- .../kotlin/org/ostelco/at/okhttp/Tests.kt | 17 +++++++------ .../prime/client/api/store/SubscriberDAO.kt | 3 ++- .../prime/storage/graph/Neo4jModule.kt | 9 ++++--- .../ostelco/prime/storage/graph/Neo4jStore.kt | 2 +- .../prime/storage/graph/GraphStoreTest.kt | 15 +++++------ .../prime/storage/graph/Neo4jStorageTest.kt | 3 ++- 8 files changed, 46 insertions(+), 30 deletions(-) diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/common/TestUser.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/common/TestUser.kt index d55a70598..3a278d77b 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/common/TestUser.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/common/TestUser.kt @@ -12,7 +12,7 @@ fun createProfile(name: String, email: String) { .name(name) .address("") .city("") - .country("") + .country("NO") .postCode("") .referralId("") diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt index 8ada341c8..286d20211 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt @@ -20,14 +20,10 @@ import org.ostelco.prime.client.model.Profile import org.ostelco.prime.client.model.PurchaseRecordList import org.ostelco.prime.client.model.Subscription import org.ostelco.prime.client.model.SubscriptionStatus +import java.lang.AssertionError import java.time.Instant import java.util.* -import kotlin.test.assertEquals -import kotlin.test.assertFails -import kotlin.test.assertFalse -import kotlin.test.assertNotNull -import kotlin.test.assertNull -import kotlin.test.assertTrue +import kotlin.test.* class ProfileTest { @@ -42,7 +38,7 @@ class ProfileTest { .name("Test Profile User") .address("") .city("") - .country("") + .country("NO") .postCode("") .referralId("") @@ -88,7 +84,6 @@ class ProfileTest { .address("") .postCode("") .city("") - .country("") val clearedProfile: Profile = put { path = "/profile" @@ -101,7 +96,19 @@ class ProfileTest { assertEquals("", clearedProfile.address, "Incorrect 'address' in response after clearing profile") assertEquals("", clearedProfile.postCode, "Incorrect 'postcode' in response after clearing profile") assertEquals("", clearedProfile.city, "Incorrect 'city' in response after clearing profile") - assertEquals("", clearedProfile.country, "Incorrect 'country' in response after clearing profile") + + updatedProfile + .country("") + + // A test in 'HttpClientUtil' checks for status code 200 while the + // expected status code is actually 400. + assertFailsWith(AssertionError::class, "Incorrectly accepts that 'country' is cleared/not set") { + put { + path = "/profile" + body = updatedProfile + subscriberId = email + } + } } @Test diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt index 715353b6c..5e7bdf7a9 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt @@ -9,6 +9,7 @@ import org.ostelco.at.common.getLogger import org.ostelco.at.common.randomInt import org.ostelco.at.okhttp.ClientFactory.clientForSubject import org.ostelco.prime.client.api.DefaultApi +import org.ostelco.prime.client.ApiException import org.ostelco.prime.client.model.ApplicationToken import org.ostelco.prime.client.model.Consent import org.ostelco.prime.client.model.PaymentSource @@ -20,10 +21,7 @@ import org.ostelco.prime.client.model.Profile import org.ostelco.prime.client.model.SubscriptionStatus import java.time.Instant import java.util.* -import kotlin.test.assertEquals -import kotlin.test.assertFails -import kotlin.test.assertNotNull -import kotlin.test.assertNull +import kotlin.test.* class ProfileTest { @@ -39,7 +37,7 @@ class ProfileTest { .name("Test Profile User") .address("") .city("") - .country("") + .country("NO") .postCode("") .referralId("") @@ -70,7 +68,6 @@ class ProfileTest { .address("") .postCode("") .city("") - .country("") val clearedProfile: Profile = client.updateProfile(updatedProfile) @@ -79,7 +76,13 @@ class ProfileTest { assertEquals("", clearedProfile.address, "Incorrect 'address' in response after clearing profile") assertEquals("", clearedProfile.postCode, "Incorrect 'postcode' in response after clearing profile") assertEquals("", clearedProfile.city, "Incorrect 'city' in response after clearing profile") - assertEquals("", clearedProfile.country, "Incorrect 'country' in response after clearing profile") + + updatedProfile + .country("") + + assertFailsWith(ApiException::class, "Incorrectly accepts that 'country' is cleared/not set") { + client.updateProfile(updatedProfile) + } } @Test diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAO.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAO.kt index 11b750282..db00add39 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAO.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAO.kt @@ -73,7 +73,8 @@ interface SubscriberDAO { fun isValidProfile(profile: Subscriber?): Boolean { return (profile != null && !profile.name.isEmpty() - && !profile.email.isEmpty()) + && !profile.email.isEmpty() + && !profile.country.isEmpty()) } /** diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt index d9c3ea8ce..b731e6189 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt @@ -51,12 +51,15 @@ fun initDatabase() { price = Price(0, "NOK"), properties = mapOf("noOfBytes" to "1_000_000_000"))) - val segment = Segment(id = "all") - Neo4jStoreSingleton.createSegment(segment) + val segments = listOf( + Segment(id = "no"), + Segment(id = "sg") + ) + segments.map { Neo4jStoreSingleton.createSegment(it) } val offer = Offer( id = "default_offer", - segments = listOf("all"), + segments = listOf("no"), products = listOf("1GB_249NOK", "2GB_299NOK", "3GB_349NOK", "5GB_399NOK")) Neo4jStoreSingleton.createOffer(offer) } diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index 733f9b63c..b64b150f1 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -152,6 +152,7 @@ object Neo4jStoreSingleton : GraphStore { val bundleId = subscriber.id val either = subscriberStore.create(subscriber, transaction) + .flatMap { subscriberToSegmentStore.create(subscriber.id, subscriber.country.toLowerCase(), transaction) } if (referredBy != null) { // Give 1 GB if subscriber is referred either @@ -190,7 +191,6 @@ object Neo4jStoreSingleton : GraphStore { Either.right(Unit) } }.flatMap { subscriberToBundleStore.create(subscriber.id, bundleId, transaction) } - .flatMap { subscriberToSegmentStore.create(subscriber.id, "all", transaction) } .ifFailedThenRollback(transaction) } // << END diff --git a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt index 3d46a5fc1..cedfbc9b2 100644 --- a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt +++ b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt @@ -48,19 +48,19 @@ class GraphStoreTest { price = Price(0, "NOK"), properties = mapOf("noOfBytes" to "1_000_000_000"))) - val allSegment = Segment(id = "all") + val allSegment = Segment(id = "no") Neo4jStoreSingleton.createSegment(allSegment) } @Test fun `add subscriber`() { - Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME), referredBy = null) + Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME, country = COUNTRY), referredBy = null) .mapLeft { fail(it.message) } Neo4jStoreSingleton.getSubscriber(EMAIL).bimap( { fail(it.message) }, - { assertEquals(Subscriber(email = EMAIL, name = NAME, referralId = EMAIL), it) }) + { assertEquals(Subscriber(email = EMAIL, name = NAME, referralId = EMAIL, country = COUNTRY), it) }) // TODO vihang: fix argument captor for neo4j-store tests // val bundleArgCaptor: ArgumentCaptor = ArgumentCaptor.forClass(Bundle::class.java) @@ -71,7 +71,7 @@ class GraphStoreTest { @Test fun `fail to add subscriber with invalid referred by`() { - Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME), referredBy = "blah") + Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME, country = COUNTRY), referredBy = "blah") .fold({ assertEquals( expected = "Failed to create REFERRED - blah -> foo@bar.com", @@ -83,7 +83,7 @@ class GraphStoreTest { @Test fun `add subscription`() { - Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME), referredBy = null) + Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME, country = COUNTRY), referredBy = null) .mapLeft { fail(it.message) } Neo4jStoreSingleton.addSubscription(EMAIL, MSISDN) @@ -107,7 +107,7 @@ class GraphStoreTest { @Test fun `set and get Purchase record`() { - assert(Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME), referredBy = null).isRight()) + assert(Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME, country = COUNTRY), referredBy = null).isRight()) val product = createProduct("1GB_249NOK", 24900) val now = Instant.now().toEpochMilli() @@ -129,7 +129,7 @@ class GraphStoreTest { @Test fun `create products, offer, segment and then get products for a subscriber`() { - assert(Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME), referredBy = null).isRight()) + assert(Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME, country = COUNTRY), referredBy = null).isRight()) Neo4jStoreSingleton.createProduct(createProduct("1GB_249NOK", 24900)) Neo4jStoreSingleton.createProduct(createProduct("2GB_299NOK", 29900)) @@ -215,6 +215,7 @@ class GraphStoreTest { companion object { const val EMAIL = "foo@bar.com" const val NAME = "Test User" + const val COUNTRY = "NO" const val MSISDN = "4712345678" @ClassRule diff --git a/prime/src/integration-tests/kotlin/org/ostelco/prime/storage/graph/Neo4jStorageTest.kt b/prime/src/integration-tests/kotlin/org/ostelco/prime/storage/graph/Neo4jStorageTest.kt index bc960b646..cefb2fc6c 100644 --- a/prime/src/integration-tests/kotlin/org/ostelco/prime/storage/graph/Neo4jStorageTest.kt +++ b/prime/src/integration-tests/kotlin/org/ostelco/prime/storage/graph/Neo4jStorageTest.kt @@ -36,7 +36,7 @@ class Neo4jStorageTest { sleep(MILLIS_TO_WAIT_WHEN_STARTING_UP.toLong()) storage.removeSubscriber(EPHERMERAL_EMAIL) - storage.addSubscriber(Subscriber(EPHERMERAL_EMAIL), referredBy = null) + storage.addSubscriber(Subscriber(EPHERMERAL_EMAIL, country = COUNTRY), referredBy = null) .mapLeft { fail(it.message) } storage.addSubscription(EPHERMERAL_EMAIL, MSISDN) .mapLeft { fail(it.message) } @@ -90,6 +90,7 @@ class Neo4jStorageTest { private const val EPHERMERAL_EMAIL = "attherate@dotcom.com" private const val MSISDN = "4747116996" + private const val COUNTRY = "NO" private const val MILLIS_TO_WAIT_WHEN_STARTING_UP = 3000 From c32eb6e2954a5561d196a3e68b1525a7b7da4f9c Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Fri, 21 Sep 2018 14:21:21 +0200 Subject: [PATCH 16/85] Fixes to 'add support for default segment per country' (from code review) --- .../kotlin/org/ostelco/at/jersey/Tests.kt | 3 +- .../kotlin/org/ostelco/at/okhttp/Tests.kt | 3 +- .../prime/storage/graph/Neo4jModule.kt | 6 ++-- .../ostelco/prime/storage/graph/Neo4jStore.kt | 31 +++++++++++++++++-- .../prime/storage/graph/GraphStoreTest.kt | 7 +++-- 5 files changed, 38 insertions(+), 12 deletions(-) diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt index 286d20211..da04f7da3 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt @@ -97,8 +97,7 @@ class ProfileTest { assertEquals("", clearedProfile.postCode, "Incorrect 'postcode' in response after clearing profile") assertEquals("", clearedProfile.city, "Incorrect 'city' in response after clearing profile") - updatedProfile - .country("") + updatedProfile.country("") // A test in 'HttpClientUtil' checks for status code 200 while the // expected status code is actually 400. diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt index 5e7bdf7a9..40d04e0be 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt @@ -77,8 +77,7 @@ class ProfileTest { assertEquals("", clearedProfile.postCode, "Incorrect 'postcode' in response after clearing profile") assertEquals("", clearedProfile.city, "Incorrect 'city' in response after clearing profile") - updatedProfile - .country("") + updatedProfile.country("") assertFailsWith(ApiException::class, "Incorrectly accepts that 'country' is cleared/not set") { client.updateProfile(updatedProfile) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt index b731e6189..e640fe8a3 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt @@ -52,14 +52,14 @@ fun initDatabase() { properties = mapOf("noOfBytes" to "1_000_000_000"))) val segments = listOf( - Segment(id = "no"), - Segment(id = "sg") + Segment(id = Neo4jStoreSingleton.getSegmentNameFromCountryCode("NO")), + Segment(id = Neo4jStoreSingleton.getSegmentNameFromCountryCode("SG")) ) segments.map { Neo4jStoreSingleton.createSegment(it) } val offer = Offer( id = "default_offer", - segments = listOf("no"), + segments = listOf(Neo4jStoreSingleton.getSegmentNameFromCountryCode("NO")), products = listOf("1GB_249NOK", "2GB_299NOK", "3GB_349NOK", "5GB_399NOK")) Neo4jStoreSingleton.createOffer(offer) } diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index b64b150f1..002051767 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -152,7 +152,23 @@ object Neo4jStoreSingleton : GraphStore { val bundleId = subscriber.id val either = subscriberStore.create(subscriber, transaction) - .flatMap { subscriberToSegmentStore.create(subscriber.id, subscriber.country.toLowerCase(), transaction) } + .flatMap { + subscriberToSegmentStore + .create(subscriber.id, + getSegmentNameFromCountryCode(subscriber.country), + transaction) + .mapLeft { storeError -> + if (storeError is NotFoundError && storeError.type == segmentEntity.name) { + ValidationError( + type = subscriberEntity.name, + id = subscriber.id, + message = "Unsupported country: ${subscriber.country}") + } else { + storeError + } + } + } + if (referredBy != null) { // Give 1 GB if subscriber is referred either @@ -195,6 +211,17 @@ object Neo4jStoreSingleton : GraphStore { } // << END + // Helper for naming of default segments based on country code. + + fun getSegmentNameFromCountryCode(country: String) : String { + val segmentName = when (country.toUpperCase()) { + "NO" -> "Norway_${country}" + "SG" -> "Singapore_${country}" + else -> "" + } + return segmentName.toLowerCase() + } + override fun updateSubscriber(subscriber: Subscriber): Either = writeTransaction { subscriberStore.update(subscriber, transaction) .ifFailedThenRollback(transaction) @@ -717,4 +744,4 @@ fun Either.ifFailedThenRollback(transaction: Transaction): Either Date: Fri, 21 Sep 2018 17:32:03 +0200 Subject: [PATCH 17/85] Adds sorting of list of sources on 'created' timestamp (WIP) --- .../paymentprocessor/StripePaymentProcessor.kt | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt index 6b6f980ca..a10984b4c 100644 --- a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt +++ b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt @@ -21,6 +21,13 @@ class StripePaymentProcessor : PaymentProcessor { sources.add(SourceDetailsInfo(it.id, getAccountType(details), details)) } sources +// sources.sortWith(Comparator { +// override fun compare(p1: SourceDetailsInfo, p2: SourceDetailsInfo) : Int when { +// p1.id > p2.id -> 1 +// p1.id == p2.id -> 0 +// else -> -1 +// } +// }) } private fun getAccountType(details: Map) : String { @@ -30,6 +37,7 @@ class StripePaymentProcessor : PaymentProcessor { /* Returns detailed 'account details' for the given Stripe source/account. Note that including the fields 'id' and 'type' are manadatory. */ private fun getAccountDetails(accountInfo: ExternalAccount) : Map { + logger.info(">>> details: {}", accountInfo) when (accountInfo) { is Card -> { return mapOf("id" to accountInfo.id, @@ -43,6 +51,9 @@ class StripePaymentProcessor : PaymentProcessor { "country" to accountInfo.country, "currency" to accountInfo.currency, "cvcCheck" to accountInfo.cvcCheck, + "created" to accountInfo.metadata.getOrElse("created") { + "${System.currentTimeMillis() / 1000}" + }, "expMonth" to accountInfo.expMonth, "expYear" to accountInfo.expYear, "fingerprint" to accountInfo.fingerprint, @@ -119,7 +130,8 @@ class StripePaymentProcessor : PaymentProcessor { override fun addSource(customerId: String, sourceId: String): Either = either("Failed to add source $sourceId to customer $customerId") { val customer = Customer.retrieve(customerId) - val params = mapOf("source" to sourceId) + val params = mapOf("source" to sourceId, + "metadata" to mapOf("created" to "${System.currentTimeMillis() / 1000}")) SourceInfo(customer.sources.create(params).id) } From 279d7b19f2c23326ba589062c2461c0a12a75880 Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Mon, 24 Sep 2018 14:41:42 +0200 Subject: [PATCH 18/85] Adds sorting in desceding order of list of sources based on the 'created' timestamp (WIP) --- .../StripePaymentProcessorTest.kt | 19 +++++++++++++++ .../StripePaymentProcessor.kt | 23 ++++++++----------- .../prime/paymentprocessor/core/Model.kt | 12 +++++----- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt b/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt index 6b0d8aecc..31d237d3b 100644 --- a/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt +++ b/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt @@ -1,5 +1,7 @@ package org.ostelco.prime.paymentprocessor +import arrow.core.flatMap +import arrow.core.right import com.stripe.Stripe import com.stripe.model.Token import org.junit.After @@ -68,6 +70,23 @@ class StripePaymentProcessorTest { assertEquals(false, result.isRight()) } + @Test + fun ensureSortedSourceList() { + + val addedSources = listOf( + paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()), + paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()), + paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + ) + + // Should now be sorted descending by the "created" timestamp. + val sortedSources = paymentProcessor.getSavedSources(stripeCustomerId) + + assertEquals(true, sortedSources.isRight()) + + println(">>> stored sources: ${sortedSources}") + } + @Test fun addSourceToCustomerAndRemove() { diff --git a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt index a10984b4c..08866d32b 100644 --- a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt +++ b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt @@ -8,26 +8,19 @@ import com.stripe.model.* import org.ostelco.prime.paymentprocessor.core.* import com.stripe.model.Customer + class StripePaymentProcessor : PaymentProcessor { private val logger by getLogger() override fun getSavedSources(customerId: String): Either> = either("Failed to retrieve sources for customer $customerId") { - val sources = mutableListOf() val customer = Customer.retrieve(customerId) - customer.sources.data.forEach { + val sources: List = customer.sources.data.map { val details = getAccountDetails(it) - sources.add(SourceDetailsInfo(it.id, getAccountType(details), details)) + SourceDetailsInfo(it.id, getAccountType(details), details) } - sources -// sources.sortWith(Comparator { -// override fun compare(p1: SourceDetailsInfo, p2: SourceDetailsInfo) : Int when { -// p1.id > p2.id -> 1 -// p1.id == p2.id -> 0 -// else -> -1 -// } -// }) + sources.sortedByDescending { it.details.get("created") as String } } private fun getAccountType(details: Map) : String { @@ -52,7 +45,7 @@ class StripePaymentProcessor : PaymentProcessor { "currency" to accountInfo.currency, "cvcCheck" to accountInfo.cvcCheck, "created" to accountInfo.metadata.getOrElse("created") { - "${System.currentTimeMillis() / 1000}" + getSecondsSinceEpoch() }, "expMonth" to accountInfo.expMonth, "expYear" to accountInfo.expYear, @@ -77,6 +70,10 @@ class StripePaymentProcessor : PaymentProcessor { } } + private fun getSecondsSinceEpoch() : Long { + return System.currentTimeMillis() / 1000L + } + override fun createPaymentProfile(userEmail: String): Either = either("Failed to create profile for user $userEmail") { val customerParams = mapOf("email" to userEmail) @@ -131,7 +128,7 @@ class StripePaymentProcessor : PaymentProcessor { either("Failed to add source $sourceId to customer $customerId") { val customer = Customer.retrieve(customerId) val params = mapOf("source" to sourceId, - "metadata" to mapOf("created" to "${System.currentTimeMillis() / 1000}")) + "metadata" to mapOf("created" to getSecondsSinceEpoch())) SourceInfo(customer.sources.create(params).id) } diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/core/Model.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/core/Model.kt index bf608316c..21c0b9301 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/core/Model.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/core/Model.kt @@ -1,13 +1,13 @@ package org.ostelco.prime.paymentprocessor.core -class PlanInfo(val id: String) +data class PlanInfo(val id: String) -class ProductInfo(val id: String) +data class ProductInfo(val id: String) -class ProfileInfo(val id: String) +data class ProfileInfo(val id: String) -class SourceInfo(val id: String) +data class SourceInfo(val id: String) -class SourceDetailsInfo(val id: String, val type: String, val details: Map) +data class SourceDetailsInfo(val id: String, val type: String, val details: Map) -class SubscriptionInfo(val id: String) +data class SubscriptionInfo(val id: String) From eab6b8d99d24868c6d6400d5117049779aa68085 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Tue, 25 Sep 2018 02:21:13 +0200 Subject: [PATCH 19/85] Minor refactoring --- .circleci/config.yml | 14 ++++++++++---- .../org/ostelco/prime/storage/graph/Neo4jModule.kt | 9 ++++++--- .../org/ostelco/prime/storage/graph/Neo4jStore.kt | 11 ----------- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bb33aa710..f155d7d1e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,7 +39,13 @@ jobs: - run: name: Build entire repo command: ./gradlew clean build -info -s -x :neo4j-store:test - + - run: + name: Push Gradle cache + command: | + rm -f ~/.gradle/caches/modules-2/modules-2.lock + rm -fr ~/.gradle/caches/*/plugin-resolution/ + tar -czvf ~/caches.tar.gz -C ~/.gradle/caches . + gsutil cp ~/caches.tar.gz gs://pi-ostelco-core-gradle-cache # persisting the entire project with its generated artifacts. They are needed in the build-image job below. # the default working directory in circleci is ~/project/ - persist_to_workspace: @@ -342,9 +348,9 @@ workflows: - deploy-to-dev: requires: - update-dev-endpoints - - create-PR-to-master: - requires: - - deploy-to-dev +# - create-PR-to-master: +# requires: +# - deploy-to-dev deploy-to-prod: jobs: diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt index e640fe8a3..18ad1be2f 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt @@ -52,18 +52,21 @@ fun initDatabase() { properties = mapOf("noOfBytes" to "1_000_000_000"))) val segments = listOf( - Segment(id = Neo4jStoreSingleton.getSegmentNameFromCountryCode("NO")), - Segment(id = Neo4jStoreSingleton.getSegmentNameFromCountryCode("SG")) + Segment(id = getSegmentNameFromCountryCode("NO")), + Segment(id = getSegmentNameFromCountryCode("SG")) ) segments.map { Neo4jStoreSingleton.createSegment(it) } val offer = Offer( id = "default_offer", - segments = listOf(Neo4jStoreSingleton.getSegmentNameFromCountryCode("NO")), + segments = listOf(getSegmentNameFromCountryCode("NO")), products = listOf("1GB_249NOK", "2GB_299NOK", "3GB_349NOK", "5GB_399NOK")) Neo4jStoreSingleton.createOffer(offer) } +// Helper for naming of default segments based on country code. +fun getSegmentNameFromCountryCode(countryCode: String) : String = "country-$countryCode".toLowerCase() + class Config { lateinit var host: String lateinit var protocol: String diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index 002051767..b4ac4c111 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -211,17 +211,6 @@ object Neo4jStoreSingleton : GraphStore { } // << END - // Helper for naming of default segments based on country code. - - fun getSegmentNameFromCountryCode(country: String) : String { - val segmentName = when (country.toUpperCase()) { - "NO" -> "Norway_${country}" - "SG" -> "Singapore_${country}" - else -> "" - } - return segmentName.toLowerCase() - } - override fun updateSubscriber(subscriber: Subscriber): Either = writeTransaction { subscriberStore.update(subscriber, transaction) .ifFailedThenRollback(transaction) From 757694d4e41598a46b47771d820d095684dff76c Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Tue, 25 Sep 2018 12:50:44 +0200 Subject: [PATCH 20/85] Adds sorting in desceding order of list of sources based on the 'created' timestamp --- .../StripePaymentProcessorTest.kt | 29 +++++++++++-------- .../StripePaymentProcessor.kt | 26 ++++++++++++----- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt b/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt index 31d237d3b..7cc9029f1 100644 --- a/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt +++ b/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt @@ -1,7 +1,6 @@ package org.ostelco.prime.paymentprocessor -import arrow.core.flatMap -import arrow.core.right +import arrow.core.getOrElse import com.stripe.Stripe import com.stripe.model.Token import org.junit.After @@ -71,20 +70,26 @@ class StripePaymentProcessorTest { } @Test - fun ensureSortedSourceList() { + fun ensureSourcesSorted() { - val addedSources = listOf( - paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()), - paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()), - paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) - ) + run { + paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + // Ensure that not all sources falls within the same second. + Thread.sleep(1_001) + paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + } - // Should now be sorted descending by the "created" timestamp. - val sortedSources = paymentProcessor.getSavedSources(stripeCustomerId) + // Should be in descending sorted order by the "created" timestamp. + val sources = paymentProcessor.getSavedSources(stripeCustomerId) - assertEquals(true, sortedSources.isRight()) + val createdTimestamps = sources.getOrElse { + fail("The 'created' field is missing from the list of sources: ${sources}") + }.map { it.details["created"] as Long } - println(">>> stored sources: ${sortedSources}") + val createdTimestampsSorted = createdTimestamps.sortedByDescending { it } + + assertEquals(createdTimestampsSorted, createdTimestamps, + "The list of sources is not in descending sorted order by 'created' timestamp: ${sources}") } @Test diff --git a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt index 08866d32b..2af72ecb4 100644 --- a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt +++ b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt @@ -20,7 +20,7 @@ class StripePaymentProcessor : PaymentProcessor { val details = getAccountDetails(it) SourceDetailsInfo(it.id, getAccountType(details), details) } - sources.sortedByDescending { it.details.get("created") as String } + sources.sortedByDescending { it.details.get("created") as Long } } private fun getAccountType(details: Map) : String { @@ -28,9 +28,8 @@ class StripePaymentProcessor : PaymentProcessor { } /* Returns detailed 'account details' for the given Stripe source/account. - Note that including the fields 'id' and 'type' are manadatory. */ + Note that including the fields 'id', 'type' and 'created' are manadatory. */ private fun getAccountDetails(accountInfo: ExternalAccount) : Map { - logger.info(">>> details: {}", accountInfo) when (accountInfo) { is Card -> { return mapOf("id" to accountInfo.id, @@ -44,9 +43,8 @@ class StripePaymentProcessor : PaymentProcessor { "country" to accountInfo.country, "currency" to accountInfo.currency, "cvcCheck" to accountInfo.cvcCheck, - "created" to accountInfo.metadata.getOrElse("created") { - getSecondsSinceEpoch() - }, + "created" to getCreatedTimestampFromMetadata(accountInfo.id, + accountInfo.metadata), "expMonth" to accountInfo.expMonth, "expYear" to accountInfo.expYear, "fingerprint" to accountInfo.fingerprint, @@ -58,6 +56,7 @@ class StripePaymentProcessor : PaymentProcessor { is Source -> { return mapOf("id" to accountInfo.id, "type" to "source", + "created" to accountInfo.created, "typeData" to accountInfo.typeData, "owner" to accountInfo.owner) } @@ -65,11 +64,24 @@ class StripePaymentProcessor : PaymentProcessor { logger.error("Received unsupported Stripe source/account type: {}", accountInfo) return mapOf("id" to accountInfo.id, - "type" to "unsupported") + "type" to "unsupported", + "created" to getSecondsSinceEpoch()) } } } + /* As Stripe returns stored metadata as String, check for and return the 'created' timestamp from metadata. If not + found log and return current time as the timestamp. */ + private fun getCreatedTimestampFromMetadata(id: String, metadata: Map) : Long { + val created: String? = metadata.get("created") as? String + return created?.toLongOrNull() ?: run { + logger.warn("No 'created' timestamp found in metadata for Stripe account {}", + id) + getSecondsSinceEpoch() + } + } + + /* Seconds since Epoch in UTC zone. */ private fun getSecondsSinceEpoch() : Long { return System.currentTimeMillis() / 1000L } From 8172e6fda4d0b10090014de266450ea1c4280166 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 25 Sep 2018 13:13:32 +0200 Subject: [PATCH 21/85] How to run an export from the command line --- sample-agent/run-export.sh | 80 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100755 sample-agent/run-export.sh diff --git a/sample-agent/run-export.sh b/sample-agent/run-export.sh new file mode 100755 index 000000000..d49d189ab --- /dev/null +++ b/sample-agent/run-export.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +# +# Exploratory code to run an export using bigquery. +# + + +# +# Check for dependencies +# + +DEPENDENCIES="gcloud kubectl gsutil" + +for dep in $DEPENDENCIES ; do + if [[ -z $(which $dep) ]] ; then + echo "ERROR: Could not find dependency $dep" + fi +done + +# +# Figure out relevant parts of the environment and check their +# sanity. +# + + +PROJECT_ID=$(gcloud config get-value project) + +if [[ -z "$PROJECT_ID" ]] ; then + echo "ERROR: Unknown google project ID" + exit 1 +fi + + +EXPORTER_PODNAME=$(kubectl get pods | grep exporter- | awk '{print $1}') +if [[ -z "$EXPORTER_PODNAME" ]] ; then + echo "ERROR: Unknown exporter podname" + exit 1 +fi + + +# +# Run an export inside the kubernetes cluster, then parse +# the output of the script thar ran the export +# +TEMPFILE=tmpfile.txt +# mpfile=$(mktemp /tmp/abc-script.XXXXXX) + +# kubectl exec -it "${EXPORTER_PODNAME}" -- /bin/bash -c /export_data.sh > "$TEMPFILE" + +# # Fail if the exec failed +# retVal=$? +# if [ $retVal -ne 0 ]; then +# echo "ERROR: Failed to export data:" +# cat $TMPFILE +# rm $TMPFILE +# exit 1 +# fi + +# +# Parse the output of the tmpfile +# + + +EXPORT_ID=$(grep "Starting export job for" $TEMPFILE | awk '{print $5}' | sed 's/\r$//' ) + +echo "export id = $EXPORT_ID" + +PURCHASES_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID-purchases.csv" +SUB_2_MSISSDN_MAPPING_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID-sub2msisdn.csv" +CONSUMPTION_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID.csv" + +# +# Then copy the CSV files to local storage (current directory) +# + + +gsutil cp $PURCHASES_GS . +gsutil cp $SUB_2_MSISSDN_MAPPING_GS . +gsutil cp $CONSUMPTION_GS . + From 9519201cba04d3a1df0d3f89e927048bef52a7ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 25 Sep 2018 13:23:51 +0200 Subject: [PATCH 22/85] Adding more tests --- sample-agent/run-export.sh | 39 ++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/sample-agent/run-export.sh b/sample-agent/run-export.sh index d49d189ab..39b271bc9 100755 --- a/sample-agent/run-export.sh +++ b/sample-agent/run-export.sh @@ -1,9 +1,28 @@ #!/bin/bash +## +## Exploratory code to run an export using bigquery. +## + + # -# Exploratory code to run an export using bigquery. +# Get command line parameter, which should be an existing +# directory in which to store the results # +TARGET_DIR=$1 +if [[ ! -z "$TARGET_DIR" ]] ; then + echo "$0 Missing parameter" + echo "usage $0 target-dir" + exit 1 +fi + + +if [[ ! -d "$TARGET_DIR" ]] ; then + echo "$0 parameter does not designate an existing directory" + echo "usage $0 target-dir" + exit 1 +fi # # Check for dependencies @@ -42,8 +61,8 @@ fi # Run an export inside the kubernetes cluster, then parse # the output of the script thar ran the export # -TEMPFILE=tmpfile.txt -# mpfile=$(mktemp /tmp/abc-script.XXXXXX) +#TEMPFILE="$(mktemp /tmp/abc-script.XXXXXX)" +TEMPFILE="tmpfile.txt" # kubectl exec -it "${EXPORTER_PODNAME}" -- /bin/bash -c /export_data.sh > "$TEMPFILE" @@ -63,7 +82,6 @@ TEMPFILE=tmpfile.txt EXPORT_ID=$(grep "Starting export job for" $TEMPFILE | awk '{print $5}' | sed 's/\r$//' ) -echo "export id = $EXPORT_ID" PURCHASES_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID-purchases.csv" SUB_2_MSISSDN_MAPPING_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID-sub2msisdn.csv" @@ -73,8 +91,13 @@ CONSUMPTION_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID.csv" # Then copy the CSV files to local storage (current directory) # +gsutil cp $PURCHASES_GS $TARGET_DIR +gsutil cp $SUB_2_MSISSDN_MAPPING_GS $TARGET_DIR +gsutil cp $CONSUMPTION_GS $TARGET_DIR -gsutil cp $PURCHASES_GS . -gsutil cp $SUB_2_MSISSDN_MAPPING_GS . -gsutil cp $CONSUMPTION_GS . - +# +# Finally output the ID of the export, since that's +# what will be used by users of this script to access +# the output +# +echo $EXPORT_ID From 8ee1249d3346018b0205a917f4b122df5ae21c9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 25 Sep 2018 13:26:28 +0200 Subject: [PATCH 23/85] Braino, inverted test --- sample-agent/run-export.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sample-agent/run-export.sh b/sample-agent/run-export.sh index 39b271bc9..659bd885d 100755 --- a/sample-agent/run-export.sh +++ b/sample-agent/run-export.sh @@ -11,7 +11,7 @@ # TARGET_DIR=$1 -if [[ ! -z "$TARGET_DIR" ]] ; then +if [[ -z "$TARGET_DIR" ]] ; then echo "$0 Missing parameter" echo "usage $0 target-dir" exit 1 From d33c7b7f502575d844a3eeaf69310fe8616cb143 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 25 Sep 2018 13:30:14 +0200 Subject: [PATCH 24/85] This seems to do the job now with respect to generating an export --- sample-agent/run-export.sh | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/sample-agent/run-export.sh b/sample-agent/run-export.sh index 659bd885d..801521bf9 100755 --- a/sample-agent/run-export.sh +++ b/sample-agent/run-export.sh @@ -64,25 +64,25 @@ fi #TEMPFILE="$(mktemp /tmp/abc-script.XXXXXX)" TEMPFILE="tmpfile.txt" -# kubectl exec -it "${EXPORTER_PODNAME}" -- /bin/bash -c /export_data.sh > "$TEMPFILE" - -# # Fail if the exec failed -# retVal=$? -# if [ $retVal -ne 0 ]; then -# echo "ERROR: Failed to export data:" -# cat $TMPFILE -# rm $TMPFILE -# exit 1 -# fi +kubectl exec -it "${EXPORTER_PODNAME}" -- /bin/bash -c /export_data.sh > "$TEMPFILE" + +# Fail if the exec failed +retVal=$? +if [ $retVal -ne 0 ]; then + echo "ERROR: Failed to export data:" + cat $TMPFILE + rm $TMPFILE + exit 1 +fi # -# Parse the output of the tmpfile +# Parse the output of the tmpfile, getting the export ID, and +# the google filestore URLs for the output files. # EXPORT_ID=$(grep "Starting export job for" $TEMPFILE | awk '{print $5}' | sed 's/\r$//' ) - PURCHASES_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID-purchases.csv" SUB_2_MSISSDN_MAPPING_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID-sub2msisdn.csv" CONSUMPTION_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID.csv" @@ -95,9 +95,17 @@ gsutil cp $PURCHASES_GS $TARGET_DIR gsutil cp $SUB_2_MSISSDN_MAPPING_GS $TARGET_DIR gsutil cp $CONSUMPTION_GS $TARGET_DIR +# +# Clean up the tempfile +# + +rm "$TEMPFILE" + # # Finally output the ID of the export, since that's # what will be used by users of this script to access # the output # + echo $EXPORT_ID +exit 0 From 654816b0e85d60e7af11aa9f25bc8a6041b67896 Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Tue, 25 Sep 2018 14:30:45 +0200 Subject: [PATCH 25/85] Add new columns to consumption tables. --- dataflow-pipelines/src/main/resources/table_schema.ddl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dataflow-pipelines/src/main/resources/table_schema.ddl b/dataflow-pipelines/src/main/resources/table_schema.ddl index 2e6d229b3..5c13066d3 100644 --- a/dataflow-pipelines/src/main/resources/table_schema.ddl +++ b/dataflow-pipelines/src/main/resources/table_schema.ddl @@ -3,7 +3,9 @@ CREATE TABLE IF NOT EXISTS ( msisdn STRING NOT NULL, bytes INT64 NOT NULL, - timestamp TIMESTAMP NOT NULL + timestamp TIMESTAMP NOT NULL, + apn STRING NOT NULL, + mccMnc STRING NOT NULL ) PARTITION BY DATE(timestamp); @@ -14,6 +16,8 @@ CREATE TABLE IF NOT EXISTS msisdn STRING NOT NULL, bucketBytes INT64 NOT NULL, bundleBytes INT64 NOT NULL, - timestamp TIMESTAMP NOT NULL + timestamp TIMESTAMP NOT NULL, + apn STRING NOT NULL, + mccMnc STRING NOT NULL ) PARTITION BY DATE(timestamp); \ No newline at end of file From 34a5b92ca9a49402c7f565fe078316ed33c9831e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 25 Sep 2018 15:06:26 +0200 Subject: [PATCH 26/85] No de-pseudoizer --- sample-agent/run-export.sh | 6 +++--- sample-agent/sample-agent.sh | 35 ++++++++++++++++++++++++++++++----- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/sample-agent/run-export.sh b/sample-agent/run-export.sh index 801521bf9..5ac31e336 100755 --- a/sample-agent/run-export.sh +++ b/sample-agent/run-export.sh @@ -1,10 +1,11 @@ #!/bin/bash ## -## Exploratory code to run an export using bigquery. +## Run an export, return the identifier for the export, put the +## files from the export in a directory denoted as the single +## command line parameter. ## - # # Get command line parameter, which should be an existing # directory in which to store the results @@ -17,7 +18,6 @@ if [[ -z "$TARGET_DIR" ]] ; then exit 1 fi - if [[ ! -d "$TARGET_DIR" ]] ; then echo "$0 parameter does not designate an existing directory" echo "usage $0 target-dir" diff --git a/sample-agent/sample-agent.sh b/sample-agent/sample-agent.sh index 13611c83c..5b997cd72 100755 --- a/sample-agent/sample-agent.sh +++ b/sample-agent/sample-agent.sh @@ -1,5 +1,17 @@ #!/bin/bash -IMPORTER_URL=http://127.00.1:8080/importer +# IMPORTER_URL=http://127.00.1:8080/importer + + +# +# Figure out where this script is running from +# + + +# Absolute path to this script, e.g. /home/user/bin/foo.sh +SCRIPT=$(readlink -f "$0") +# Absolute path this script is in, thus /home/user/bin +SCRIPTPATH=$(dirname "$SCRIPT") +echo $SCRIPTPATH ## Check that credentials are set up @@ -8,11 +20,20 @@ IMPORTER_URL=http://127.00.1:8080/importer ## Fetch the input data form the exporter -# tbd +EXPORTID=0802c66be1ce4e2dba22f988b3ce24f7 +# EXPORTID=$($SCRIPTPATH/run-export.sh target-dir) + +if [[ -z "$EXPORTID" ]] ; then + echo "$0 Could not determine export ID, bailing out" + exit 1 +fi ## Calculate the output, and put into a file +ALLSUBSCRIBERIDS=$(awk -F, '!/^subscriberId/{print $1'} "target-dir/$EXPORTID-sub2msisdn.csv") +echo $ALLSUBSCRIBERIDS + TMPFILE=tmpfile.yml cat > $TMPFILE <> $TMPFILE ; done + + ## Send it to the importer -curl --data-binary @$TMPFILE $IMPORTER_URL +echo curl --data-binary @$TMPFILE $IMPORTER_URL From f5a401732278ccf526d620832c376489d1c7c60a Mon Sep 17 00:00:00 2001 From: Martin Cederlof Date: Tue, 25 Sep 2018 15:09:02 +0200 Subject: [PATCH 27/85] Added metric for mnc mcc norway --- bq-metrics-extractor/config/config.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/bq-metrics-extractor/config/config.yaml b/bq-metrics-extractor/config/config.yaml index f92f105ce..fc088bd3f 100644 --- a/bq-metrics-extractor/config/config.yaml +++ b/bq-metrics-extractor/config/config.yaml @@ -92,6 +92,26 @@ bqmetrics: WHERE timestamp >= TIMESTAMP_SUB(TIMESTAMP_TRUNC(CURRENT_TIMESTAMP(), DAY), INTERVAL 1 DAY) AND timestamp < TIMESTAMP_TRUNC(CURRENT_TIMESTAMP(), DAY) ), 0) as count + - type: gauge + name: total_data_used_today_mnc_242_mcc_01 + help: Total data used today MNC 242 MCC 01 + resultColumn: count + sql: > + SELECT COALESCE ( + (SELECT sum(bucketBytes) AS count FROM `pantel-2decb.data_consumption.raw_consumption` + WHERE timestamp >= TIMESTAMP_TRUNC(CURRENT_TIMESTAMP(), DAY) + AND mccMnc = "24201"), 0) as count + - type: gauge + name: total_data_used_yesterday_mnc_242_mcc_01 + help: Total data used yesterday MNC 242 MCC 01 + resultColumn: count + sql: > + SELECT COALESCE ( + ( SELECT sum(bucketBytes) AS count FROM `pantel-2decb.data_consumption.raw_consumption` + WHERE timestamp >= TIMESTAMP_SUB(TIMESTAMP_TRUNC(CURRENT_TIMESTAMP(), DAY), INTERVAL 1 DAY) + AND timestamp < TIMESTAMP_TRUNC(CURRENT_TIMESTAMP(), DAY) + AND mccMnc = "24201"), 0) as count + - type: gauge name: revenue_today help: Revenue generated today From 53019bb26bdfb26002b65cd435d222c05c127628 Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Tue, 25 Sep 2018 15:26:45 +0200 Subject: [PATCH 28/85] Fixes comment in 'sort sources in desceding' order code --- .../prime/paymentprocessor/StripePaymentProcessor.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt index 2af72ecb4..e54a7ac44 100644 --- a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt +++ b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt @@ -70,8 +70,10 @@ class StripePaymentProcessor : PaymentProcessor { } } - /* As Stripe returns stored metadata as String, check for and return the 'created' timestamp from metadata. If not - found log and return current time as the timestamp. */ + /* Handle type conversion when reading the 'created' field from the + metadata returned from Stripe. (It might seem like that Stripe + returns stored metadata values as strings, even if they where stored + using an another type. Needs to be verified.) */ private fun getCreatedTimestampFromMetadata(id: String, metadata: Map) : Long { val created: String? = metadata.get("created") as? String return created?.toLongOrNull() ?: run { From e26b2314f9be5f08cf99a1dc4755a770df45b8df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 25 Sep 2018 15:32:31 +0200 Subject: [PATCH 29/85] Hacked-together version of full traversal (closed loop) that might just work --- sample-agent/sample-agent.sh | 39 ++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/sample-agent/sample-agent.sh b/sample-agent/sample-agent.sh index 5b997cd72..9251aee97 100755 --- a/sample-agent/sample-agent.sh +++ b/sample-agent/sample-agent.sh @@ -20,20 +20,43 @@ echo $SCRIPTPATH ## Fetch the input data form the exporter -EXPORTID=0802c66be1ce4e2dba22f988b3ce24f7 -# EXPORTID=$($SCRIPTPATH/run-export.sh target-dir) +EXPORT_ID=0802c66be1ce4e2dba22f988b3ce24f7 +# EXPORT_ID=$($SCRIPTPATH/run-export.sh target-dir) -if [[ -z "$EXPORTID" ]] ; then +if [[ -z "$EXPORT_ID" ]] ; then echo "$0 Could not determine export ID, bailing out" exit 1 fi ## Calculate the output, and put into a file +## that will be a single-column CSV file containing the +## members of the updated segment + +PROJECT_ID=$(gcloud config get-value project) + +PURCHASES_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID-purchases.csv" +SUB_2_MSISSDN_MAPPING_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID-sub2msisdn.csv" +CONSUMPTION_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID.csv" +RESULT_SEGMENT_PSEUDO_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID-resultsegment-pseudoanonymized.csv" +RESULT_SEGMENT_CLEAR_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID-resultsegment-cleartext.csv" + +SEGMENT_TMPFILE_PSEUDO="tmpsegment-pseudo.csv" +SEGMENT_TMPFILE_CLEAR="tmpsegment-clear.csv" +awk -F, '!/^subscriberId/{print $1'} "target-dir/$EXPORT_ID-sub2msisdn.csv" > $SEGMENT_TMPFILE_PSEUDO +gsutil cp $SEGMENT_TMPFILE_PSEUDO $RESULT_SEGMENT_PSEUDO_GS + +## Run some script to make sure that we can get deanonumized pseudothing. +## At this point we give the actual content of that file, since we copy it back +## but eventually we may in fact send the URL instead of the actual data, letting +## the Prime read the dataset from google cloud storage instead. + +## (so we should rally copy back $RESULT_SEGMENT_CLEARTEXT_GS insted of the _PSEUDO_ +## file) + + +gsutil cp $RESULT_SEGMENT_PSEUDO_GS $SEGMENT_TMPFILE_CLEAR -ALLSUBSCRIBERIDS=$(awk -F, '!/^subscriberId/{print $1'} "target-dir/$EXPORTID-sub2msisdn.csv") -echo $ALLSUBSCRIBERIDS - TMPFILE=tmpfile.yml cat > $TMPFILE <> $TMPFILE ; done - +awk '{print " - " $1}' $SEGMENT_TMPFILE_CLEAR >> $TMPFILE +# for x in $ALLSUBSCRIBERIDS ; do echo " - $x" >> $TMPFILE ; done ## Send it to the importer From 27ab3a348dbfb7fb81b2c26e43133318b7fe0c9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 25 Sep 2018 16:04:42 +0200 Subject: [PATCH 30/85] Refactoring a little, calculating gs coordinates in separate file, also figuring out google environment in separate file --- ...ependencies_get_environment_coordinates.sh | 38 +++++++++++++++++ sample-agent/run-export.sh | 42 +++++-------------- sample-agent/sample-agent.sh | 34 ++++++--------- sample-agent/set-gs-names.sh | 20 +++++++++ 4 files changed, 80 insertions(+), 54 deletions(-) create mode 100644 sample-agent/check_dependencies_get_environment_coordinates.sh create mode 100644 sample-agent/set-gs-names.sh diff --git a/sample-agent/check_dependencies_get_environment_coordinates.sh b/sample-agent/check_dependencies_get_environment_coordinates.sh new file mode 100644 index 000000000..20b1a7c07 --- /dev/null +++ b/sample-agent/check_dependencies_get_environment_coordinates.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +## Intended to be sourced by other programs + + +# +# Check for dependencies +# + +DEPENDENCIES="gcloud kubectl gsutil" + +for dep in $DEPENDENCIES ; do + if [[ -z $(which $dep) ]] ; then + echo "ERROR: Could not find dependency $dep" + fi +done + +# +# Figure out relevant parts of the environment and check their +# sanity. +# + + +PROJECT_ID=$(gcloud config get-value project) + +if [[ -z "$PROJECT_ID" ]] ; then + echo "ERROR: Unknown google project ID" + exit 1 +fi + + +EXPORTER_PODNAME=$(kubectl get pods | grep exporter- | awk '{print $1}') +if [[ -z "$EXPORTER_PODNAME" ]] ; then + echo "ERROR: Unknown exporter podname" + exit 1 +fi + + diff --git a/sample-agent/run-export.sh b/sample-agent/run-export.sh index 5ac31e336..b7fb26826 100755 --- a/sample-agent/run-export.sh +++ b/sample-agent/run-export.sh @@ -6,6 +6,15 @@ ## command line parameter. ## + + +# Absolute path to this script, e.g. /home/user/bin/foo.sh +SCRIPT=$(readlink -f "$0") +# Absolute path this script is in, thus /home/user/bin +SCRIPTPATH=$(dirname "$SCRIPT") +echo $SCRIPTPATH + + # # Get command line parameter, which should be an existing # directory in which to store the results @@ -24,38 +33,7 @@ if [[ ! -d "$TARGET_DIR" ]] ; then exit 1 fi -# -# Check for dependencies -# - -DEPENDENCIES="gcloud kubectl gsutil" - -for dep in $DEPENDENCIES ; do - if [[ -z $(which $dep) ]] ; then - echo "ERROR: Could not find dependency $dep" - fi -done - -# -# Figure out relevant parts of the environment and check their -# sanity. -# - - -PROJECT_ID=$(gcloud config get-value project) - -if [[ -z "$PROJECT_ID" ]] ; then - echo "ERROR: Unknown google project ID" - exit 1 -fi - - -EXPORTER_PODNAME=$(kubectl get pods | grep exporter- | awk '{print $1}') -if [[ -z "$EXPORTER_PODNAME" ]] ; then - echo "ERROR: Unknown exporter podname" - exit 1 -fi - +$SCRIPTPATH/check_dependencies_get_environment_coordinates.sh # # Run an export inside the kubernetes cluster, then parse diff --git a/sample-agent/sample-agent.sh b/sample-agent/sample-agent.sh index 9251aee97..f23e60e64 100755 --- a/sample-agent/sample-agent.sh +++ b/sample-agent/sample-agent.sh @@ -14,6 +14,13 @@ SCRIPTPATH=$(dirname "$SCRIPT") echo $SCRIPTPATH +# +# Get coordinates telling the script where in this world +# it is, and what files it should relate to and so on. +# + +. $SCRIPTPATH/check_dependencies_get_environment_coordinates.sh + ## Check that credentials are set up # tbd @@ -23,22 +30,7 @@ echo $SCRIPTPATH EXPORT_ID=0802c66be1ce4e2dba22f988b3ce24f7 # EXPORT_ID=$($SCRIPTPATH/run-export.sh target-dir) -if [[ -z "$EXPORT_ID" ]] ; then - echo "$0 Could not determine export ID, bailing out" - exit 1 -fi - -## Calculate the output, and put into a file -## that will be a single-column CSV file containing the -## members of the updated segment - -PROJECT_ID=$(gcloud config get-value project) - -PURCHASES_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID-purchases.csv" -SUB_2_MSISSDN_MAPPING_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID-sub2msisdn.csv" -CONSUMPTION_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID.csv" -RESULT_SEGMENT_PSEUDO_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID-resultsegment-pseudoanonymized.csv" -RESULT_SEGMENT_CLEAR_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID-resultsegment-cleartext.csv" +. $SCRIPTPATH/check_dependencies_get_environment_coordinates.sh SEGMENT_TMPFILE_PSEUDO="tmpsegment-pseudo.csv" SEGMENT_TMPFILE_CLEAR="tmpsegment-clear.csv" @@ -57,9 +49,9 @@ gsutil cp $SEGMENT_TMPFILE_PSEUDO $RESULT_SEGMENT_PSEUDO_GS gsutil cp $RESULT_SEGMENT_PSEUDO_GS $SEGMENT_TMPFILE_CLEAR -TMPFILE=tmpfile.yml +IMPORTFILE_YML=tmpfile.yml -cat > $TMPFILE < $IMPORTFILE_YML <> $TMPFILE -# for x in $ALLSUBSCRIBERIDS ; do echo " - $x" >> $TMPFILE ; done - +awk '{print " - " $1}' $SEGMENT_IMPORTFILE_CLEAR >> $IMPORTFILE_YML ## Send it to the importer -echo curl --data-binary @$TMPFILE $IMPORTER_URL +echo curl --data-binary @$IMPORTFILE_YML $IMPORTER_URL diff --git a/sample-agent/set-gs-names.sh b/sample-agent/set-gs-names.sh new file mode 100644 index 000000000..34075a239 --- /dev/null +++ b/sample-agent/set-gs-names.sh @@ -0,0 +1,20 @@ +#!/bin/bash + + +if [[ -z "$PROJECT_ID" ]] ; then + echo "$0 PROJECT_ID variable not set, cannot determine google filestore coordinates" + exit 1 +fi + + +if [[ -z "$EXPORT_ID" ]] ; then + echo "$0 EXPORT_ID variable not set, cannot determine google filestore coordinates" + exit 1 +fi + + +PURCHASES_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID-purchases.csv" +SUB_2_MSISSDN_MAPPING_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID-sub2msisdn.csv" +CONSUMPTION_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID.csv" +RESULT_SEGMENT_PSEUDO_GS="gs://${PROJECT_ID}-dataconsumption-export/${EXPORT_ID}-resultsegment-pseudoanonymized.csv" +RESULT_SEGMENT_CLEAR_GS="gs://${PROJECT_ID}-dataconsumption-export/${EXPORT_ID}-resultsegment-cleartext.csv" From fc255fab49526b41f441d82506705faa647785eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 25 Sep 2018 16:12:13 +0200 Subject: [PATCH 31/85] Refactoring for fun and profit. Awaiting the latest importer code to remove pseudo-anonymization --- ...ependencies_get_environment_coordinates.sh | 35 ++++++++++--------- sample-agent/sample-agent.sh | 12 ++----- 2 files changed, 22 insertions(+), 25 deletions(-) diff --git a/sample-agent/check_dependencies_get_environment_coordinates.sh b/sample-agent/check_dependencies_get_environment_coordinates.sh index 20b1a7c07..7e4ff7a18 100644 --- a/sample-agent/check_dependencies_get_environment_coordinates.sh +++ b/sample-agent/check_dependencies_get_environment_coordinates.sh @@ -7,32 +7,35 @@ # Check for dependencies # -DEPENDENCIES="gcloud kubectl gsutil" +if [[ -z "$DEPENDENCIES ]] ; then -for dep in $DEPENDENCIES ; do - if [[ -z $(which $dep) ]] ; then + DEPENDENCIES="gcloud kubectl gsutil" + + for dep in $DEPENDENCIES ; do + if [[ -z $(which $dep) ]] ; then echo "ERROR: Could not find dependency $dep" - fi -done + fi + done +fi # # Figure out relevant parts of the environment and check their # sanity. # +if [[ -z "$PROJECT_ID" ]] ; then + PROJECT_ID=$(gcloud config get-value project) -PROJECT_ID=$(gcloud config get-value project) - -if [[ -z "$PROJECT_ID" ]] ; then - echo "ERROR: Unknown google project ID" - exit 1 + if [[ -z "$PROJECT_ID" ]] ; then + echo "ERROR: Unknown google project ID" + exit 1 + fi fi - -EXPORTER_PODNAME=$(kubectl get pods | grep exporter- | awk '{print $1}') if [[ -z "$EXPORTER_PODNAME" ]] ; then - echo "ERROR: Unknown exporter podname" - exit 1 + EXPORTER_PODNAME=$(kubectl get pods | grep exporter- | awk '{print $1}') + if [[ -z "$EXPORTER_PODNAME" ]] ; then + echo "ERROR: Unknown exporter podname" + exit 1 + fi fi - - diff --git a/sample-agent/sample-agent.sh b/sample-agent/sample-agent.sh index f23e60e64..34d3901c0 100755 --- a/sample-agent/sample-agent.sh +++ b/sample-agent/sample-agent.sh @@ -6,10 +6,7 @@ # Figure out where this script is running from # - -# Absolute path to this script, e.g. /home/user/bin/foo.sh SCRIPT=$(readlink -f "$0") -# Absolute path this script is in, thus /home/user/bin SCRIPTPATH=$(dirname "$SCRIPT") echo $SCRIPTPATH @@ -21,10 +18,6 @@ echo $SCRIPTPATH . $SCRIPTPATH/check_dependencies_get_environment_coordinates.sh -## Check that credentials are set up - - # tbd - ## Fetch the input data form the exporter EXPORT_ID=0802c66be1ce4e2dba22f988b3ce24f7 @@ -45,10 +38,8 @@ gsutil cp $SEGMENT_TMPFILE_PSEUDO $RESULT_SEGMENT_PSEUDO_GS ## (so we should rally copy back $RESULT_SEGMENT_CLEARTEXT_GS insted of the _PSEUDO_ ## file) - gsutil cp $RESULT_SEGMENT_PSEUDO_GS $SEGMENT_TMPFILE_CLEAR - IMPORTFILE_YML=tmpfile.yml cat > $IMPORTFILE_YML <> $IMPORTFILE_YML ## Send it to the importer echo curl --data-binary @$IMPORTFILE_YML $IMPORTER_URL + +rm $SEGMENT_TMPFILE_PSEUDO +rm $SEGMENT_TMPFILE_CLEAR From 52a368af8c62289c8dd0cef4237b13ea506253e9 Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Tue, 25 Sep 2018 17:11:21 +0200 Subject: [PATCH 32/85] Add reverse lookup for subscribers. --- exporter/Dockerfile | 3 ++ exporter/script/map_subscribers.sh | 42 ++++++++++++++++++++++++++ exporter/script/subscriber-schema.json | 7 +++++ 3 files changed, 52 insertions(+) create mode 100644 exporter/script/map_subscribers.sh create mode 100644 exporter/script/subscriber-schema.json diff --git a/exporter/Dockerfile b/exporter/Dockerfile index 9e003a63c..471f1ecb7 100644 --- a/exporter/Dockerfile +++ b/exporter/Dockerfile @@ -18,9 +18,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY script/idle.sh /idle.sh COPY script/export_data.sh /export_data.sh COPY script/delete_export_data.sh /delete_export_data.sh +COPY script/map_subscribers.sh /map_subscribers.sh +COPY script/subscriber-schema.json /subscriber-schema.json RUN chmod +x /idle.sh RUN chmod +x /export_data.sh RUN chmod +x /delete_export_data.sh +RUN chmod +x /map_subscribers.sh CMD ["/idle.sh"] \ No newline at end of file diff --git a/exporter/script/map_subscribers.sh b/exporter/script/map_subscribers.sh new file mode 100644 index 000000000..1c38c5f7b --- /dev/null +++ b/exporter/script/map_subscribers.sh @@ -0,0 +1,42 @@ +#!/bin/bash +#set -x + +exportId=$1 +if [ -z "$1" ]; then + echo "To convert subscribers, specify the id of the export operation" + exit +fi +exportId=${exportId//-} +exportId=${exportId,,} +projectId=pantel-2decb + +csvfile=$projectId-dataconsumption-export/${exportId}-resultsegment-pseudoanonymized.csv +outputCsvfile=$projectId-dataconsumption-export/${exportId}-resultsegment-cleartext.csv +inputSubscriberTable=exported_pseudonyms.${exportId}_pseudo_subscriber +subscriberPseudonymsTable=exported_pseudonyms.${exportId}_subscriber +outputSubscriberTable=exported_pseudonyms.${exportId}_clear_subscriber + + +echo "Importing data from csv $csvfile" +bq --location=EU load --replace --source_format=CSV $projectId:$inputSubscriberTable gs://$csvfile /subscriber-schema.json +echo "Exported data to $inputSubscriberTable" + +echo "Creating table $dataConsumptionTable" +# SQL for joining pseudonym & hourly consumption tables. +read -r -d '' sqlForJoin << EOM +SELECT + DISTINCT(sub.subscriberId), ps.subscriberId as email +FROM + \`$inputSubscriberTable\` as sub +JOIN + \`$subscriberPseudonymsTable\` as ps +ON ps.pseudoid = sub.subscriberId +EOM + +# Run the query using bq & dump results to the new table +bq --location=EU --format=none query --destination_table $outputSubscriberTable --replace --use_legacy_sql=false $sqlForJoin +echo "Created table $outputSubscriberTable" + +echo "Exporting data to csv $outputCsvfile" +bq --location=EU extract --destination_format=CSV $outputSubscriberTable gs://$outputCsvfile +echo "Exported data to gs://$outputCsvfile" diff --git a/exporter/script/subscriber-schema.json b/exporter/script/subscriber-schema.json new file mode 100644 index 000000000..e5ab3cbce --- /dev/null +++ b/exporter/script/subscriber-schema.json @@ -0,0 +1,7 @@ +[ + { + "mode": "REQUIRED", + "name": "subscriberId", + "type": "STRING" + } +] From e7d0b52f10384bcd49fea92e41ad2f702bb5b285 Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Tue, 25 Sep 2018 17:44:10 +0200 Subject: [PATCH 33/85] Change column names --- exporter/script/map_subscribers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/script/map_subscribers.sh b/exporter/script/map_subscribers.sh index 1c38c5f7b..38a8c1dfd 100644 --- a/exporter/script/map_subscribers.sh +++ b/exporter/script/map_subscribers.sh @@ -25,7 +25,7 @@ echo "Creating table $dataConsumptionTable" # SQL for joining pseudonym & hourly consumption tables. read -r -d '' sqlForJoin << EOM SELECT - DISTINCT(sub.subscriberId), ps.subscriberId as email + DISTINCT(sub.subscriberId) as pseudoId, ps.subscriberId FROM \`$inputSubscriberTable\` as sub JOIN From fc573010dccf2553968941fd88ea18bdd692062b Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Tue, 25 Sep 2018 17:44:54 +0200 Subject: [PATCH 34/85] Fix log line --- exporter/script/map_subscribers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/script/map_subscribers.sh b/exporter/script/map_subscribers.sh index 38a8c1dfd..9a34ad9e5 100644 --- a/exporter/script/map_subscribers.sh +++ b/exporter/script/map_subscribers.sh @@ -21,7 +21,7 @@ echo "Importing data from csv $csvfile" bq --location=EU load --replace --source_format=CSV $projectId:$inputSubscriberTable gs://$csvfile /subscriber-schema.json echo "Exported data to $inputSubscriberTable" -echo "Creating table $dataConsumptionTable" +echo "Creating table $outputSubscriberTable" # SQL for joining pseudonym & hourly consumption tables. read -r -d '' sqlForJoin << EOM SELECT From 17446dbf82890bf1f3bf1bb2fee3ca2ef62f2fed Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Tue, 25 Sep 2018 17:52:54 +0200 Subject: [PATCH 35/85] Add URLDECODE function --- exporter/script/map_subscribers.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/exporter/script/map_subscribers.sh b/exporter/script/map_subscribers.sh index 9a34ad9e5..901a4854b 100644 --- a/exporter/script/map_subscribers.sh +++ b/exporter/script/map_subscribers.sh @@ -24,8 +24,16 @@ echo "Exported data to $inputSubscriberTable" echo "Creating table $outputSubscriberTable" # SQL for joining pseudonym & hourly consumption tables. read -r -d '' sqlForJoin << EOM +CREATE TEMP FUNCTION URLDECODE(url STRING) AS (( + SELECT SAFE_CONVERT_BYTES_TO_STRING( + ARRAY_TO_STRING(ARRAY_AGG( + IF(STARTS_WITH(y, '%'), FROM_HEX(SUBSTR(y, 2)), CAST(y AS BYTES)) ORDER BY i + ), b'')) + FROM UNNEST(REGEXP_EXTRACT_ALL(url, r"%[0-9a-fA-F]{2}|[^%]+")) AS y WITH OFFSET AS i +)); + SELECT - DISTINCT(sub.subscriberId) as pseudoId, ps.subscriberId + DISTINCT(sub.subscriberId) as pseudoId, URLDECODE(ps.subscriberId) FROM \`$inputSubscriberTable\` as sub JOIN From d10c9a100941e5663ec5596a2674a122fc5d0a46 Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Tue, 25 Sep 2018 17:55:31 +0200 Subject: [PATCH 36/85] add coulmn name --- exporter/script/map_subscribers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/script/map_subscribers.sh b/exporter/script/map_subscribers.sh index 901a4854b..98400bf9f 100644 --- a/exporter/script/map_subscribers.sh +++ b/exporter/script/map_subscribers.sh @@ -33,7 +33,7 @@ CREATE TEMP FUNCTION URLDECODE(url STRING) AS (( )); SELECT - DISTINCT(sub.subscriberId) as pseudoId, URLDECODE(ps.subscriberId) + DISTINCT(sub.subscriberId) as pseudoId, URLDECODE(ps.subscriberId) as subscriberId FROM \`$inputSubscriberTable\` as sub JOIN From c2252de49ae86d27ca542ad0ff058523a85eaf1b Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Tue, 25 Sep 2018 18:04:16 +0200 Subject: [PATCH 37/85] Update documentation. --- exporter/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/exporter/README.md b/exporter/README.md index cd8e306bc..27e4df869 100644 --- a/exporter/README.md +++ b/exporter/README.md @@ -31,6 +31,16 @@ kubectl exec -it -- /bin/bash # Run exporter from the above shell /export_data.sh +# This results in 3 csv files in GCS +1) Data consumption Records: gs://pantel-2decb-dataconsumption-export/.csv +2) Purchase Records: gs://pantel-2decb-dataconsumption-export/-purchases.csv +3) Subscriber to MSISDN mappings: gs://pantel-2decb-dataconsumption-export/-sub2msisdn.csv + +# Run subsciber reverse lookup from the above shell +/map_subscribers.sh + +# Delete all tables and files for an export +/delete_export_data.sh # Delete deployment kubectl delete deployment exporter From 519d63d9f79377ae8e6763087e6344212c1e66ee Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Wed, 26 Sep 2018 09:02:17 +0200 Subject: [PATCH 38/85] Fixes incorrect package name in neo4j unit test --- .../kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt index 07b7a06de..3de14305f 100644 --- a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt +++ b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt @@ -48,7 +48,7 @@ class GraphStoreTest { price = Price(0, CURRENCY), properties = mapOf("noOfBytes" to "1_000_000_000"))) - val allSegment = Segment(id = Neo4jStoreSingleton.getSegmentNameFromCountryCode(COUNTRY)) + val allSegment = Segment(id = getSegmentNameFromCountryCode(COUNTRY)) Neo4jStoreSingleton.createSegment(allSegment) } From 92735aac3ea9875f8bbdd7a0af1b35f77cee29fa Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Wed, 26 Sep 2018 09:22:33 +0200 Subject: [PATCH 39/85] Remove the extra tables created for lookup --- exporter/script/delete_export_data.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/exporter/script/delete_export_data.sh b/exporter/script/delete_export_data.sh index 2d59e7e2b..740c99fa9 100644 --- a/exporter/script/delete_export_data.sh +++ b/exporter/script/delete_export_data.sh @@ -19,6 +19,9 @@ csvfile=$projectId-dataconsumption-export/$exportId.csv purchasesCsvfile=$projectId-dataconsumption-export/$exportId-purchases.csv sub2msisdnCsvfile=$projectId-dataconsumption-export/$exportId-sub2msisdn.csv +inputSubscriberTable=exported_pseudonyms.${exportId}_pseudo_subscriber +outputSubscriberTable=exported_pseudonyms.${exportId}_clear_subscriber + echo "Cleaning all data for export $exportId" echo "Deleting Table $msisdnPseudonymsTable" bq rm -f -t $msisdnPseudonymsTable @@ -35,6 +38,12 @@ bq rm -f -t $dataConsumptionTable echo "Deleting Table $purchaseRecordsTable" bq rm -f -t $purchaseRecordsTable +echo "Deleting Table $inputSubscriberTable" +bq rm -f -t $inputSubscriberTable + +echo "Deleting Table $outputSubscriberTable" +bq rm -f -t $outputSubscriberTable + echo "Deleting csv gs://$csvfile" gsutil rm gs://$csvfile From f233a85a24c039b4a864d57515fe64d5561c3dfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 26 Sep 2018 13:06:24 +0200 Subject: [PATCH 40/85] Comments --- sample-agent/sample-agent.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sample-agent/sample-agent.sh b/sample-agent/sample-agent.sh index 34d3901c0..9b72fc8d4 100755 --- a/sample-agent/sample-agent.sh +++ b/sample-agent/sample-agent.sh @@ -59,7 +59,8 @@ segment: subscribers: EOF - +# Adding the list of subscribers in clear text (indented six spaces +# with a leading "-" as per YAML list syntax. awk '{print " - " $1}' $SEGMENT_IMPORTFILE_CLEAR >> $IMPORTFILE_YML ## Send it to the importer From 59d6bbc11ce7cc7f4b91db7c2a4b82a87656b252 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Tue, 25 Sep 2018 15:49:21 +0200 Subject: [PATCH 41/85] Fixes --- .../org/ostelco/prime/admin/api/Resources.kt | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/admin-api/src/main/kotlin/org/ostelco/prime/admin/api/Resources.kt b/admin-api/src/main/kotlin/org/ostelco/prime/admin/api/Resources.kt index dd2477fc8..04714845c 100644 --- a/admin-api/src/main/kotlin/org/ostelco/prime/admin/api/Resources.kt +++ b/admin-api/src/main/kotlin/org/ostelco/prime/admin/api/Resources.kt @@ -7,6 +7,7 @@ import org.ostelco.prime.model.ProductClass import org.ostelco.prime.model.Segment import org.ostelco.prime.module.getResource import org.ostelco.prime.storage.AdminDataSource +import javax.ws.rs.DELETE import javax.ws.rs.POST import javax.ws.rs.PUT import javax.ws.rs.Path @@ -69,6 +70,9 @@ class SegmentResource { // @Path("/{segment-id}") // fun getSegment(@PathParam("segment-id") segmentId: String) = adminDataSource.getSegment(segmentId) + /** + * Create new [Segment] + */ @POST fun createSegment(segment: Segment): Response { return adminDataSource.createSegment(segment) @@ -76,6 +80,9 @@ class SegmentResource { { Response.status(Response.Status.CREATED).build() }) } + /** + * Update existing [Segment]. Replace existing subscriber list with new list. + */ @PUT @Path("/{segment-id}") fun updateSegment( @@ -94,6 +101,24 @@ class SegmentResource { { Response.ok().build() }) } + /** + * Add individual subscriber to a [Segment] + */ + @POST + @Path("/{segment-id}/subscriber/{subscriber-id}") + fun addSubscriberToSegment(segment: Segment): Response { + TODO("Vihang: Needs implementation") + } + + /** + * Add individual subscriber to a [Segment] + */ + @DELETE + @Path("/{segment-id}/subscriber/{subscriber-id}") + fun removeSubscriberFromSegment(segment: Segment): Response { + TODO("Vihang: Needs implementation") + } + // private fun toStoredSegment(segment: Segment): org.ostelco.prime.model.Segment { // return org.ostelco.prime.model.Segment( // segment.id, From 53d8060ce483eb9bf3c23b9b0917af63d0ea6856 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 26 Sep 2018 14:06:07 +0200 Subject: [PATCH 42/85] Work in progress, untested, don't try this yet. Needs to be stepped up to working status one bit at a time, most pieces should mostly be in place though --- sample-agent/sample-agent.sh | 183 +++++++++++++++++++++++++++++++++-- 1 file changed, 175 insertions(+), 8 deletions(-) diff --git a/sample-agent/sample-agent.sh b/sample-agent/sample-agent.sh index 9b72fc8d4..9b18abac9 100755 --- a/sample-agent/sample-agent.sh +++ b/sample-agent/sample-agent.sh @@ -1,6 +1,20 @@ #!/bin/bash # IMPORTER_URL=http://127.00.1:8080/importer +# +## Todo: +## o Refactor using methods, adding comments and structure +## o Make it work (again). +## o add invocation of map_subscribers.sh in the pseudoanonymizer-pod to get +## a result back, using that for generating the yaml. +## o Declare victory with respect to closing the loop, merge the branch, +## then start improving the loop on a daily basis. + + + +### +### PRELIMINARIES +### # # Figure out where this script is running from @@ -12,24 +26,177 @@ echo $SCRIPTPATH # -# Get coordinates telling the script where in this world -# it is, and what files it should relate to and so on. +# Check for dependencies being satisfied +# + +DEPENDENCIES="gcloud kubectl gsutil" + +for dep in $DEPENDENCIES ; do + if [[ -z $(which $dep) ]] ; then + echo "ERROR: Could not find dependency $dep" + fi +done + + +# +# Figure out relevant parts of the environment and check their +# sanity. +# + +PROJECT_ID=$(gcloud config get-value project) + +if [[ -z "$PROJECT_ID" ]] ; then + echo "ERROR: Unknown google project ID" + exit 1 +fi + +EXPORTER_PODNAME=$(kubectl get pods | grep exporter- | awk '{print $1}') +if [[ -z "$EXPORTER_PODNAME" ]] ; then + echo "ERROR: Unknown exporter podname" + exit 1 +fi + + + +### +### VALIDATING AND PARSING COMMAND LINE PARAMETERS +### + + +# +# Get command line parameter, which should be an existing +# directory in which to store the results +# + +TARGET_DIR=$1 +if [[ -z "$TARGET_DIR" ]] ; then + echo "$0 Missing parameter" + echo "usage $0 target-dir" + exit 1 +fi + +if [[ ! -d "$TARGET_DIR" ]] ; then + echo "$0 parameter does not designate an existing directory" + echo "usage $0 target-dir" + exit 1 +fi + + +### +### COMMUNICATION WITH EXPORTER SCRIPTS RUNNING IN A KUBERNETES PODS +### + + + +# +# Run named script on the inside of the kubernetes exporter pod, +# put the output from running that script into a temporary file, return the +# name of that temporary file as the result of running the function. +# The second argument is the intent of the invocation, and is used +# when producing error messages: +# +# runScriptOnExporterPod /export_data.sh "export data" +# +function runScriptOnExportrerPod { + local scriptname=$1 + local intentDescription=$2 + + #TEMPFILE="$(mktemp /tmp/abc-script.XXXXXX)" + TEMPFILE="tmpfile.txt" + + + kubectl exec -it "${EXPORTER_PODNAME}" -- /bin/bash -c "$scriptname" > "$TEMPFILE" + + # Fail if the exec failed + retVal=$? + if [[ $retVal -ne 0 ]]; then + echo "ERROR: Failed to $intentDescription" + cat $TMPFILE + rm $TMPFILE + exit 1 + fi + echo $TMPFILE +} + +# +# Create a data export batch, return a string identifying that +# batch. Typical usage: +# EXPORT_ID=$(exportDataFromExporterPad) +# +function exportDataFromExporterPod { + local tmpfilename=$(runScriptOnExporterPod /export_data.sh "export data") + local exportId=$(grep "Starting export job for" $tmpfilename | awk '{print $5}' | sed 's/\r$//' ) + if [[ -z "$exportId" ]] ; then + echo "$0 Could not get export batch from exporter pod" + fi + echo exportId +} + +function mapPseudosToUserids { + local tmpfile=$(runScriptOnExporterPod /map_subscribers.sh "mapping pseudoids to subscriber ids") + + # XXX Map, then transform$(runScriptOnExporterPod /export_data.sh "export data")$(runScriptOnExporterPod /export_data.sh "export data") +} + +# +# Generate the Google filesystem names of components associated with +# a particular export ID: Typical usage +# +# PURCHASES_GS=$(gsExportCsvFilename "purchases") + +function gsExportCsvFilename { + local componentName=$1 + if [[ -n "$componentName" ]] ; then + componentName="-$componentName" + fi + return "gs://${PROJECT_ID}-dataconsumption-export/${EXPORT_ID}${componentName}.csv" +} + + + +function importedCsvFilename { + local importDirectory=$1 + local componentName=$2 + if [[ -n "$componentName" ]] ; then + componentName="-$componentName" + fi + return "${importDirectory}/${EXPORT_ID}${componentName}.csv" +} + + +### +### MAIN SCRIPT +### + +# EXPORT_ID=0802c66be1ce4e2dba22f988b3ce24f7 +EXPORT_ID=$(exportDataFromExporterPad) + + +# +# Get the IDs of the various parts being exported # -. $SCRIPTPATH/check_dependencies_get_environment_coordinates.sh -## Fetch the input data form the exporter +for component in "purchases" "sub2msisdn" "" ; do + gsutil cp $(gsExportCsvFilename $component) $(importedCsvFilename $TARGET_DIR $component) +done + -EXPORT_ID=0802c66be1ce4e2dba22f988b3ce24f7 -# EXPORT_ID=$($SCRIPTPATH/run-export.sh target-dir) +echo $EXPORT_ID + + +## +## Generate the yaml output +## -. $SCRIPTPATH/check_dependencies_get_environment_coordinates.sh SEGMENT_TMPFILE_PSEUDO="tmpsegment-pseudo.csv" SEGMENT_TMPFILE_CLEAR="tmpsegment-clear.csv" -awk -F, '!/^subscriberId/{print $1'} "target-dir/$EXPORT_ID-sub2msisdn.csv" > $SEGMENT_TMPFILE_PSEUDO +awk -F, '!/^subscriberId/{print $1'} $(importedCsvVilename $TARGET_DIR "sub2msisdn") > $SEGMENT_TMPFILE_PSEUDO gsutil cp $SEGMENT_TMPFILE_PSEUDO $RESULT_SEGMENT_PSEUDO_GS +mapPseudosToUserids # Or some such + ## Run some script to make sure that we can get deanonumized pseudothing. ## At this point we give the actual content of that file, since we copy it back ## but eventually we may in fact send the URL instead of the actual data, letting From 5afcc730d9b60e193a49323906624d7be5683b96 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Tue, 25 Sep 2018 20:56:49 +0200 Subject: [PATCH 43/85] Neo4j transaction extension with reversal and final actions. --- .../ostelco/prime/storage/graph/Neo4jStore.kt | 127 ++++++++++-------- .../prime/storage/graph/PrimeTransaction.kt | 73 ++++++++++ .../org/ostelco/prime/storage/graph/Schema.kt | 8 +- 3 files changed, 146 insertions(+), 62 deletions(-) create mode 100644 neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/PrimeTransaction.kt diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index b4ac4c111..ad21ea340 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -1,7 +1,7 @@ package org.ostelco.prime.storage.graph import arrow.core.Either -import arrow.core.Tuple4 +import arrow.core.Tuple3 import arrow.core.flatMap import org.neo4j.driver.v1.Transaction import org.ostelco.prime.analytics.AnalyticsService @@ -357,7 +357,7 @@ object Neo4jStoreSingleton : GraphStore { sourceId: String?, saveCard: Boolean): Either = writeTransaction { - val result = getProduct(subscriberId, sku, transaction) + getProduct(subscriberId, sku, transaction) // If we can't find the product, return not-found .mapLeft { org.ostelco.prime.paymentprocessor.core.NotFoundError("Product unavailable") } .flatMap { product: Product -> @@ -367,40 +367,52 @@ object Neo4jStoreSingleton : GraphStore { { paymentProcessor.createPaymentProfile(subscriberId) }, { profileInfo -> Either.right(profileInfo) } ) - .map { profileInfo -> Pair(product, profileInfo) } + .map { profileInfo -> Pair(product, profileInfo.id) } } - .flatMap { (product, profileInfo) -> + .flatMap { (product, paymentCustomerId) -> // Add payment source if (sourceId != null) { - paymentProcessor.getSavedSources(profileInfo.id) - .fold( - { - Either.left(org.ostelco.prime.paymentprocessor.core.BadGatewayError("Failed to fetch sources for user", it.description)) - }, - { - var linkedSource = sourceId - if (!it.any{ sourceDetailsInfo -> sourceDetailsInfo.id == sourceId }) { - paymentProcessor.addSource(profileInfo.id, sourceId).map { sourceInfo -> linkedSource = sourceInfo.id } - } - Either.right(Triple(product,profileInfo, linkedSource)) - } - ) + // First fetch all existing saved sources + paymentProcessor.getSavedSources(paymentCustomerId) + .fold( + { + Either.left(org.ostelco.prime.paymentprocessor.core.BadGatewayError("Failed to fetch sources for user", it.description)) + }, + { + // If the sourceId is not found in existing list of saved sources, + // then save the source + if (!it.any { sourceDetailsInfo -> sourceDetailsInfo.id == sourceId }) { + paymentProcessor.addSource(paymentCustomerId, sourceId) + // TODO payment: Should we remove the sourceId for saveCard == false even when captureCharge has failed? + // For success case, saved source is removed after "capture charge" is saveCard == false. + // Making sure same happens even for failure case by linking reversal action to transaction + .finallyDo(transaction) { _ -> removePaymentSource(saveCard, paymentCustomerId, sourceId) } + .map { sourceInfo -> Triple(product, paymentCustomerId, sourceInfo.id) } + } else { + Either.right(Triple(product, paymentCustomerId, sourceId)) + } + } + ) } else { - Either.right(Triple(product, profileInfo, null)) + Either.right(Triple(product, paymentCustomerId, null)) } } - .flatMap { (product, profileInfo, savedSourceId) -> + .flatMap { (product, paymentCustomerId, sourceId) -> // Authorize stripe charge for this purchase val price = product.price //TODO: If later steps fail, then refund the authorized charge - paymentProcessor.authorizeCharge(profileInfo.id, savedSourceId, price.amount, price.currency) + paymentProcessor.authorizeCharge(paymentCustomerId, sourceId, price.amount, price.currency) .mapLeft { apiError -> - logger.error("failed to authorize purchase for customerId ${profileInfo.id}, sourceId $savedSourceId, sku $sku") + logger.error("failed to authorize purchase for paymentCustomerId $paymentCustomerId, sourceId $sourceId, sku $sku") apiError } - .map { chargeId -> Tuple4(profileInfo, savedSourceId, chargeId, product) } + .linkReversalActionToTransaction(transaction) { chargeId -> + paymentProcessor.refundCharge(chargeId) + logger.error("failed to refund charge for paymentCustomerId $paymentCustomerId, chargeId $chargeId. Fix this in Stripe dashboard") + } + .map { chargeId -> Tuple3(product, paymentCustomerId, chargeId) } } - .flatMap { (profileInfo, savedSourceId, chargeId, product) -> + .flatMap { (product, paymentCustomerId, chargeId) -> val purchaseRecord = PurchaseRecord( id = chargeId, product = product, @@ -409,51 +421,57 @@ object Neo4jStoreSingleton : GraphStore { // Create purchase record createPurchaseRecordRelation(subscriberId, purchaseRecord, transaction) .mapLeft { storeError -> - paymentProcessor.refundCharge(chargeId) - logger.error("failed to save purchase record, for customerId ${profileInfo.id}, chargeId $chargeId, payment will be unclaimed in Stripe") + logger.error("failed to save purchase record, for paymentCustomerId $paymentCustomerId, chargeId $chargeId, payment will be unclaimed in Stripe") BadGatewayError(storeError.message) } - // Notify OCS .flatMap { //TODO: While aborting transactions, send a record with "reverted" status analyticsReporter.reportPurchaseInfo( purchaseRecord = purchaseRecord, subscriberId = subscriberId, status = "success") - //TODO vihang: Handle errors (when it becomes available) - ocs.topup(subscriberId, sku) + // TODO vihang: handle currency conversion analyticsReporter.reportMetric(REVENUE, product.price.amount.toLong()) analyticsReporter.reportMetric(USERS_PAID_AT_LEAST_ONCE, getPaidSubscriberCount(transaction)) - Either.right(Tuple4(profileInfo, savedSourceId, chargeId, product)) + Either.right(Tuple3(product, paymentCustomerId, chargeId)) } } - .mapLeft { error -> - transaction.failure() - error + .flatMap { (product, paymentCustomerId, chargeId) -> + // Notify OCS + ocs.topup(subscriberId, sku) + .bimap({ BadGatewayError(description = "Failed to perform topup", externalErrorMessage = it) }, + { Tuple3(product, paymentCustomerId, chargeId) }) } + .map { (product, paymentCustomerId, chargeId) -> + + // Even if the "capture charge operation" failed, we do not want to rollback. + // In that case, we just want to log it at error level. + // These transactions can then me manually changed before they are auto rollback'ed in 'X' days. + paymentProcessor.captureCharge(chargeId, paymentCustomerId) + .mapLeft { + // TODO payment: retry capture charge + logger.error("Capture failed for paymentCustomerId $paymentCustomerId, chargeId $chargeId, Fix this in Stripe Dashboard") + } + + // Ignore failure to capture charge and always send Either.right() + ProductInfo(product.sku) + } + .ifFailedThenRollback(transaction) + } + // << END - result.map { (profileInfo, _, chargeId, _) -> - // Capture the charge, our database have been updated. - paymentProcessor.captureCharge(chargeId, profileInfo.id) - .mapLeft { - // TODO payment: retry capture charge - logger.error("Capture failed for customerId ${profileInfo.id}, chargeId $chargeId, Fix this in Stripe Dashboard") + private fun removePaymentSource(saveCard: Boolean, paymentCustomerId: String, sourceId: String) { + // In case we fail to remove saved source, we log it at error level. + // These saved sources can then me manually removed. + if (!saveCard) { + paymentProcessor.removeSource(paymentCustomerId, sourceId) + .mapLeft { paymentError -> + logger.error("Failed to remove card, for customerId $paymentCustomerId, sourceId $sourceId") + paymentError } } - result.map { (profileInfo, savedSourceId, _, _) -> - // Remove the payment source - if (!saveCard && savedSourceId != null) { - paymentProcessor.removeSource(profileInfo.id, savedSourceId) - .mapLeft { paymentError -> - logger.error("Failed to remove card, for customerId ${profileInfo.id}, sourceId $savedSourceId") - paymentError - } - } - } - result.map { (_, _, _, product) -> ProductInfo(product.sku) } } - // << END override fun getPurchaseRecords(subscriberId: String): Either> { return readTransaction { @@ -726,11 +744,4 @@ object Neo4jStoreSingleton : GraphStore { // override fun getSegment(id: String): Segment? = segmentStore.get(id)?.let { Segment().apply { this.id = it.id } } // override fun getProductClass(id: String): ProductClass? = productClassStore.get(id) -} - -fun Either.ifFailedThenRollback(transaction: Transaction): Either { - if (this.isLeft()) { - transaction.failure() - } - return this -} +} \ No newline at end of file diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/PrimeTransaction.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/PrimeTransaction.kt new file mode 100644 index 000000000..dd6dfecc5 --- /dev/null +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/PrimeTransaction.kt @@ -0,0 +1,73 @@ +package org.ostelco.prime.storage.graph + +import arrow.core.Either +import org.neo4j.driver.v1.Transaction +import org.ostelco.prime.storage.graph.ActionType.FINAL +import org.ostelco.prime.storage.graph.ActionType.REVERSAL + +class PrimeTransaction(private val transaction: Transaction) : Transaction by transaction { + + private val reversalActions = mutableListOf<() -> Unit>() + private val finalActions = mutableListOf<() -> Unit>() + + private fun toActionList(actionType: ActionType) = when (actionType) { + REVERSAL -> reversalActions + FINAL -> finalActions + } + + private fun doActions(actionType: ActionType) { + val actions = toActionList(actionType) + while (actions.isNotEmpty()) { + actions[0]() + actions.removeAt(0) + } + } + + fun addAction(actionType: ActionType, action: () -> Unit) { + toActionList(actionType).add(action) + } + + override fun failure() { + transaction.failure() + doActions(REVERSAL) + } + + override fun close() { + transaction.close() + finalActions.reverse() + doActions(FINAL) + } +} + +enum class ActionType { + REVERSAL, + FINAL, +} + +typealias Action

= (P) -> Unit + +private fun Either.addAction( + primeTransaction: PrimeTransaction, + action: Action, + actionType: ActionType): Either { + + this.map { param -> + primeTransaction.addAction(actionType) { + action(param) + } + } + return this +} + +fun Either.linkReversalActionToTransaction( + primeTransaction: PrimeTransaction, + reversalAction: Action): Either = addAction(primeTransaction, reversalAction, REVERSAL) + +fun Either.finallyDo( + primeTransaction: PrimeTransaction, + finalAction: Action): Either = addAction(primeTransaction, finalAction, FINAL) + +fun Either.ifFailedThenRollback(primeTransaction: PrimeTransaction): Either = mapLeft { error -> + primeTransaction.failure() + error +} \ No newline at end of file diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Schema.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Schema.kt index 4500633c0..396b28350 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Schema.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Schema.kt @@ -301,7 +301,7 @@ fun readTransaction(action: ReadTransaction.() -> R): R = Neo4jClient.driver.session(READ) .use { session -> session.readTransaction { - action(ReadTransaction(it)) + action(ReadTransaction(PrimeTransaction(it))) } } @@ -309,12 +309,12 @@ fun writeTransaction(action: WriteTransaction.() -> R): R = Neo4jClient.driver.session(WRITE) .use { session -> session.writeTransaction { - action(WriteTransaction(it)) + action(WriteTransaction(PrimeTransaction(it))) } } -data class ReadTransaction(val transaction: Transaction) -data class WriteTransaction(val transaction: Transaction) +data class ReadTransaction(val transaction: PrimeTransaction) +data class WriteTransaction(val transaction: PrimeTransaction) // // Object mapping functions From 8e4aeb92b4d2658d2547071f39a96dcbda6b06ef Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Thu, 27 Sep 2018 09:47:03 +0200 Subject: [PATCH 44/85] Remove unused function. --- .../publishers/ActiveUsersPublisher.kt | 21 +------------------ 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt index 502f87fd2..7fc594b02 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt @@ -31,26 +31,7 @@ object ActiveUsersPublisher : private val pseudonymizerService by lazy { getResource() } - private var gson: Gson = createGson() - - private fun createGson(): Gson { - val builder = GsonBuilder() - // Type for this conversion is explicitly set to java.util.Map - // This is needed because of kotlin's own Map interface - val mapType = object : TypeToken>() {}.type - val serializer = JsonSerializer> { src, _, _ -> - val array = JsonArray() - src.forEach { k, v -> - val property = JsonObject() - property.addProperty("key", k) - property.addProperty("value", v) - array.add(property) - } - array - } - builder.registerTypeAdapter(mapType, serializer) - return builder.create() - } + private var gson: Gson = GsonBuilder().create() private fun convertToJson(activeUsersInfo: ActiveUsersInfo): ByteString = ByteString.copyFromUtf8(gson.toJson(activeUsersInfo)) From 448776bd85f7e5ae480d1986eb0b0b5cf7322a4e Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Thu, 27 Sep 2018 09:47:19 +0200 Subject: [PATCH 45/85] Do not URL encode msisdn We need them only for subscribers (email) --- .../ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt index 7fc594b02..e6bc6e884 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt @@ -42,8 +42,7 @@ object ActiveUsersPublisher : val activeUsersInfoBuilder = ActiveUsersInfo.newBuilder().setTimestamp(Timestamps.fromMillis(timestamp)) for (user in userList) { val userBuilder = org.ostelco.analytics.api.User.newBuilder() - val encodedSubscriberId = URLEncoder.encode(user.msisdn, "UTF-8") - val pseudonym = pseudonymizerService.getSubscriberIdPseudonym(encodedSubscriberId, timestamp).pseudonym + val pseudonym = pseudonymizerService.getSubscriberIdPseudonym(user.msisdn, timestamp).pseudonym activeUsersInfoBuilder.addUsers(userBuilder.setApn(user.apn).setMncMcc(user.mncMcc).setMsisdn(pseudonym).build()) } From 14a85e7b4304793284b58f58befba4cd2fd2ab52 Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Thu, 27 Sep 2018 09:49:32 +0200 Subject: [PATCH 46/85] Remove unused import statements --- .../prime/analytics/publishers/ActiveUsersPublisher.kt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt index e6bc6e884..3c8d60db4 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt @@ -5,10 +5,6 @@ import com.google.api.core.ApiFutures import com.google.api.gax.rpc.ApiException import com.google.gson.Gson import com.google.gson.GsonBuilder -import com.google.gson.JsonArray -import com.google.gson.JsonObject -import com.google.gson.JsonSerializer -import com.google.gson.reflect.TypeToken import com.google.protobuf.ByteString import com.google.protobuf.util.Timestamps import com.google.pubsub.v1.PubsubMessage @@ -18,7 +14,6 @@ import org.ostelco.prime.getLogger import org.ostelco.prime.metrics.api.User import org.ostelco.prime.module.getResource import org.ostelco.prime.pseudonymizer.PseudonymizerService -import java.net.URLEncoder import java.time.Instant /** From cd8a8415393d05520633baf5326b02aa23d12bd1 Mon Sep 17 00:00:00 2001 From: Martin Cederlof Date: Thu, 27 Sep 2018 10:30:45 +0200 Subject: [PATCH 47/85] Updated queries The APN "loltel-test" has home MNC MCC 24201 --- bq-metrics-extractor/config/config.yaml | 33 ++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/bq-metrics-extractor/config/config.yaml b/bq-metrics-extractor/config/config.yaml index fc088bd3f..80f389f08 100644 --- a/bq-metrics-extractor/config/config.yaml +++ b/bq-metrics-extractor/config/config.yaml @@ -93,25 +93,50 @@ bqmetrics: AND timestamp < TIMESTAMP_TRUNC(CURRENT_TIMESTAMP(), DAY) ), 0) as count - type: gauge - name: total_data_used_today_mnc_242_mcc_01 - help: Total data used today MNC 242 MCC 01 + name: total_data_used_today_loltel_test + help: Total data used today loltel-test resultColumn: count sql: > SELECT COALESCE ( (SELECT sum(bucketBytes) AS count FROM `pantel-2decb.data_consumption.raw_consumption` WHERE timestamp >= TIMESTAMP_TRUNC(CURRENT_TIMESTAMP(), DAY) + AND apn = "loltel-test" AND mccMnc = "24201"), 0) as count - type: gauge - name: total_data_used_yesterday_mnc_242_mcc_01 - help: Total data used yesterday MNC 242 MCC 01 + name: total_data_used_yesterday_lotlel_test + help: Total data used yesterday loltel-test resultColumn: count sql: > SELECT COALESCE ( ( SELECT sum(bucketBytes) AS count FROM `pantel-2decb.data_consumption.raw_consumption` WHERE timestamp >= TIMESTAMP_SUB(TIMESTAMP_TRUNC(CURRENT_TIMESTAMP(), DAY), INTERVAL 1 DAY) AND timestamp < TIMESTAMP_TRUNC(CURRENT_TIMESTAMP(), DAY) + AND apn = "loltel-test" AND mccMnc = "24201"), 0) as count + + - type: gauge + name: total_roaming_data_used_today_loltel_test + help: Total roaming data used today loltel-test + resultColumn: count + sql: > + SELECT COALESCE ( + (SELECT sum(bucketBytes) AS count FROM `pantel-2decb.data_consumption.raw_consumption` + WHERE timestamp >= TIMESTAMP_TRUNC(CURRENT_TIMESTAMP(), DAY) + AND apn = "loltel-test" + AND mccMnc != "24201"), 0) as count + - type: gauge + name: total_roaming_data_used_yesterday_lotlel_test + help: Total roaming data used yesterday loltel-test + resultColumn: count + sql: > + SELECT COALESCE ( + ( SELECT sum(bucketBytes) AS count FROM `pantel-2decb.data_consumption.raw_consumption` + WHERE timestamp >= TIMESTAMP_SUB(TIMESTAMP_TRUNC(CURRENT_TIMESTAMP(), DAY), INTERVAL 1 DAY) + AND timestamp < TIMESTAMP_TRUNC(CURRENT_TIMESTAMP(), DAY) + AND apn = "loltel-test" + AND mccMnc != "24201"), 0) as count + - type: gauge name: revenue_today help: Revenue generated today From 701a892eca90b5802ec5c7c3599a057cc7f1113b Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Wed, 26 Sep 2018 16:04:31 +0200 Subject: [PATCH 48/85] Adds support for removing payment sources --- .../org/ostelco/at/common/StripePayment.kt | 9 -- .../org/ostelco/at/jersey/HttpClientUtil.kt | 11 +++ .../kotlin/org/ostelco/at/jersey/Tests.kt | 96 ++++++++++++------- .../kotlin/org/ostelco/at/okhttp/Tests.kt | 79 ++++++++++----- .../client/api/resources/PaymentResource.kt | 25 +++-- .../prime/client/api/store/SubscriberDAO.kt | 2 + .../client/api/store/SubscriberDAOImpl.kt | 15 +++ .../StripePaymentProcessorTest.kt | 65 ++++++++++--- .../StripePaymentProcessor.kt | 11 ++- .../ostelco/prime/apierror/ApiErrorCodes.kt | 1 + .../paymentprocessor/PaymentProcessor.kt | 3 +- prime/infra/dev/prime-client-api.yaml | 22 +++++ prime/infra/prod/prime-client-api.yaml | 22 +++++ 13 files changed, 275 insertions(+), 86 deletions(-) diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt index 5e933ac13..a87422548 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt @@ -55,15 +55,6 @@ object StripePayment { return token.card.id } - fun getCardIdForSourceId(sourceId: String) : String { - - // https://stripe.com/docs/api/java#create_source - Stripe.apiKey = System.getenv("STRIPE_API_KEY") - - val source = Source.retrieve(sourceId) - return source.id - } - /** * Obtains 'default source' directly from Stripe. Use in tests to * verify that the correspondng 'setDefaultSource' API works as diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/HttpClientUtil.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/HttpClientUtil.kt index 4426da7e1..c8b04a7fc 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/HttpClientUtil.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/HttpClientUtil.kt @@ -54,6 +54,17 @@ inline fun put(execute: HttpRequest.() -> Unit): T { return response.readEntity(object : GenericType() {}) } +/** + * DSL function for DELETE operation + */ +inline fun delete(execute: HttpRequest.() -> Unit): T { + val request = HttpRequest().apply(execute) + val response = HttpClient.send(request.path, request.queryParams, request.headerParams, request.subscriberId) + .delete() + assertEquals(200, response.status) { response.readEntity(String::class.java) } + return response.readEntity(object : GenericType() {}) +} + fun assertEquals(expected: T, actual: T, lazyMessage: () -> String) { var message = "" if (expected != actual) { diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt index da04f7da3..177aa4bc6 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt @@ -283,7 +283,7 @@ class SourceTest { subscriberId = email } - val ids = createdIds.map { getIdFromStripe(it) } + val ids = createdIds.map { getCardIdForTokenFromStripe(it) } assert(sources.isNotEmpty()) { "Expected at least one payment source for profile $email" } assert(sources.map{ it.id }.containsAll(ids)) @@ -297,37 +297,6 @@ class SourceTest { } } - private fun getIdFromStripe(tokenId : String) : String { - if (tokenId.startsWith("src_")) { - return StripePayment.getCardIdForSourceId(tokenId) - } - return StripePayment.getCardIdForTokenId(tokenId) - } - - private fun createTokenWithStripe(email: String) : String { - val tokenId = StripePayment.createPaymentTokenId() - - post { - path = "/paymentSources" - subscriberId = email - queryParams = mapOf("sourceId" to tokenId) - } - - return tokenId - } - - private fun createSourceWithStripe(email: String) : String { - val sourceId = StripePayment.createPaymentSourceId() - - post { - path = "/paymentSources" - subscriberId = email - queryParams = mapOf("sourceId" to sourceId) - } - - return sourceId - } - @Test fun `jersey test - PUT source set default`() { @@ -374,6 +343,69 @@ class SourceTest { assertEquals(newCardId, StripePayment.getDefaultSourceForCustomer(customerId), "Expected $newCardId to be default source for $customerId") } + + @Test + fun `okhttp test - DELETE source`() { + + StripePayment.deleteAllCustomers() + + val email = "purchase-${randomInt()}@test.com" + createProfile(name = "Test Payment Source", email = email) + + Thread.sleep(200) + + val createdIds = listOf(getCardIdForTokenFromStripe(createTokenWithStripe(email)), + createSourceWithStripe(email)) + + val deletedIds = createdIds.map { it -> removeSourceWithStripe(email, it) } + + assert(createdIds.containsAll(deletedIds.toSet())) { + "Failed to delete one or more sources: ${createdIds.toSet() - deletedIds.toSet()}" + } + } + + // Helpers for source handling with Stripe. + + private fun getCardIdForTokenFromStripe(id: String) : String { + if (id.startsWith("tok_")) { + return StripePayment.getCardIdForTokenId(id) + } + return id + } + + private fun createTokenWithStripe(email: String) : String { + val tokenId = StripePayment.createPaymentTokenId() + + post { + path = "/paymentSources" + subscriberId = email + queryParams = mapOf("sourceId" to tokenId) + } + + return tokenId + } + + private fun createSourceWithStripe(email: String) : String { + val sourceId = StripePayment.createPaymentSourceId() + + post { + path = "/paymentSources" + subscriberId = email + queryParams = mapOf("sourceId" to sourceId) + } + + return sourceId + } + + private fun removeSourceWithStripe(email: String, sourceId: String) : String { + val removedSource = delete { + path = "/paymentSources" + subscriberId = email + queryParams = mapOf("sourceId" to sourceId) + } + + return removedSource.id + } } class PurchaseTest { diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt index 40d04e0be..dbf79dda5 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt @@ -246,7 +246,7 @@ class SourceTest { val sources = client.listSources() - val ids = createdIds.map { getIdFromStripe(it) } + val ids = createdIds.map { getCardIdForTokenFromStripe(it) } assert(sources.isNotEmpty()) { "Expected at least one payment source for profile $email" } assert(sources.map{ it.id }.containsAll(ids)) @@ -260,29 +260,6 @@ class SourceTest { } } - private fun getIdFromStripe(tokenId : String) : String { - if (tokenId.startsWith("src_")) { - return StripePayment.getCardIdForSourceId(tokenId) - } - return StripePayment.getCardIdForTokenId(tokenId) - } - - private fun createTokenWithStripe(client : DefaultApi) : String { - val tokenId = StripePayment.createPaymentTokenId() - - client.createSource(tokenId) - - return tokenId - } - - private fun createSourceWithStripe(client : DefaultApi) : String { - val sourceId = StripePayment.createPaymentSourceId() - - client.createSource(sourceId) - - return sourceId - } - @Test fun `okhttp test - PUT source set default`() { @@ -319,6 +296,60 @@ class SourceTest { assertEquals(newCardId, StripePayment.getDefaultSourceForCustomer(customerId), "Expected $newCardId to be default source for $customerId") } + + @Test + fun `okhttp test - DELETE source`() { + + StripePayment.deleteAllCustomers() + + val email = "purchase-${randomInt()}@test.com" + createProfile(name = "Test Payment Source", email = email) + + val client = clientForSubject(subject = email) + + Thread.sleep(200) + + val createdIds = listOf(getCardIdForTokenFromStripe(createTokenWithStripe(client)), + createSourceWithStripe(client)) + + val deletedIds = createdIds.map { it -> deleteSourceWithStripe(client, it) } + + assert(createdIds.containsAll(deletedIds.toSet())) { + "Failed to delete one or more sources: ${createdIds.toSet() - deletedIds.toSet()}" + } + } + + // Helpers for source handling with Stripe. + + private fun getCardIdForTokenFromStripe(id: String) : String { + if (id.startsWith("tok_")) { + return StripePayment.getCardIdForTokenId(id) + } + return id + } + + private fun createTokenWithStripe(client: DefaultApi) : String { + val tokenId = StripePayment.createPaymentTokenId() + + client.createSource(tokenId) + + return tokenId + } + + private fun createSourceWithStripe(client: DefaultApi) : String { + val sourceId = StripePayment.createPaymentSourceId() + + client.createSource(sourceId) + + return sourceId + } + + private fun deleteSourceWithStripe(client : DefaultApi, sourceId : String) : String { + + val removedSource = client.removeSource(sourceId) + + return removedSource.id + } } class PurchaseTest { diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/resources/PaymentResource.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/resources/PaymentResource.kt index 737bb3440..ff050e8d7 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/resources/PaymentResource.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/resources/PaymentResource.kt @@ -5,12 +5,7 @@ import org.ostelco.prime.client.api.auth.AccessTokenPrincipal import org.ostelco.prime.client.api.store.SubscriberDAO import org.ostelco.prime.getLogger import javax.validation.constraints.NotNull -import javax.ws.rs.GET -import javax.ws.rs.POST -import javax.ws.rs.PUT -import javax.ws.rs.Path -import javax.ws.rs.Produces -import javax.ws.rs.QueryParam +import javax.ws.rs.* import javax.ws.rs.core.Response /** @@ -72,4 +67,22 @@ class PaymentResource(private val dao: SubscriberDAO) { { sourceInfo -> Response.status(Response.Status.OK).entity(sourceInfo)} ).build() } + + @DELETE + @Produces("application/json") + fun removeSource(@Auth token: AccessTokenPrincipal?, + @NotNull + @QueryParam("sourceId") + sourceId: String): Response { + if (token == null) { + return Response.status(Response.Status.UNAUTHORIZED) + .build() + } + + return dao.removeSource(token.name, sourceId) + .fold( + { apiError -> Response.status(apiError.status).entity(asJson(apiError)) }, + { sourceInfo -> Response.status(Response.Status.OK).entity(sourceInfo)} + ).build() + } } diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAO.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAO.kt index db00add39..39f377634 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAO.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAO.kt @@ -65,6 +65,8 @@ interface SubscriberDAO { fun listSources(subscriberId: String): Either> + fun removeSource(subscriberId: String, sourceId: String): Either + companion object { /** diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt index 3280f5e23..9212a8b7d 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt @@ -351,4 +351,19 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu .mapLeft { mapPaymentErrorToApiError("Failed to list sources", ApiErrorCode.FAILED_TO_FETCH_PAYMENT_SOURCES_LIST, it) } } } + + override fun removeSource(subscriberId: String, sourceId: String): Either { + return paymentProcessor.getPaymentProfile(subscriberId) + .fold( + { + paymentProcessor.createPaymentProfile(subscriberId) + .mapLeft { error -> mapPaymentErrorToApiError(error.description, ApiErrorCode.FAILED_TO_SET_DEFAULT_PAYMENT_SOURCE, error) } + }, + { profileInfo -> Either.right(profileInfo) } + ) + .flatMap { profileInfo -> + paymentProcessor.removeSource(profileInfo.id, sourceId) + .mapLeft { mapPaymentErrorToApiError("Failed to remove payment source", ApiErrorCode.FAILED_TO_REMOVE_PAYMENT_SOURCE, it) } + } + } } diff --git a/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt b/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt index 7cc9029f1..4f1987749 100644 --- a/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt +++ b/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt @@ -1,7 +1,10 @@ package org.ostelco.prime.paymentprocessor import arrow.core.getOrElse +import arrow.core.right +import arrow.core.some import com.stripe.Stripe +import com.stripe.model.Source import com.stripe.model.Token import org.junit.After import org.junit.Before @@ -18,19 +21,40 @@ class StripePaymentProcessorTest { private var stripeCustomerId = "" - fun createPaymentSourceId(): String { + private fun createPaymentTokenId() : String { val cardMap = mapOf( "number" to "4242424242424242", "exp_month" to 8, "exp_year" to 2019, "cvc" to "314") - val tokenMap = mapOf("card" to cardMap) + val token = Token.create(tokenMap) return token.id } + private fun createPaymentSourceId() : String { + + val sourceMap = mapOf( + "type" to "card", + "card" to mapOf( + "number" to "4242424242424242", + "exp_month" to 8, + "exp_year" to 2019, + "cvc" to "314"), + "owner" to mapOf( + "address" to mapOf( + "city" to "Oslo", + "country" to "Norway" + ), + "email" to "me@somewhere.com") + ) + + val source = Source.create(sourceMap) + return source.id + } + private fun addCustomer() { val resultAdd = paymentProcessor.createPaymentProfile(testCustomer) assertEquals(true, resultAdd.isRight()) @@ -73,7 +97,7 @@ class StripePaymentProcessorTest { fun ensureSourcesSorted() { run { - paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) // Ensure that not all sources falls within the same second. Thread.sleep(1_001) paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) @@ -84,18 +108,37 @@ class StripePaymentProcessorTest { val createdTimestamps = sources.getOrElse { fail("The 'created' field is missing from the list of sources: ${sources}") - }.map { it.details["created"] as Long } + }.map { it.details["created"] as Long } val createdTimestampsSorted = createdTimestamps.sortedByDescending { it } - assertEquals(createdTimestampsSorted, createdTimestamps, + assertEquals(createdTimestamps, createdTimestampsSorted, "The list of sources is not in descending sorted order by 'created' timestamp: ${sources}") } + @Test + fun addAndRemoveMultipleSources() { + + val sources= listOf( + paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()), + paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + ) + + val sourcesRemoved = sources.map { + paymentProcessor.removeSource(stripeCustomerId, it.getOrElse { + fail("Failed to remove source ${it}") + }.id) + } + + sourcesRemoved.forEach { it -> + assertEquals(true, it.isRight(), "Unexpected failure when removing source ${it}") + } + } + @Test fun addSourceToCustomerAndRemove() { - val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) val resultStoredSources = paymentProcessor.getSavedSources(stripeCustomerId) assertEquals(1, resultStoredSources.fold({ 0 }, { it.size })) @@ -111,8 +154,8 @@ class StripePaymentProcessorTest { } @Test - fun addSourceToCustomerTwise() { - val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + fun addSourceToCustomerTwice() { + val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) val resultStoredSources = paymentProcessor.getSavedSources(stripeCustomerId) assertEquals(1, resultStoredSources.fold({ 0 }, { it.size })) @@ -133,7 +176,7 @@ class StripePaymentProcessorTest { @Test fun addDefaultSourceAndRemove() { - val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) assertEquals(true, resultAddSource.isRight()) val resultAddDefault = paymentProcessor.setDefaultSource(stripeCustomerId, resultAddSource.fold({ "" }, { it.id })) @@ -149,7 +192,7 @@ class StripePaymentProcessorTest { @Test fun createAuthorizeChargeAndRefund() { - val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) assertEquals(true, resultAddSource.isRight()) val resultAuthorizeCharge = paymentProcessor.authorizeCharge(stripeCustomerId, resultAddSource.fold({ "" }, { it.id }), 1000, "nok") @@ -176,7 +219,7 @@ class StripePaymentProcessorTest { @Test fun subscribeAndUnsubscribePlan() { - val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) assertEquals(true, resultAddSource.isRight()) val resultCreateProduct = paymentProcessor.createProduct("TestSku") diff --git a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt index e54a7ac44..eb5e329be 100644 --- a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt +++ b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt @@ -233,9 +233,16 @@ class StripePaymentProcessor : PaymentProcessor { Refund.create(refundParams).charge } - override fun removeSource(customerId: String, sourceId: String): Either = + override fun removeSource(customerId: String, sourceId: String): Either = either("Failed to remove source $sourceId from customer $customerId") { - Customer.retrieve(customerId).sources.retrieve(sourceId).delete().id + val accountInfo = Customer.retrieve(customerId).sources.retrieve(sourceId) + when (accountInfo) { + is Card -> accountInfo.delete() + is Source -> accountInfo.detach() + else -> + Either.left(BadGatewayError("Attempt to remove unsupported account-type ${accountInfo}")) + } + SourceInfo(sourceId) } private fun either(errorDescription: String, action: () -> RETURN): Either { diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiErrorCodes.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiErrorCodes.kt index 4b0950a03..849144266 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiErrorCodes.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiErrorCodes.kt @@ -18,6 +18,7 @@ enum class ApiErrorCode { FAILED_TO_STORE_PAYMENT_SOURCE, FAILED_TO_SET_DEFAULT_PAYMENT_SOURCE, FAILED_TO_FETCH_PAYMENT_SOURCES_LIST, + FAILED_TO_REMOVE_PAYMENT_SOURCE, FAILED_TO_UPDATE_PROFILE, FAILED_TO_FETCH_CONSENT, FAILED_TO_IMPORT_OFFER, diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/PaymentProcessor.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/PaymentProcessor.kt index 098abdb17..eda338db4 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/PaymentProcessor.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/PaymentProcessor.kt @@ -124,6 +124,5 @@ interface PaymentProcessor { * @param sourceId id of the payment source * @return id if removed */ - fun removeSource(customerId: String, sourceId: String): Either - + fun removeSource(customerId: String, sourceId: String): Either } \ No newline at end of file diff --git a/prime/infra/dev/prime-client-api.yaml b/prime/infra/dev/prime-client-api.yaml index 4f258dfa6..4038789bc 100644 --- a/prime/infra/dev/prime-client-api.yaml +++ b/prime/infra/dev/prime-client-api.yaml @@ -151,6 +151,28 @@ paths: description: "User not found." security: - auth0_jwt: [] + delete: + description: "Remove a payment source for user" + produces: + - application/json + operationId: "removeSource" + parameters: + - name: sourceId + in: query + description: "The stripe-id of the source to be removed" + required: true + type: string + responses: + 200: + description: "Successfully removed the source" + schema: + $ref: '#/definitions/PaymentSource' + 400: + description: "The source could not be removed" + 404: + description: "No such source for user" + security: + - auth0_jwt: [] "/products": get: description: "Get all products for the user." diff --git a/prime/infra/prod/prime-client-api.yaml b/prime/infra/prod/prime-client-api.yaml index 2312044bf..023650751 100644 --- a/prime/infra/prod/prime-client-api.yaml +++ b/prime/infra/prod/prime-client-api.yaml @@ -151,6 +151,28 @@ paths: description: "User not found." security: - auth0_jwt: [] + delete: + description: "Remove a payment source for user" + produces: + - application/json + operationId: "removeSource" + parameters: + - name: sourceId + in: query + description: "The stripe-id of the source to be removed" + required: true + type: string + responses: + 200: + description: "Successfully removed the source" + schema: + $ref: '#/definitions/PaymentSource' + 400: + description: "The source could not be removed" + 404: + description: "No such source for user" + security: + - auth0_jwt: [] "/products": get: description: "Get all products for the user." From e587be277a10572ff81c65e32fbbf8056fa93916 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Thu, 27 Sep 2018 10:34:32 +0200 Subject: [PATCH 49/85] Removing best effort metrics from prime if those metrics are now exported via bigquery extractor --- .../ostelco/prime/client/api/metrics/Metrics.kt | 2 -- .../prime/client/api/store/SubscriberDAOImpl.kt | 17 ++++++++--------- .../ostelco/prime/storage/graph/Neo4jStore.kt | 5 ----- .../prime/analytics/AnalyticsReporter.kt | 5 ----- .../ostelco/prime/analytics/AnalyticsService.kt | 6 +----- 5 files changed, 9 insertions(+), 26 deletions(-) diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/metrics/Metrics.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/metrics/Metrics.kt index 9bd73a7c8..e61d240a9 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/metrics/Metrics.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/metrics/Metrics.kt @@ -3,7 +3,6 @@ package org.ostelco.prime.client.api.metrics import org.ostelco.prime.analytics.AnalyticsService import org.ostelco.prime.analytics.PrimeMetric.TOTAL_USERS import org.ostelco.prime.analytics.PrimeMetric.USERS_ACQUIRED_THROUGH_REFERRALS -import org.ostelco.prime.analytics.PrimeMetric.USERS_PAID_AT_LEAST_ONCE import org.ostelco.prime.module.getResource import org.ostelco.prime.storage.AdminDataSource @@ -13,7 +12,6 @@ val adminStore: AdminDataSource = getResource() fun reportMetricsAtStartUp() { analyticsService.reportMetric(TOTAL_USERS, adminStore.getSubscriberCount()) analyticsService.reportMetric(USERS_ACQUIRED_THROUGH_REFERRALS, adminStore.getReferredSubscriberCount()) - analyticsService.reportMetric(USERS_PAID_AT_LEAST_ONCE, adminStore.getPaidSubscriberCount()) } fun updateMetricsOnNewSubscriber() { diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt index 3280f5e23..801a84084 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt @@ -3,11 +3,6 @@ package org.ostelco.prime.client.api.store import arrow.core.Either import arrow.core.flatMap import org.ostelco.prime.analytics.AnalyticsService -import org.ostelco.prime.analytics.PrimeMetric.REVENUE -import org.ostelco.prime.client.api.metrics.updateMetricsOnNewSubscriber -import org.ostelco.prime.client.api.model.Consent -import org.ostelco.prime.client.api.model.Person -import org.ostelco.prime.client.api.model.SubscriptionStatus import org.ostelco.prime.apierror.ApiError import org.ostelco.prime.apierror.ApiErrorCode import org.ostelco.prime.apierror.BadGatewayError @@ -16,6 +11,10 @@ import org.ostelco.prime.apierror.InsufficientStorageError import org.ostelco.prime.apierror.NotFoundError import org.ostelco.prime.apierror.mapPaymentErrorToApiError import org.ostelco.prime.apierror.mapStorageErrorToApiError +import org.ostelco.prime.client.api.metrics.updateMetricsOnNewSubscriber +import org.ostelco.prime.client.api.model.Consent +import org.ostelco.prime.client.api.model.Person +import org.ostelco.prime.client.api.model.SubscriptionStatus import org.ostelco.prime.getLogger import org.ostelco.prime.model.ActivePseudonyms import org.ostelco.prime.model.ApplicationToken @@ -231,13 +230,13 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu purchaseRecord = purchaseRecord, subscriberId = subscriberId, status = "success") - //TODO: Handle errors (when it becomes available) - ocsSubscriberService.topup(subscriberId, sku) - // TODO vihang: handle currency conversion - analyticsReporter.reportMetric(REVENUE, product.price.amount.toLong()) Either.right(Unit) } } + .flatMap { + ocsSubscriberService.topup(subscriberId, sku) + .mapLeft { errorReason -> BadGatewayError(description = errorReason, errorCode = ApiErrorCode.FAILED_TO_PURCHASE_PRODUCT) } + } } override fun purchaseProduct( diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index ad21ea340..1dab69abd 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -5,8 +5,6 @@ import arrow.core.Tuple3 import arrow.core.flatMap import org.neo4j.driver.v1.Transaction import org.ostelco.prime.analytics.AnalyticsService -import org.ostelco.prime.analytics.PrimeMetric.REVENUE -import org.ostelco.prime.analytics.PrimeMetric.USERS_PAID_AT_LEAST_ONCE import org.ostelco.prime.getLogger import org.ostelco.prime.model.Bundle import org.ostelco.prime.model.Offer @@ -431,9 +429,6 @@ object Neo4jStoreSingleton : GraphStore { subscriberId = subscriberId, status = "success") - // TODO vihang: handle currency conversion - analyticsReporter.reportMetric(REVENUE, product.price.amount.toLong()) - analyticsReporter.reportMetric(USERS_PAID_AT_LEAST_ONCE, getPaidSubscriberCount(transaction)) Either.right(Tuple3(product, paymentCustomerId, chargeId)) } } diff --git a/ocs/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsReporter.kt b/ocs/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsReporter.kt index e564c291c..0a0b19e8f 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsReporter.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsReporter.kt @@ -1,7 +1,6 @@ package org.ostelco.prime.analytics import com.lmax.disruptor.EventHandler -import org.ostelco.prime.analytics.PrimeMetric.MEGABYTES_CONSUMED import org.ostelco.prime.disruptor.EventMessageType.CREDIT_CONTROL_REQUEST import org.ostelco.prime.disruptor.OcsEvent import org.ostelco.prime.getLogger @@ -34,10 +33,6 @@ object AnalyticsReporter : EventHandler { bundleBytes = event.bundleBytes, apn = event.request?.serviceInformation?.psInformation?.calledStationId, mccMnc = event.request?.serviceInformation?.psInformation?.sgsnMccMnc) - analyticsReporter.reportMetric( - primeMetric = MEGABYTES_CONSUMED, - value = (event.request?.msccList?.firstOrNull()?.used?.totalOctets ?: 0L) / 1_000_000) - } } } diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt index f91b0afd6..ee2ff6a1d 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt @@ -1,6 +1,5 @@ package org.ostelco.prime.analytics -import org.ostelco.prime.analytics.MetricType.COUNTER import org.ostelco.prime.analytics.MetricType.GAUGE import org.ostelco.prime.model.PurchaseRecord @@ -14,11 +13,8 @@ enum class PrimeMetric(val metricType: MetricType) { // sorted alphabetically ACTIVE_SESSIONS(GAUGE), - MEGABYTES_CONSUMED(COUNTER), - REVENUE(COUNTER), TOTAL_USERS(GAUGE), - USERS_ACQUIRED_THROUGH_REFERRALS(GAUGE), - USERS_PAID_AT_LEAST_ONCE(GAUGE); + USERS_ACQUIRED_THROUGH_REFERRALS(GAUGE); val metricName: String get() = name.toLowerCase() From 96fa2939fc42f49d859909aa49417342603d9b49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 27 Sep 2018 11:30:14 +0200 Subject: [PATCH 50/85] Stepping stone towards greater things, not done yet --- sample-agent/sample-agent.sh | 129 +++++++++++++++++++++-------------- 1 file changed, 79 insertions(+), 50 deletions(-) diff --git a/sample-agent/sample-agent.sh b/sample-agent/sample-agent.sh index 9b18abac9..67d4bee15 100755 --- a/sample-agent/sample-agent.sh +++ b/sample-agent/sample-agent.sh @@ -12,6 +12,32 @@ + +### +### VALIDATING AND PARSING COMMAND LINE PARAMETERS +### + + +# +# Get command line parameter, which should be an existing +# directory in which to store the results +# + +TARGET_DIR=$1 +if [[ -z "$TARGET_DIR" ]] ; then + echo "$0 Missing parameter" + echo "usage $0 target-dir" + exit 1 +fi + +if [[ ! -d "$TARGET_DIR" ]] ; then + echo "$0 $TARGET_DIR is not a directory" + echo "usage $0 target-dir" + exit 1 +fi + + + ### ### PRELIMINARIES ### @@ -58,36 +84,10 @@ fi -### -### VALIDATING AND PARSING COMMAND LINE PARAMETERS -### - - -# -# Get command line parameter, which should be an existing -# directory in which to store the results -# - -TARGET_DIR=$1 -if [[ -z "$TARGET_DIR" ]] ; then - echo "$0 Missing parameter" - echo "usage $0 target-dir" - exit 1 -fi - -if [[ ! -d "$TARGET_DIR" ]] ; then - echo "$0 parameter does not designate an existing directory" - echo "usage $0 target-dir" - exit 1 -fi - - ### ### COMMUNICATION WITH EXPORTER SCRIPTS RUNNING IN A KUBERNETES PODS ### - - # # Run named script on the inside of the kubernetes exporter pod, # put the output from running that script into a temporary file, return the @@ -95,15 +95,15 @@ fi # The second argument is the intent of the invocation, and is used # when producing error messages: # -# runScriptOnExporterPod /export_data.sh "export data" +# runScriptOnExporterPod /export_data.sh "export data" # -function runScriptOnExportrerPod { +function runScriptOnExporterPod { local scriptname=$1 local intentDescription=$2 #TEMPFILE="$(mktemp /tmp/abc-script.XXXXXX)" + # XXX Also should be lowercase TEMPFILE="tmpfile.txt" - kubectl exec -it "${EXPORTER_PODNAME}" -- /bin/bash -c "$scriptname" > "$TEMPFILE" @@ -111,29 +111,39 @@ function runScriptOnExportrerPod { retVal=$? if [[ $retVal -ne 0 ]]; then echo "ERROR: Failed to $intentDescription" - cat $TMPFILE - rm $TMPFILE + cat $TEMPFILE + rm $TEMPFILE exit 1 fi - echo $TMPFILE + + # Return result by setting resutlvar to be the temporary filename + echo $TEMPFILE } + # # Create a data export batch, return a string identifying that # batch. Typical usage: -# EXPORT_ID=$(exportDataFromExporterPad) +# EXPORT_ID=$(exportDataFromExporterPod) # function exportDataFromExporterPod { - local tmpfilename=$(runScriptOnExporterPod /export_data.sh "export data") - local exportId=$(grep "Starting export job for" $tmpfilename | awk '{print $5}' | sed 's/\r$//' ) + local tmpfilename="$(runScriptOnExporterPod /export_data.sh "export data")" + if [[ -z "$tmpfilename" ]] ; then + echo "$0 ERROR: Running the runScriptOnExporterPod failed to return the name of a resultfile." + exit 1 + fi + + local exportId="$(grep "Starting export job for" $tmpfilename | awk '{print $5}' | sed 's/\r$//' )" + if [[ -z "$exportId" ]] ; then echo "$0 Could not get export batch from exporter pod" fi - echo exportId + rm $tmpfilename + echo $exportId } function mapPseudosToUserids { - local tmpfile=$(runScriptOnExporterPod /map_subscribers.sh "mapping pseudoids to subscriber ids") + local tmpfile="$(runScriptOnExporterPod /map_subscribers.sh "mapping pseudoids to subscriber ids")" # XXX Map, then transform$(runScriptOnExporterPod /export_data.sh "export data")$(runScriptOnExporterPod /export_data.sh "export data") } @@ -142,25 +152,42 @@ function mapPseudosToUserids { # Generate the Google filesystem names of components associated with # a particular export ID: Typical usage # -# PURCHASES_GS=$(gsExportCsvFilename "purchases") +# PURCHASES_GS="$(gsExportCsvFilename "ab234245cvsr" "purchases")" function gsExportCsvFilename { - local componentName=$1 - if [[ -n "$componentName" ]] ; then + local exportId=$1 + local componentName=$2 + if [[ -z "$exportId" ]] ; then + echo "$0 Internal error: gsExportCsvFilename got a null exportId" + exit 1 + fi + if [[ -z "$componentName" ]] ; then componentName="-$componentName" fi - return "gs://${PROJECT_ID}-dataconsumption-export/${EXPORT_ID}${componentName}.csv" + echo "gs://${PROJECT_ID}-dataconsumption-export/${exportId}${componentName}.csv" } function importedCsvFilename { - local importDirectory=$1 - local componentName=$2 + local exportId=$1 + local importDirectory=$2 + local componentName=$3 + + if [[ -z "$exportId" ]] ; then + echo "$0 Internal error: importedCsvFilename got a null exportId" + exit 1 + fi + + if [[ -z "$importDirectory" ]] ; then + echo "$0 Internal error: importDirectory got a null exportId" + exit 1 + fi + if [[ -n "$componentName" ]] ; then componentName="-$componentName" fi - return "${importDirectory}/${EXPORT_ID}${componentName}.csv" + echo "${importDirectory}/${exportId}${componentName}.csv" } @@ -168,22 +195,24 @@ function importedCsvFilename { ### MAIN SCRIPT ### -# EXPORT_ID=0802c66be1ce4e2dba22f988b3ce24f7 -EXPORT_ID=$(exportDataFromExporterPad) +# EXPORT_ID=0802c66be1ce4e2dba22f988b3ce24f7 +EXPORT_ID="$(exportDataFromExporterPod)" +echo "EXPORT_ID = $EXPORT_ID" # # Get the IDs of the various parts being exported # - for component in "purchases" "sub2msisdn" "" ; do - gsutil cp $(gsExportCsvFilename $component) $(importedCsvFilename $TARGET_DIR $component) + source="$(gsExportCsvFilename $EXPORT_ID $component)" + destination="$(importedCsvFilename $EXPORT_ID $TARGET_DIR $component)" + gsutil cp $source $destination done -echo $EXPORT_ID - +echo "EXITING AT LINE $LINENO" +exit ## ## Generate the yaml output From 741976d0aba28529b2c8655e45837fd260f389fd Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Thu, 27 Sep 2018 11:49:20 +0200 Subject: [PATCH 51/85] Use protobuf util to convert json --- .../publishers/ActiveUsersPublisher.kt | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt index 3c8d60db4..97bb094a7 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt @@ -5,6 +5,9 @@ import com.google.api.core.ApiFutures import com.google.api.gax.rpc.ApiException import com.google.gson.Gson import com.google.gson.GsonBuilder +import com.google.gson.TypeAdapter +import com.google.gson.stream.JsonReader +import com.google.gson.stream.JsonWriter import com.google.protobuf.ByteString import com.google.protobuf.util.Timestamps import com.google.pubsub.v1.PubsubMessage @@ -15,7 +18,7 @@ import org.ostelco.prime.metrics.api.User import org.ostelco.prime.module.getResource import org.ostelco.prime.pseudonymizer.PseudonymizerService import java.time.Instant - +import com.google.protobuf.util.JsonFormat /** * This class publishes the active users information events to the Google Cloud Pub/Sub. */ @@ -29,7 +32,7 @@ object ActiveUsersPublisher : private var gson: Gson = GsonBuilder().create() private fun convertToJson(activeUsersInfo: ActiveUsersInfo): ByteString = - ByteString.copyFromUtf8(gson.toJson(activeUsersInfo)) + ByteString.copyFromUtf8(JsonFormat.printer().print(activeUsersInfo)) fun publish(userList: List) { @@ -67,3 +70,16 @@ object ActiveUsersPublisher : }, singleThreadScheduledExecutor) } } + +//class ActiveUsersInfoAdapter : TypeAdapter() { +// override fun read(reader: JsonReader?): ActiveUsersInfo { +// TODO("not implemented") //To change body of created functions use File | Settings | File Templates. +// return ActiveUsersInfo.newBuilder().build() +// } +// +// override fun write(jsonWriter: JsonWriter?, usersInfo: ActiveUsersInfo?) { +// if( usersInfo != null) { +// jsonWriter?.jsonValue(JsonFormat.printer().print(usersInfo)); +// } +// } +//} \ No newline at end of file From 96fd95693bcf3b8dc935fa3b2195d62735c89c38 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Thu, 27 Sep 2018 11:52:43 +0200 Subject: [PATCH 52/85] Deleting only those stripe customers which are created as part of the test in AT. --- .../org/ostelco/at/common/StripePayment.kt | 12 +- .../kotlin/org/ostelco/at/jersey/Tests.kt | 357 +++++++++--------- .../kotlin/org/ostelco/at/okhttp/Tests.kt | 220 ++++++----- 3 files changed, 309 insertions(+), 280 deletions(-) diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt index 5e933ac13..8d7562a9a 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt @@ -91,15 +91,11 @@ object StripePayment { return customers.filter { it.email.equals(email) }.first().id } - fun deleteAllCustomers() { + fun deleteCustomer(email: String) { // https://stripe.com/docs/api/java#create_card_token Stripe.apiKey = System.getenv("STRIPE_API_KEY") - - do { - val customers = Customer.list(emptyMap()).data - customers.forEach { customer -> - customer.delete() - } - } while (customers.isNotEmpty()) + val customers = Customer.list(emptyMap()).data + customers.filter { it.email == email } + .forEach { it.delete() } } } \ No newline at end of file diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt index da04f7da3..0ece47245 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt @@ -20,10 +20,15 @@ import org.ostelco.prime.client.model.Profile import org.ostelco.prime.client.model.PurchaseRecordList import org.ostelco.prime.client.model.Subscription import org.ostelco.prime.client.model.SubscriptionStatus -import java.lang.AssertionError import java.time.Instant import java.util.* -import kotlin.test.* +import kotlin.test.assertEquals +import kotlin.test.assertFails +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue class ProfileTest { @@ -237,74 +242,79 @@ class SourceTest { @Test fun `jersey test - POST source create`() { - StripePayment.deleteAllCustomers() - val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Payment Source", email = email) + try { - val tokenId = StripePayment.createPaymentTokenId() + createProfile(name = "Test Payment Source", email = email) - // Ties source with user profile both local and with Stripe - post { - path = "/paymentSources" - subscriberId = email - queryParams = mapOf("sourceId" to tokenId) - } + val tokenId = StripePayment.createPaymentTokenId() + + // Ties source with user profile both local and with Stripe + post { + path = "/paymentSources" + subscriberId = email + queryParams = mapOf("sourceId" to tokenId) + } - Thread.sleep(200) + Thread.sleep(200) - val sources: PaymentSourceList = get { - path = "/paymentSources" - subscriberId = email - } - assert(sources.isNotEmpty()) { "Expected at least one payment source for profile $email" } + val sources: PaymentSourceList = get { + path = "/paymentSources" + subscriberId = email + } + assert(sources.isNotEmpty()) { "Expected at least one payment source for profile $email" } - val cardId = StripePayment.getCardIdForTokenId(tokenId) - assertNotNull(sources.first { it.id == cardId }, "Expected card $cardId in list of payment sources for profile $email") + val cardId = StripePayment.getCardIdForTokenId(tokenId) + assertNotNull(sources.first { it.id == cardId }, "Expected card $cardId in list of payment sources for profile $email") + } finally { + StripePayment.deleteCustomer(email = email) + } } @Test fun `jersey test - GET list sources`() { - StripePayment.deleteAllCustomers() - val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Payment Source", email = email) + try { + createProfile(name = "Test Payment Source", email = email) - Thread.sleep(200) + Thread.sleep(200) - val createdIds = listOf(createTokenWithStripe(email), - createSourceWithStripe(email), - createTokenWithStripe(email), - createSourceWithStripe(email)) + val createdIds = listOf(createTokenWithStripe(email), + createSourceWithStripe(email), + createTokenWithStripe(email), + createSourceWithStripe(email)) - val sources : PaymentSourceList = get { - path = "/paymentSources" - subscriberId = email - } + val sources: PaymentSourceList = get { + path = "/paymentSources" + subscriberId = email + } - val ids = createdIds.map { getIdFromStripe(it) } + val ids = createdIds.map { getIdFromStripe(it) } - assert(sources.isNotEmpty()) { "Expected at least one payment source for profile $email" } - assert(sources.map{ it.id }.containsAll(ids)) - { "Expected to find all of $ids in list of sources for profile $email" } + assert(sources.isNotEmpty()) { "Expected at least one payment source for profile $email" } + assert(sources.map { it.id }.containsAll(ids)) + { "Expected to find all of $ids in list of sources for profile $email" } - sources.forEach { - assert(it.id.isNotEmpty()) { "Expected 'id' to be set in source account details for profile $email" } - assert(arrayOf("card", "source").contains(it.type)) { - "Unexpected source account type ${it.type} for profile $email" + sources.forEach { + assert(it.id.isNotEmpty()) { "Expected 'id' to be set in source account details for profile $email" } + assert(arrayOf("card", "source").contains(it.type)) { + "Unexpected source account type ${it.type} for profile $email" + } } + } finally { + StripePayment.deleteCustomer(email = email) } } - private fun getIdFromStripe(tokenId : String) : String { + private fun getIdFromStripe(tokenId: String): String { if (tokenId.startsWith("src_")) { return StripePayment.getCardIdForSourceId(tokenId) } return StripePayment.getCardIdForTokenId(tokenId) } - private fun createTokenWithStripe(email: String) : String { + private fun createTokenWithStripe(email: String): String { val tokenId = StripePayment.createPaymentTokenId() post { @@ -316,7 +326,7 @@ class SourceTest { return tokenId } - private fun createSourceWithStripe(email: String) : String { + private fun createSourceWithStripe(email: String): String { val sourceId = StripePayment.createPaymentSourceId() post { @@ -331,48 +341,50 @@ class SourceTest { @Test fun `jersey test - PUT source set default`() { - StripePayment.deleteAllCustomers() - val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Payment Source", email = email) + try { + createProfile(name = "Test Payment Source", email = email) - val tokenId = StripePayment.createPaymentTokenId() - val cardId = StripePayment.getCardIdForTokenId(tokenId) + val tokenId = StripePayment.createPaymentTokenId() + val cardId = StripePayment.getCardIdForTokenId(tokenId) - // Ties source with user profile both local and with Stripe - post { - path = "/paymentSources" - subscriberId = email - queryParams = mapOf("sourceId" to tokenId) - } + // Ties source with user profile both local and with Stripe + post { + path = "/paymentSources" + subscriberId = email + queryParams = mapOf("sourceId" to tokenId) + } - Thread.sleep(200) + Thread.sleep(200) - val newTokenId = StripePayment.createPaymentTokenId() - val newCardId = StripePayment.getCardIdForTokenId(newTokenId) + val newTokenId = StripePayment.createPaymentTokenId() + val newCardId = StripePayment.getCardIdForTokenId(newTokenId) - post { - path = "/paymentSources" - subscriberId = email - queryParams = mapOf("sourceId" to newTokenId) - } + post { + path = "/paymentSources" + subscriberId = email + queryParams = mapOf("sourceId" to newTokenId) + } - // TODO: Update to fetch the Stripe customerId from 'admin' API when ready. - val customerId = StripePayment.getCustomerIdForEmail(email) + // TODO: Update to fetch the Stripe customerId from 'admin' API when ready. + val customerId = StripePayment.getCustomerIdForEmail(email) - // Verify that original 'sourceId/card' is default. - assertEquals(cardId, StripePayment.getDefaultSourceForCustomer(customerId), - "Expected $cardId to be default source for $customerId") + // Verify that original 'sourceId/card' is default. + assertEquals(cardId, StripePayment.getDefaultSourceForCustomer(customerId), + "Expected $cardId to be default source for $customerId") - // Set new default card. - put { - path = "/paymentSources" - subscriberId = email - queryParams = mapOf("sourceId" to newCardId) - } + // Set new default card. + put { + path = "/paymentSources" + subscriberId = email + queryParams = mapOf("sourceId" to newCardId) + } - assertEquals(newCardId, StripePayment.getDefaultSourceForCustomer(customerId), - "Expected $newCardId to be default source for $customerId") + assertEquals(newCardId, StripePayment.getDefaultSourceForCustomer(customerId), + "Expected $newCardId to be default source for $customerId") + } finally { + StripePayment.deleteCustomer(email = email) + } } } @@ -381,151 +393,156 @@ class PurchaseTest { @Test fun `jersey test - POST products purchase`() { - StripePayment.deleteAllCustomers() - val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Purchase User", email = email) + try { + createProfile(name = "Test Purchase User", email = email) - val balanceBefore = get> { - path = "/bundles" - subscriberId = email - }.first().balance + val balanceBefore = get> { + path = "/bundles" + subscriberId = email + }.first().balance - val productSku = "1GB_249NOK" - val sourceId = StripePayment.createPaymentTokenId() + val productSku = "1GB_249NOK" + val sourceId = StripePayment.createPaymentTokenId() - post { - path = "/products/$productSku/purchase" - subscriberId = email - queryParams = mapOf("sourceId" to sourceId) - } + post { + path = "/products/$productSku/purchase" + subscriberId = email + queryParams = mapOf("sourceId" to sourceId) + } - Thread.sleep(100) // wait for 100 ms for balance to be updated in db + Thread.sleep(100) // wait for 100 ms for balance to be updated in db - val balanceAfter = get> { - path = "/bundles" - subscriberId = email - }.first().balance + val balanceAfter = get> { + path = "/bundles" + subscriberId = email + }.first().balance - assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") + assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") - val purchaseRecords: PurchaseRecordList = get { - path = "/purchases" - subscriberId = email - } + val purchaseRecords: PurchaseRecordList = get { + path = "/purchases" + subscriberId = email + } - purchaseRecords.sortBy { it.timestamp } + purchaseRecords.sortBy { it.timestamp } - assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } - assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } + assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + } finally { + StripePayment.deleteCustomer(email = email) + } } @Test fun `jersey test - POST products purchase using default source`() { - StripePayment.deleteAllCustomers() - val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Purchase User with Default Payment Source", email = email) + try { + createProfile(name = "Test Purchase User with Default Payment Source", email = email) - val sourceId = StripePayment.createPaymentTokenId() + val sourceId = StripePayment.createPaymentTokenId() - val paymentSource: PaymentSource = post { - path = "/paymentSources" - subscriberId = email - queryParams = mapOf("sourceId" to sourceId) - } + val paymentSource: PaymentSource = post { + path = "/paymentSources" + subscriberId = email + queryParams = mapOf("sourceId" to sourceId) + } - assertNotNull(paymentSource.id, message = "Failed to create payment source") + assertNotNull(paymentSource.id, message = "Failed to create payment source") - val balanceBefore = get> { - path = "/bundles" - subscriberId = email - }.first().balance + val balanceBefore = get> { + path = "/bundles" + subscriberId = email + }.first().balance - val productSku = "1GB_249NOK" + val productSku = "1GB_249NOK" - post { - path = "/products/$productSku/purchase" - subscriberId = email - } + post { + path = "/products/$productSku/purchase" + subscriberId = email + } - Thread.sleep(100) // wait for 100 ms for balance to be updated in db + Thread.sleep(100) // wait for 100 ms for balance to be updated in db - val balanceAfter = get> { - path = "/bundles" - subscriberId = email - }.first().balance + val balanceAfter = get> { + path = "/bundles" + subscriberId = email + }.first().balance - assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") + assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") - val purchaseRecords: PurchaseRecordList = get { - path = "/purchases" - subscriberId = email - } + val purchaseRecords: PurchaseRecordList = get { + path = "/purchases" + subscriberId = email + } - purchaseRecords.sortBy { it.timestamp } + purchaseRecords.sortBy { it.timestamp } - assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } - assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } + assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + } finally { + StripePayment.deleteCustomer(email = email) + } } @Test fun `jersey test - POST products purchase add source then pay with it`() { - StripePayment.deleteAllCustomers() - val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Purchase User with Default Payment Source", email = email) + try { + createProfile(name = "Test Purchase User with Default Payment Source", email = email) - val sourceId = StripePayment.createPaymentTokenId() + val sourceId = StripePayment.createPaymentTokenId() - val paymentSource: PaymentSource = post { - path = "/paymentSources" - subscriberId = email - queryParams = mapOf("sourceId" to sourceId) - } + val paymentSource: PaymentSource = post { + path = "/paymentSources" + subscriberId = email + queryParams = mapOf("sourceId" to sourceId) + } - assertNotNull(paymentSource.id, message = "Failed to create payment source") + assertNotNull(paymentSource.id, message = "Failed to create payment source") - val subscriptionStatusBefore: SubscriptionStatus = get { - path = "/subscription/status" - subscriberId = email - } - val balanceBefore = subscriptionStatusBefore.remaining + val subscriptionStatusBefore: SubscriptionStatus = get { + path = "/subscription/status" + subscriberId = email + } + val balanceBefore = subscriptionStatusBefore.remaining - val productSku = "1GB_249NOK" + val productSku = "1GB_249NOK" - post { - path = "/products/$productSku/purchase" - subscriberId = email - queryParams = mapOf("sourceId" to paymentSource.id) - } + post { + path = "/products/$productSku/purchase" + subscriberId = email + queryParams = mapOf("sourceId" to paymentSource.id) + } - Thread.sleep(100) // wait for 100 ms for balance to be updated in db + Thread.sleep(100) // wait for 100 ms for balance to be updated in db - val subscriptionStatusAfter: SubscriptionStatus = get { - path = "/subscription/status" - subscriberId = email - } - val balanceAfter = subscriptionStatusAfter.remaining + val subscriptionStatusAfter: SubscriptionStatus = get { + path = "/subscription/status" + subscriberId = email + } + val balanceAfter = subscriptionStatusAfter.remaining - assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") + assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") - val purchaseRecords: PurchaseRecordList = get { - path = "/purchases" - subscriberId = email - } + val purchaseRecords: PurchaseRecordList = get { + path = "/purchases" + subscriberId = email + } - purchaseRecords.sortBy { it.timestamp } + purchaseRecords.sortBy { it.timestamp } - assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } - assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } + assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + } finally { + StripePayment.deleteCustomer(email = email) + } } - @Test fun `jersey test - POST products purchase without payment`() { diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt index 40d04e0be..07f1115f1 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt @@ -8,8 +8,8 @@ import org.ostelco.at.common.expectedProducts import org.ostelco.at.common.getLogger import org.ostelco.at.common.randomInt import org.ostelco.at.okhttp.ClientFactory.clientForSubject -import org.ostelco.prime.client.api.DefaultApi import org.ostelco.prime.client.ApiException +import org.ostelco.prime.client.api.DefaultApi import org.ostelco.prime.client.model.ApplicationToken import org.ostelco.prime.client.model.Consent import org.ostelco.prime.client.model.PaymentSource @@ -21,7 +21,11 @@ import org.ostelco.prime.client.model.Profile import org.ostelco.prime.client.model.SubscriptionStatus import java.time.Instant import java.util.* -import kotlin.test.* +import kotlin.test.assertEquals +import kotlin.test.assertFails +import kotlin.test.assertFailsWith +import kotlin.test.assertNotNull +import kotlin.test.assertNull class ProfileTest { @@ -205,58 +209,62 @@ class SourceTest { @Test fun `okhttp test - POST source create`() { - StripePayment.deleteAllCustomers() - val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Payment Source", email = email) + try { + createProfile(name = "Test Payment Source", email = email) - val client = clientForSubject(subject = email) + val client = clientForSubject(subject = email) - val tokenId = StripePayment.createPaymentTokenId() - val cardId = StripePayment.getCardIdForTokenId(tokenId) + val tokenId = StripePayment.createPaymentTokenId() + val cardId = StripePayment.getCardIdForTokenId(tokenId) - // Ties source with user profile both local and with Stripe - client.createSource(tokenId) + // Ties source with user profile both local and with Stripe + client.createSource(tokenId) - Thread.sleep(200) + Thread.sleep(200) - val sources = client.listSources() + val sources = client.listSources() - assert(sources.isNotEmpty()) { "Expected at least one payment source for profile $email" } - assertNotNull(sources.first { it.id == cardId }, - "Expected card $cardId in list of payment sources for profile $email") + assert(sources.isNotEmpty()) { "Expected at least one payment source for profile $email" } + assertNotNull(sources.first { it.id == cardId }, + "Expected card $cardId in list of payment sources for profile $email") + } finally { + StripePayment.deleteCustomer(email = email) + } } @Test fun `okhttp test - GET list sources`() { - StripePayment.deleteAllCustomers() - val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Payment Source", email = email) + try { + createProfile(name = "Test Payment Source", email = email) - val client = clientForSubject(subject = email) + val client = clientForSubject(subject = email) - Thread.sleep(200) + Thread.sleep(200) - val createdIds = listOf(createTokenWithStripe(client), - createSourceWithStripe(client), - createTokenWithStripe(client), - createSourceWithStripe(client)) + val createdIds = listOf(createTokenWithStripe(client), + createSourceWithStripe(client), + createTokenWithStripe(client), + createSourceWithStripe(client)) - val sources = client.listSources() + val sources = client.listSources() - val ids = createdIds.map { getIdFromStripe(it) } + val ids = createdIds.map { getIdFromStripe(it) } - assert(sources.isNotEmpty()) { "Expected at least one payment source for profile $email" } - assert(sources.map{ it.id }.containsAll(ids)) - { "Expected to find all of $ids in list of sources for profile $email" } + assert(sources.isNotEmpty()) { "Expected at least one payment source for profile $email" } + assert(sources.map{ it.id }.containsAll(ids)) + { "Expected to find all of $ids in list of sources for profile $email" } - sources.forEach { - assert(it.id.isNotEmpty()) { "Expected 'id' to be set in source account details for profile $email" } - assert(arrayOf("card", "source").contains(it.type)) { - "Unexpected source account type ${it.type} for profile $email" + sources.forEach { + assert(it.id.isNotEmpty()) { "Expected 'id' to be set in source account details for profile $email" } + assert(arrayOf("card", "source").contains(it.type)) { + "Unexpected source account type ${it.type} for profile $email" + } } + } finally { + StripePayment.deleteCustomer(email = email) } } @@ -286,38 +294,40 @@ class SourceTest { @Test fun `okhttp test - PUT source set default`() { - StripePayment.deleteAllCustomers() - val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Payment Source", email = email) + try { + createProfile(name = "Test Payment Source", email = email) - val client = clientForSubject(subject = email) + val client = clientForSubject(subject = email) - val tokenId = StripePayment.createPaymentTokenId() - val cardId = StripePayment.getCardIdForTokenId(tokenId) + val tokenId = StripePayment.createPaymentTokenId() + val cardId = StripePayment.getCardIdForTokenId(tokenId) - // Ties source with user profile both local and with Stripe - client.createSource(tokenId) + // Ties source with user profile both local and with Stripe + client.createSource(tokenId) - Thread.sleep(200) + Thread.sleep(200) - val newTokenId = StripePayment.createPaymentTokenId() - val newCardId = StripePayment.getCardIdForTokenId(newTokenId) + val newTokenId = StripePayment.createPaymentTokenId() + val newCardId = StripePayment.getCardIdForTokenId(newTokenId) - client.createSource(newTokenId) + client.createSource(newTokenId) - // TODO: Update to fetch the Stripe customerId from 'admin' API when ready. - val customerId = StripePayment.getCustomerIdForEmail(email) + // TODO: Update to fetch the Stripe customerId from 'admin' API when ready. + val customerId = StripePayment.getCustomerIdForEmail(email) - // Verify that original 'sourceId/card' is default. - assertEquals(cardId, StripePayment.getDefaultSourceForCustomer(customerId), - "Expected $cardId to be default source for $customerId") + // Verify that original 'sourceId/card' is default. + assertEquals(cardId, StripePayment.getDefaultSourceForCustomer(customerId), + "Expected $cardId to be default source for $customerId") - // Set new default card. - client.setDefaultSource(newCardId) + // Set new default card. + client.setDefaultSource(newCardId) - assertEquals(newCardId, StripePayment.getDefaultSourceForCustomer(customerId), - "Expected $newCardId to be default source for $customerId") + assertEquals(newCardId, StripePayment.getDefaultSourceForCustomer(customerId), + "Expected $newCardId to be default source for $customerId") + } finally { + StripePayment.deleteCustomer(email = email) + } } } @@ -326,103 +336,109 @@ class PurchaseTest { @Test fun `okhttp test - POST products purchase`() { - StripePayment.deleteAllCustomers() - val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Purchase User", email = email) + try { + createProfile(name = "Test Purchase User", email = email) - val client = clientForSubject(subject = email) + val client = clientForSubject(subject = email) - val balanceBefore = client.bundles.first().balance + val balanceBefore = client.bundles.first().balance - val sourceId = StripePayment.createPaymentTokenId() + val sourceId = StripePayment.createPaymentTokenId() - client.purchaseProduct("1GB_249NOK", sourceId, false) + client.purchaseProduct("1GB_249NOK", sourceId, false) - Thread.sleep(200) // wait for 200 ms for balance to be updated in db + Thread.sleep(200) // wait for 200 ms for balance to be updated in db - val balanceAfter = client.bundles.first().balance + val balanceAfter = client.bundles.first().balance - assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") + assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") - val purchaseRecords = client.purchaseHistory + val purchaseRecords = client.purchaseHistory - purchaseRecords.sortBy { it.timestamp } + purchaseRecords.sortBy { it.timestamp } - assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } - assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } + assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + } finally { + StripePayment.deleteCustomer(email = email) + } } @Test fun `okhttp test - POST products purchase using default source`() { - StripePayment.deleteAllCustomers() - val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Purchase User with Default Payment Source", email = email) + try { + createProfile(name = "Test Purchase User with Default Payment Source", email = email) - val sourceId = StripePayment.createPaymentTokenId() + val sourceId = StripePayment.createPaymentTokenId() - val client = clientForSubject(subject = email) + val client = clientForSubject(subject = email) - val paymentSource: PaymentSource = client.createSource(sourceId) + val paymentSource: PaymentSource = client.createSource(sourceId) - assertNotNull(paymentSource.id, message = "Failed to create payment source") + assertNotNull(paymentSource.id, message = "Failed to create payment source") - val balanceBefore = client.bundles.first().balance + val balanceBefore = client.bundles.first().balance - val productSku = "1GB_249NOK" + val productSku = "1GB_249NOK" - client.purchaseProduct(productSku, null, null) + client.purchaseProduct(productSku, null, null) - Thread.sleep(200) // wait for 200 ms for balance to be updated in db + Thread.sleep(200) // wait for 200 ms for balance to be updated in db - val balanceAfter = client.bundles.first().balance + val balanceAfter = client.bundles.first().balance - assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") + assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") - val purchaseRecords = client.purchaseHistory + val purchaseRecords = client.purchaseHistory - purchaseRecords.sortBy { it.timestamp } + purchaseRecords.sortBy { it.timestamp } - assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } - assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } + assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + } finally { + StripePayment.deleteCustomer(email = email) + } } @Test fun `okhttp test - POST products purchase add source then pay with it`() { - StripePayment.deleteAllCustomers() - val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Purchase User with Default Payment Source", email = email) + try { + createProfile(name = "Test Purchase User with Default Payment Source", email = email) - val sourceId = StripePayment.createPaymentTokenId() + val sourceId = StripePayment.createPaymentTokenId() - val client = clientForSubject(subject = email) + val client = clientForSubject(subject = email) - val paymentSource: PaymentSource = client.createSource(sourceId) + val paymentSource: PaymentSource = client.createSource(sourceId) - assertNotNull(paymentSource.id, message = "Failed to create payment source") + assertNotNull(paymentSource.id, message = "Failed to create payment source") - val balanceBefore = client.subscriptionStatus.remaining + val balanceBefore = client.subscriptionStatus.remaining - val productSku = "1GB_249NOK" + val productSku = "1GB_249NOK" - client.purchaseProduct(productSku, paymentSource.id, null) + client.purchaseProduct(productSku, paymentSource.id, null) - Thread.sleep(200) // wait for 200 ms for balance to be updated in db + Thread.sleep(200) // wait for 200 ms for balance to be updated in db - val balanceAfter = client.subscriptionStatus.remaining + val balanceAfter = client.subscriptionStatus.remaining - assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") + assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") - val purchaseRecords = client.purchaseHistory + val purchaseRecords = client.purchaseHistory - purchaseRecords.sortBy { it.timestamp } + purchaseRecords.sortBy { it.timestamp } - assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } - assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } + assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + } finally { + StripePayment.deleteCustomer(email = email) + } } @Test From 4e6446af95036815749264c25b34d3d4584fbe9d Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Thu, 27 Sep 2018 13:14:03 +0200 Subject: [PATCH 53/85] Include default values --- .../publishers/ActiveUsersPublisher.kt | 23 +++---------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt index 97bb094a7..4c796c9db 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt @@ -5,10 +5,8 @@ import com.google.api.core.ApiFutures import com.google.api.gax.rpc.ApiException import com.google.gson.Gson import com.google.gson.GsonBuilder -import com.google.gson.TypeAdapter -import com.google.gson.stream.JsonReader -import com.google.gson.stream.JsonWriter import com.google.protobuf.ByteString +import com.google.protobuf.util.JsonFormat import com.google.protobuf.util.Timestamps import com.google.pubsub.v1.PubsubMessage import org.ostelco.analytics.api.ActiveUsersInfo @@ -18,7 +16,7 @@ import org.ostelco.prime.metrics.api.User import org.ostelco.prime.module.getResource import org.ostelco.prime.pseudonymizer.PseudonymizerService import java.time.Instant -import com.google.protobuf.util.JsonFormat + /** * This class publishes the active users information events to the Google Cloud Pub/Sub. */ @@ -29,10 +27,8 @@ object ActiveUsersPublisher : private val pseudonymizerService by lazy { getResource() } - private var gson: Gson = GsonBuilder().create() - private fun convertToJson(activeUsersInfo: ActiveUsersInfo): ByteString = - ByteString.copyFromUtf8(JsonFormat.printer().print(activeUsersInfo)) + ByteString.copyFromUtf8(JsonFormat.printer().includingDefaultValueFields().print(activeUsersInfo)) fun publish(userList: List) { @@ -70,16 +66,3 @@ object ActiveUsersPublisher : }, singleThreadScheduledExecutor) } } - -//class ActiveUsersInfoAdapter : TypeAdapter() { -// override fun read(reader: JsonReader?): ActiveUsersInfo { -// TODO("not implemented") //To change body of created functions use File | Settings | File Templates. -// return ActiveUsersInfo.newBuilder().build() -// } -// -// override fun write(jsonWriter: JsonWriter?, usersInfo: ActiveUsersInfo?) { -// if( usersInfo != null) { -// jsonWriter?.jsonValue(JsonFormat.printer().print(usersInfo)); -// } -// } -//} \ No newline at end of file From 40405ac81ef263b72188efb0767d3e91a9702517 Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Thu, 27 Sep 2018 13:25:56 +0200 Subject: [PATCH 54/85] Optimize imports --- .../ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt | 2 -- 1 file changed, 2 deletions(-) diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt index 4c796c9db..80961ce86 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt @@ -3,8 +3,6 @@ package org.ostelco.prime.analytics.publishers import com.google.api.core.ApiFutureCallback import com.google.api.core.ApiFutures import com.google.api.gax.rpc.ApiException -import com.google.gson.Gson -import com.google.gson.GsonBuilder import com.google.protobuf.ByteString import com.google.protobuf.util.JsonFormat import com.google.protobuf.util.Timestamps From 394d9daefee57681ea59d12625665de50aaac155 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 27 Sep 2018 13:27:27 +0200 Subject: [PATCH 55/85] Getting to where we were before refactoring, next stop: De-anonymizing data --- sample-agent/sample-agent.sh | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/sample-agent/sample-agent.sh b/sample-agent/sample-agent.sh index 67d4bee15..a3fac56c6 100755 --- a/sample-agent/sample-agent.sh +++ b/sample-agent/sample-agent.sh @@ -158,12 +158,13 @@ function gsExportCsvFilename { local exportId=$1 local componentName=$2 if [[ -z "$exportId" ]] ; then - echo "$0 Internal error: gsExportCsvFilename got a null exportId" + echo "$0 ERROR: gsExportCsvFilename got a null exportId" exit 1 fi - if [[ -z "$componentName" ]] ; then + if [[ -n "$componentName" ]] ; then componentName="-$componentName" fi + echo "gs://${PROJECT_ID}-dataconsumption-export/${exportId}${componentName}.csv" } @@ -175,18 +176,19 @@ function importedCsvFilename { local componentName=$3 if [[ -z "$exportId" ]] ; then - echo "$0 Internal error: importedCsvFilename got a null exportId" + echo "$0 ERROR: importedCsvFilename got a null exportId" exit 1 fi if [[ -z "$importDirectory" ]] ; then - echo "$0 Internal error: importDirectory got a null exportId" + echo "$0 ERROR: importDirectory got a null exportId" exit 1 fi if [[ -n "$componentName" ]] ; then componentName="-$componentName" fi + echo "${importDirectory}/${exportId}${componentName}.csv" } @@ -201,22 +203,29 @@ EXPORT_ID="$(exportDataFromExporterPod)" echo "EXPORT_ID = $EXPORT_ID" # -# Get the IDs of the various parts being exported +# Copy all the export artifacts from gs:/ to local filesystem storage # for component in "purchases" "sub2msisdn" "" ; do + source="$(gsExportCsvFilename $EXPORT_ID $component)" - destination="$(importedCsvFilename $EXPORT_ID $TARGET_DIR $component)" - gsutil cp $source $destination + if [[ -z "$source" ]] ; then + echo "$0 ERROR: Could not determine source file for export component '$component'" + fi + + destination="$(importedCsvFilename "$EXPORT_ID" "$TARGET_DIR" "$component")" + if [[ -z "$destination" ]] ; then + echo "$0 ERROR: Could not determine destination file for export component '$component'" + fi + + gsutil cp "$source" "$destination" done -echo "EXITING AT LINE $LINENO" -exit - ## ## Generate the yaml output ## +exit SEGMENT_TMPFILE_PSEUDO="tmpsegment-pseudo.csv" @@ -229,6 +238,7 @@ mapPseudosToUserids # Or some such ## Run some script to make sure that we can get deanonumized pseudothing. ## At this point we give the actual content of that file, since we copy it back ## but eventually we may in fact send the URL instead of the actual data, letting + ## the Prime read the dataset from google cloud storage instead. ## (so we should rally copy back $RESULT_SEGMENT_CLEARTEXT_GS insted of the _PSEUDO_ From dc559c7a7f65a58cca39ecd74db7163c4d7afcc5 Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Thu, 27 Sep 2018 13:46:05 +0200 Subject: [PATCH 56/85] Reuse printer. --- .../prime/analytics/publishers/ActiveUsersPublisher.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt index 80961ce86..28274211f 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt @@ -24,10 +24,10 @@ object ActiveUsersPublisher : private val logger by getLogger() private val pseudonymizerService by lazy { getResource() } + private val jsonPrinter = JsonFormat.printer().includingDefaultValueFields() private fun convertToJson(activeUsersInfo: ActiveUsersInfo): ByteString = - ByteString.copyFromUtf8(JsonFormat.printer().includingDefaultValueFields().print(activeUsersInfo)) - + ByteString.copyFromUtf8(jsonPrinter.print(activeUsersInfo)) fun publish(userList: List) { val timestamp = Instant.now().toEpochMilli() From c778a8ef538491061d3f84b34ce0f97ad351559b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 27 Sep 2018 14:05:06 +0200 Subject: [PATCH 57/85] Closer, but not there yet --- sample-agent/sample-agent.sh | 276 ----------------------------------- 1 file changed, 276 deletions(-) delete mode 100755 sample-agent/sample-agent.sh diff --git a/sample-agent/sample-agent.sh b/sample-agent/sample-agent.sh deleted file mode 100755 index a3fac56c6..000000000 --- a/sample-agent/sample-agent.sh +++ /dev/null @@ -1,276 +0,0 @@ -#!/bin/bash -# IMPORTER_URL=http://127.00.1:8080/importer - -# -## Todo: -## o Refactor using methods, adding comments and structure -## o Make it work (again). -## o add invocation of map_subscribers.sh in the pseudoanonymizer-pod to get -## a result back, using that for generating the yaml. -## o Declare victory with respect to closing the loop, merge the branch, -## then start improving the loop on a daily basis. - - - - -### -### VALIDATING AND PARSING COMMAND LINE PARAMETERS -### - - -# -# Get command line parameter, which should be an existing -# directory in which to store the results -# - -TARGET_DIR=$1 -if [[ -z "$TARGET_DIR" ]] ; then - echo "$0 Missing parameter" - echo "usage $0 target-dir" - exit 1 -fi - -if [[ ! -d "$TARGET_DIR" ]] ; then - echo "$0 $TARGET_DIR is not a directory" - echo "usage $0 target-dir" - exit 1 -fi - - - -### -### PRELIMINARIES -### - -# -# Figure out where this script is running from -# - -SCRIPT=$(readlink -f "$0") -SCRIPTPATH=$(dirname "$SCRIPT") -echo $SCRIPTPATH - - -# -# Check for dependencies being satisfied -# - -DEPENDENCIES="gcloud kubectl gsutil" - -for dep in $DEPENDENCIES ; do - if [[ -z $(which $dep) ]] ; then - echo "ERROR: Could not find dependency $dep" - fi -done - - -# -# Figure out relevant parts of the environment and check their -# sanity. -# - -PROJECT_ID=$(gcloud config get-value project) - -if [[ -z "$PROJECT_ID" ]] ; then - echo "ERROR: Unknown google project ID" - exit 1 -fi - -EXPORTER_PODNAME=$(kubectl get pods | grep exporter- | awk '{print $1}') -if [[ -z "$EXPORTER_PODNAME" ]] ; then - echo "ERROR: Unknown exporter podname" - exit 1 -fi - - - -### -### COMMUNICATION WITH EXPORTER SCRIPTS RUNNING IN A KUBERNETES PODS -### - -# -# Run named script on the inside of the kubernetes exporter pod, -# put the output from running that script into a temporary file, return the -# name of that temporary file as the result of running the function. -# The second argument is the intent of the invocation, and is used -# when producing error messages: -# -# runScriptOnExporterPod /export_data.sh "export data" -# -function runScriptOnExporterPod { - local scriptname=$1 - local intentDescription=$2 - - #TEMPFILE="$(mktemp /tmp/abc-script.XXXXXX)" - # XXX Also should be lowercase - TEMPFILE="tmpfile.txt" - - kubectl exec -it "${EXPORTER_PODNAME}" -- /bin/bash -c "$scriptname" > "$TEMPFILE" - - # Fail if the exec failed - retVal=$? - if [[ $retVal -ne 0 ]]; then - echo "ERROR: Failed to $intentDescription" - cat $TEMPFILE - rm $TEMPFILE - exit 1 - fi - - # Return result by setting resutlvar to be the temporary filename - echo $TEMPFILE -} - - -# -# Create a data export batch, return a string identifying that -# batch. Typical usage: -# EXPORT_ID=$(exportDataFromExporterPod) -# -function exportDataFromExporterPod { - local tmpfilename="$(runScriptOnExporterPod /export_data.sh "export data")" - if [[ -z "$tmpfilename" ]] ; then - echo "$0 ERROR: Running the runScriptOnExporterPod failed to return the name of a resultfile." - exit 1 - fi - - local exportId="$(grep "Starting export job for" $tmpfilename | awk '{print $5}' | sed 's/\r$//' )" - - if [[ -z "$exportId" ]] ; then - echo "$0 Could not get export batch from exporter pod" - fi - rm $tmpfilename - echo $exportId -} - -function mapPseudosToUserids { - local tmpfile="$(runScriptOnExporterPod /map_subscribers.sh "mapping pseudoids to subscriber ids")" - - # XXX Map, then transform$(runScriptOnExporterPod /export_data.sh "export data")$(runScriptOnExporterPod /export_data.sh "export data") -} - -# -# Generate the Google filesystem names of components associated with -# a particular export ID: Typical usage -# -# PURCHASES_GS="$(gsExportCsvFilename "ab234245cvsr" "purchases")" - -function gsExportCsvFilename { - local exportId=$1 - local componentName=$2 - if [[ -z "$exportId" ]] ; then - echo "$0 ERROR: gsExportCsvFilename got a null exportId" - exit 1 - fi - if [[ -n "$componentName" ]] ; then - componentName="-$componentName" - fi - - echo "gs://${PROJECT_ID}-dataconsumption-export/${exportId}${componentName}.csv" -} - - - -function importedCsvFilename { - local exportId=$1 - local importDirectory=$2 - local componentName=$3 - - if [[ -z "$exportId" ]] ; then - echo "$0 ERROR: importedCsvFilename got a null exportId" - exit 1 - fi - - if [[ -z "$importDirectory" ]] ; then - echo "$0 ERROR: importDirectory got a null exportId" - exit 1 - fi - - if [[ -n "$componentName" ]] ; then - componentName="-$componentName" - fi - - echo "${importDirectory}/${exportId}${componentName}.csv" -} - - -### -### MAIN SCRIPT -### - - -# EXPORT_ID=0802c66be1ce4e2dba22f988b3ce24f7 -EXPORT_ID="$(exportDataFromExporterPod)" -echo "EXPORT_ID = $EXPORT_ID" - -# -# Copy all the export artifacts from gs:/ to local filesystem storage -# - -for component in "purchases" "sub2msisdn" "" ; do - - source="$(gsExportCsvFilename $EXPORT_ID $component)" - if [[ -z "$source" ]] ; then - echo "$0 ERROR: Could not determine source file for export component '$component'" - fi - - destination="$(importedCsvFilename "$EXPORT_ID" "$TARGET_DIR" "$component")" - if [[ -z "$destination" ]] ; then - echo "$0 ERROR: Could not determine destination file for export component '$component'" - fi - - gsutil cp "$source" "$destination" -done - - -## -## Generate the yaml output -## -exit - - -SEGMENT_TMPFILE_PSEUDO="tmpsegment-pseudo.csv" -SEGMENT_TMPFILE_CLEAR="tmpsegment-clear.csv" -awk -F, '!/^subscriberId/{print $1'} $(importedCsvVilename $TARGET_DIR "sub2msisdn") > $SEGMENT_TMPFILE_PSEUDO -gsutil cp $SEGMENT_TMPFILE_PSEUDO $RESULT_SEGMENT_PSEUDO_GS - -mapPseudosToUserids # Or some such - -## Run some script to make sure that we can get deanonumized pseudothing. -## At this point we give the actual content of that file, since we copy it back -## but eventually we may in fact send the URL instead of the actual data, letting - -## the Prime read the dataset from google cloud storage instead. - -## (so we should rally copy back $RESULT_SEGMENT_CLEARTEXT_GS insted of the _PSEUDO_ -## file) - -gsutil cp $RESULT_SEGMENT_PSEUDO_GS $SEGMENT_TMPFILE_CLEAR - -IMPORTFILE_YML=tmpfile.yml - -cat > $IMPORTFILE_YML <> $IMPORTFILE_YML - -## Send it to the importer -echo curl --data-binary @$IMPORTFILE_YML $IMPORTER_URL - -rm $SEGMENT_TMPFILE_PSEUDO -rm $SEGMENT_TMPFILE_CLEAR From d4ca8b5a2d59c1dbe712e0082257d24f1ec09955 Mon Sep 17 00:00:00 2001 From: Martin Cederlof Date: Thu, 27 Sep 2018 14:24:49 +0200 Subject: [PATCH 58/85] Fix pushing active user list over grpc from ocsgw --- .../org/ostelco/prime/storage/graph/GraphStoreTest.kt | 2 +- .../kotlin/org/ostelco/prime/storage/graph/SchemaTest.kt | 2 +- .../java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java | 6 +++--- .../kotlin/org/ostelco/prime/TestPrimeConfig.kt | 2 +- .../kotlin/org/ostelco/prime/ocs/OcsTest.kt | 2 +- .../org/ostelco/prime/storage/graph/Neo4jStorageTest.kt | 2 +- scripts/deploy-ocsgw.sh | 5 +++++ 7 files changed, 13 insertions(+), 8 deletions(-) diff --git a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt index 3de14305f..f0ca74eb5 100644 --- a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt +++ b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt @@ -228,7 +228,7 @@ class GraphStoreTest { HealthChecks.toRespond2xxOverHttp(7474) { port -> port.inFormat("http://\$HOST:\$EXTERNAL_PORT/browser") }, - Duration.standardSeconds(20L)) + Duration.standardSeconds(40L)) .build() @BeforeClass diff --git a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/SchemaTest.kt b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/SchemaTest.kt index dda1dc303..ddc64ff2f 100644 --- a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/SchemaTest.kt +++ b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/SchemaTest.kt @@ -201,7 +201,7 @@ class SchemaTest { HealthChecks.toRespond2xxOverHttp(7474) { port -> port.inFormat("http://\$HOST:\$EXTERNAL_PORT/browser") }, - Duration.standardSeconds(10L)) + Duration.standardSeconds(40L)) .build() @BeforeClass diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java index 8fd2b68f7..3ee6dff8c 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java @@ -287,11 +287,11 @@ private void updateAnalytics() { OcsgwAnalyticsReport.Builder builder = OcsgwAnalyticsReport.newBuilder().setActiveSessions(sessionIdMap.size()); sessionIdMap.forEach((msisdn, sessionContext) -> { try { - String apn = ccrMap.get(msisdn).getCreditControlRequest().getServiceInformation().get(0).getPsInformation().get(0).getCalledStationId(); - String mncMcc = ccrMap.get(msisdn).getCreditControlRequest().getServiceInformation().get(0).getPsInformation().get(0).getSgsnMccMnc(); + String apn = ccrMap.get(sessionContext.getSessionId()).getCreditControlRequest().getServiceInformation().get(0).getPsInformation().get(0).getCalledStationId(); + String mncMcc = ccrMap.get(sessionContext.getSessionId()).getCreditControlRequest().getServiceInformation().get(0).getPsInformation().get(0).getSgsnMccMnc(); builder.addUsers(User.newBuilder().setApn(apn).setMncMcc(mncMcc).setMsisdn(msisdn).build()); } catch (Exception e) { - LOG.info("Failed to match session info to ccr map"); + LOG.error("Failed to match session info to ccr map", e); } }); ocsgwAnalytics.sendAnalytics(builder.build()); diff --git a/prime/src/integration-tests/kotlin/org/ostelco/prime/TestPrimeConfig.kt b/prime/src/integration-tests/kotlin/org/ostelco/prime/TestPrimeConfig.kt index ef1885e27..f90e11ae6 100644 --- a/prime/src/integration-tests/kotlin/org/ostelco/prime/TestPrimeConfig.kt +++ b/prime/src/integration-tests/kotlin/org/ostelco/prime/TestPrimeConfig.kt @@ -39,7 +39,7 @@ class TestPrimeConfig { HealthChecks.toRespond2xxOverHttp(7474) { port -> port.inFormat("http://\$HOST:\$EXTERNAL_PORT/browser") }, - Duration.standardSeconds(20L)) + Duration.standardSeconds(40L)) .build() @JvmStatic diff --git a/prime/src/integration-tests/kotlin/org/ostelco/prime/ocs/OcsTest.kt b/prime/src/integration-tests/kotlin/org/ostelco/prime/ocs/OcsTest.kt index f7c47c525..ab6e08749 100644 --- a/prime/src/integration-tests/kotlin/org/ostelco/prime/ocs/OcsTest.kt +++ b/prime/src/integration-tests/kotlin/org/ostelco/prime/ocs/OcsTest.kt @@ -217,7 +217,7 @@ class OcsTest { HealthChecks.toRespond2xxOverHttp(7474) { port -> port.inFormat("http://\$HOST:\$EXTERNAL_PORT/browser") }, - Duration.standardSeconds(20L)) + Duration.standardSeconds(40L)) .build() @BeforeClass diff --git a/prime/src/integration-tests/kotlin/org/ostelco/prime/storage/graph/Neo4jStorageTest.kt b/prime/src/integration-tests/kotlin/org/ostelco/prime/storage/graph/Neo4jStorageTest.kt index cefb2fc6c..2dd16ae3d 100644 --- a/prime/src/integration-tests/kotlin/org/ostelco/prime/storage/graph/Neo4jStorageTest.kt +++ b/prime/src/integration-tests/kotlin/org/ostelco/prime/storage/graph/Neo4jStorageTest.kt @@ -105,7 +105,7 @@ class Neo4jStorageTest { HealthChecks.toRespond2xxOverHttp(7474) { port -> port.inFormat("http://\$HOST:\$EXTERNAL_PORT/browser") }, - Duration.standardSeconds(30L)) + Duration.standardSeconds(40L)) .build() @JvmStatic diff --git a/scripts/deploy-ocsgw.sh b/scripts/deploy-ocsgw.sh index ad84ea7c5..30169a00c 100755 --- a/scripts/deploy-ocsgw.sh +++ b/scripts/deploy-ocsgw.sh @@ -18,6 +18,11 @@ echo "Starting to deploy OCSGW to $variant" echo "The last thing this script will do is to look at logs from the ocsgw" echo "It will continue to do so until terminated by ^C" +if [ ! -f build/deploy/ostelco-core-${variant}.zip ]; then + echo "build/deploy/ostelco-core-${variant}.zip not found!" + echo "Did you forget gradle pack ?" + exit 1 +fi scp -oProxyJump=loltel@10.6.101.1 build/deploy/ostelco-core-${variant}.zip ubuntu@${host_ip}:. ssh -A -Jloltel@10.6.101.1 ubuntu@${host_ip} < Date: Thu, 27 Sep 2018 14:44:55 +0200 Subject: [PATCH 59/85] The sample agent we're working on --- sample-agent/sample-agent.sh | 325 +++++++++++++++++++++++++++++++++++ 1 file changed, 325 insertions(+) create mode 100644 sample-agent/sample-agent.sh diff --git a/sample-agent/sample-agent.sh b/sample-agent/sample-agent.sh new file mode 100644 index 000000000..6f394f0d2 --- /dev/null +++ b/sample-agent/sample-agent.sh @@ -0,0 +1,325 @@ +#!/bin/bash + +set -e +# IMPORTER_URL=http://127.00.1:8080/importer + +# +## Todo: +## o Refactor using methods, adding comments and structure +## o Make it work (again). +## o add invocation of map_subscribers.sh in the pseudoanonymizer-pod to get +## a result back, using that for generating the yaml. +## o Declare victory with respect to closing the loop, merge the branch, +## then start improving the loop on a daily basis. + + + + +### +### VALIDATING AND PARSING COMMAND LINE PARAMETERS +### + + +# +# Get command line parameter, which should be an existing +# directory in which to store the results +# + +TARGET_DIR=$1 +if [[ -z "$TARGET_DIR" ]] ; then + echo "$0 Missing parameter" + echo "usage $0 target-dir" + exit 1 +fi + +if [[ ! -d "$TARGET_DIR" ]] ; then + echo "$0 $TARGET_DIR is not a directory" + echo "usage $0 target-dir" + exit 1 +fi + + + +### +### PRELIMINARIES +### + + +# Be able to die from inside procedures + +trap "exit 1" TERM +export TOP_PID=$$ + +function die() { + kill -s TERM $TOP_PID +} + +# +# Check for dependencies being satisfied +# + +DEPENDENCIES="gcloud kubectl gsutil" + +for dep in $DEPENDENCIES ; do + if [[ -z $(which $dep) ]] ; then + echo "ERROR: Could not find dependency $dep" + fi +done + + +# +# Figure out relevant parts of the environment and check their +# sanity. +# + +PROJECT_ID=$(gcloud config get-value project) + +if [[ -z "$PROJECT_ID" ]] ; then + echo "ERROR: Unknown google project ID" + exit 1 +fi + +EXPORTER_PODNAME=$(kubectl get pods | grep exporter- | awk '{print $1}') +if [[ -z "$EXPORTER_PODNAME" ]] ; then + echo "ERROR: Unknown exporter podname" + exit 1 +fi + + + +### +### COMMUNICATION WITH EXPORTER SCRIPTS RUNNING IN A KUBERNETES PODS +### + +# +# Run named script on the inside of the kubernetes exporter pod, +# put the output from running that script into a temporary file, return the +# name of that temporary file as the result of running the function. +# The second argument is the intent of the invocation, and is used +# when producing error messages: +# +# runScriptOnExporterPod /export_data.sh "export data" +# +function runScriptOnExporterPod { + if [[ $# -ne 2 ]] ; then + echo "$0 ERROR: runScriptOnExporterPod requires exactly two parameters" + fi + local scriptname=$1 + local intentDescription=$2 + + #TEMPFILE="$(mktemp /tmp/abc-script.XXXXXX)" + # XXX Also should be lowercase + TEMPFILE="tmpfile.txt" + + kubectl exec -it "${EXPORTER_PODNAME}" -- /bin/bash -c "$scriptname" > "$TEMPFILE" + + # Fail if the exec failed + retVal=$? + if [[ $retVal -ne 0 ]]; then + echo "ERROR: Failed to $intentDescription" + cat $TEMPFILE + rm $TEMPFILE + die + fi + + # Return result by setting resutlvar to be the temporary filename + echo $TEMPFILE +} + + +# +# Create a data export batch, return a string identifying that +# batch. Typical usage: +# EXPORT_ID=$(exportDataFromExporterPod) +# +function exportDataFromExporterPod { + local tmpfilename="$(runScriptOnExporterPod /export_data.sh "export data")" + if [[ -z "$tmpfilename" ]] ; then + echo "$0 ERROR: Running the runScriptOnExporterPod failed to return the name of a resultfile." + die + fi + + local exportId="$(grep "Starting export job for" $tmpfilename | awk '{print $5}' | sed 's/\r$//' )" + + if [[ -z "$exportId" ]] ; then + echo "$0 Could not get export batch from exporter pod" + fi + rm $tmpfilename + echo $exportId +} + +function mapPseudosToUserids { + local tmpfile="$(runScriptOnExporterPod /map_subscribers.sh "mapping pseudoids to subscriber ids")" + # XXX Should doe some more checking here before deleting the file + # rm $tmpfile +} + +# +# Generate the Google filesystem names of components associated with +# a particular export ID: Typical usage +# +# PURCHASES_GS="$(gsExportCsvFilename "ab234245cvsr" "purchases")" + +function gsExportCsvFilename { + if [[ $# -ne 2 ]] ; then + echo "$0 ERROR: gsExportCsvFilename requires exactly two parameters, got '$@'" + die + fi + + local exportId=$1 + local componentName=$2 + if [[ -z "$exportId" ]] ; then + echo "$0 ERROR: gsExportCsvFilename got a null exportId" + die + fi + if [[ -n "$componentName" ]] ; then + componentName="-$componentName" + fi + + echo "gs://${PROJECT_ID}-dataconsumption-export/${exportId}${componentName}.csv" +} + + + +function importedCsvFilename { + if [[ $# -ne 3 ]] ; then + echo "$0 ERROR: importedCsvFilename requires exactly three parameters, got $@" + die + fi + + local exportId=$1 + local importDirectory=$2 + local componentName=$3 + + if [[ -z "$exportId" ]] ; then + echo "$0 ERROR: importedCsvFilename got a null exportId" + die + fi + + if [[ -z "$importDirectory" ]] ; then + echo "$0 ERROR: importDirectory got a null exportId" + die + fi + + if [[ -n "$componentName" ]] ; then + componentName="-$componentName" + fi + + echo "${importDirectory}/${exportId}${componentName}.csv" +} + + +### +### MAIN SCRIPT +### + + + +EXPORT_ID="$(exportDataFromExporterPod)" +echo "EXPORT_ID = $EXPORT_ID" +if [[ -z "$EXPORT_ID" ]] ; then + echo "$0 ERROR: Could not determine export id" +fi + +# +# Copy all the export artifacts from gs:/ to local filesystem storage +# + +for component in "purchases" "sub2msisdn" "" ; do + + source="$(gsExportCsvFilename "$EXPORT_ID" "$component")" + if [[ -z "$source" ]] ; then + echo "$0 ERROR: Could not determine source file for export component '$component'" + fi + + destination="$(importedCsvFilename "$EXPORT_ID" "$TARGET_DIR" "$component")" + if [[ -z "$destination" ]] ; then + echo "$0 ERROR: Could not determine destination file for export component '$component'" + fi + + gsutil cp "$source" "$destination" +done + + +## +## Generate a sample segment by just ripping out +## all the subscriber IDs in the sub2msisdn file. +## +## This is clearly not a realistic scenario, much to simple +## but it is formally correct so it will serve as a placeholder +## until we get something more realistic. +## + +SEGMENT_TMPFILE_PSEUDO="$(importedCsvFilename "$EXPORT_ID" "$TARGET_DIR" "tmpsegment-pseudo")" +awk -F, '!/^subscriberId/{print $1'} $(importedCsvFilename "$EXPORT_ID" "$TARGET_DIR" "sub2msisdn") > $SEGMENT_TMPFILE_PSEUDO + + +## +## Convert from pseudos to actual IDs +## + +# Copy the segment pseudo file to gs +RESULT_SEGMENT_PSEUDO_GS="$(gsExportCsvFilename "$EXPORT_ID" "resultsegment-pseudoanonymized")" +gsutil cp $SEGMENT_TMPFILE_PSEUDO $RESULT_SEGMENT_PSEUDO_GS + +# Then run the script that will convert it into a none-anonymized +# file and fetch the results from gs:/ +mapPseudosToUserids +RESULT_SEGMENT_CLEAR_GS="$(gsExportCsvFilename "$EXPORT_ID" "resultsegment-cleartext")" +RESULT_SEGMENT_CLEAR="$(importedCsvFilename "$EXPORT_ID" "$TARGET_DIR" "resultsegment-cleartext")" +gsutil cp "$RESULT_SEGMENT_CLEAR_GS" "$RESULT_SEGMENT_CLEAR" + +echo "Just placed the results in the file $RESULT_SEGMENT_CLEAR" +exit + +# Then extract only the column we need (the real userids) + +RESULT_SEGMENT_SINGLE_COLUMN="$(importedCsvFilename "$EXPORT_ID" "$TARGET_DIR" "resultsegment-cleartext")" +awk -F, '!/^subscriberId/{print $1'} $RESULT_SEGMENT_CLEAR > $RESULT_SEGMENT_SINGLE_COLUMN + +## Run some script to make sure that we can get deanonumized pseudothing. +## At this point we give the actual content of that file, since we copy it back +## but eventually we may in fact send the URL instead of the actual data, letting + +## the Prime read the dataset from google cloud storage instead. + +## (so we should rally copy back $RESULT_SEGMENT_CLEARTEXT_GS insted of the _PSEUDO_ +## file) + +gsutil cp "$RESULT_SEGMENT_PSEUDO_GS" "$SEGMENT_TMPFILE_CLEAR" + +exit + +## +## Generate the yaml output +## + + +IMPORTFILE_YML=tmpfile.yml + +cat > $IMPORTFILE_YML <> $IMPORTFILE_YML + +## Send it to the importer +echo curl --data-binary @$IMPORTFILE_YML $IMPORTER_URL + +rm $SEGMENT_TMPFILE_PSEUDO +rm $SEGMENT_TMPFILE_CLEAR From fa915e2971210680e268a1723ade61a81fb61f33 Mon Sep 17 00:00:00 2001 From: Martin Cederlof Date: Thu, 27 Sep 2018 14:45:17 +0200 Subject: [PATCH 60/85] Added keepalive to ocsgw metrics --- analytics-grpc-api/src/main/proto/prime_metrics.proto | 1 + .../org/ostelco/prime/analytics/AnalyticsGrpcService.kt | 6 ++++-- .../java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java | 1 + .../main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java | 4 ++-- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/analytics-grpc-api/src/main/proto/prime_metrics.proto b/analytics-grpc-api/src/main/proto/prime_metrics.proto index f7c8ac851..8f76627d9 100644 --- a/analytics-grpc-api/src/main/proto/prime_metrics.proto +++ b/analytics-grpc-api/src/main/proto/prime_metrics.proto @@ -14,6 +14,7 @@ service OcsgwAnalyticsService { message OcsgwAnalyticsReport { uint32 activeSessions = 1; repeated User users = 2; + bool keepAlive = 3; } message User { diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt index d7e94fcbe..bf63afd51 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt @@ -48,8 +48,10 @@ class AnalyticsGrpcService : OcsgwAnalyticsServiceGrpc.OcsgwAnalyticsServiceImpl * @param request provides current active session as a counter with a timestamp */ override fun onNext(request: OcsgwAnalyticsReport) { - CustomMetricsRegistry.updateMetricValue(ACTIVE_SESSIONS, request.activeSessions.toLong()) - ActiveUsersPublisher.publish(request.usersList) + if (!request.keepAlive) { + CustomMetricsRegistry.updateMetricValue(ACTIVE_SESSIONS, request.activeSessions.toLong()) + ActiveUsersPublisher.publish(request.usersList) + } } override fun onError(t: Throwable) { diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java index 3ee6dff8c..4a471e05c 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java @@ -285,6 +285,7 @@ private void updateAnalytics() { OcsgwAnalyticsReport.Builder builder = OcsgwAnalyticsReport.newBuilder().setActiveSessions(sessionIdMap.size()); + builder.setKeepAlive(false); sessionIdMap.forEach((msisdn, sessionContext) -> { try { String apn = ccrMap.get(sessionContext.getSessionId()).getCreditControlRequest().getServiceInformation().get(0).getPsInformation().get(0).getCalledStationId(); diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java index 0e3790626..4e8524cff 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java @@ -42,7 +42,7 @@ class OcsgwMetrics { private ScheduledFuture keepAliveFuture = null; - private OcsgwAnalyticsReport lastActiveSessions = OcsgwAnalyticsReport.newBuilder().build(); + private OcsgwAnalyticsReport lastActiveSessions = OcsgwAnalyticsReport.newBuilder().setKeepAlive(true).build(); OcsgwMetrics(String metricsServerHostname, ServiceAccountJwtAccessCredentials credentials) { @@ -126,7 +126,7 @@ public void onNext(OcsgwAnalyticsReply value) { private void initKeepAlive() { // this is used to keep connection alive keepAliveFuture = executorService.scheduleWithFixedDelay(() -> { - sendAnalytics(lastActiveSessions); + sendAnalytics(OcsgwAnalyticsReport.newBuilder().setKeepAlive(true).build()); }, 15, 50, From 61d0fc3e927460c1242fb325f8b0f1e38c6f2229 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 27 Sep 2018 15:04:19 +0200 Subject: [PATCH 61/85] If there was actual data in the consumptiond data, this might actually work :-) --- sample-agent/sample-agent.sh | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/sample-agent/sample-agent.sh b/sample-agent/sample-agent.sh index 6f394f0d2..fb7a7ee95 100644 --- a/sample-agent/sample-agent.sh +++ b/sample-agent/sample-agent.sh @@ -258,37 +258,29 @@ awk -F, '!/^subscriberId/{print $1'} $(importedCsvFilename "$EXPORT_ID" "$TARGET ## Convert from pseudos to actual IDs ## + +RESULTSEG_PSEUDO_BASENAME="resultsegment-pseudoanonymized" +RESULTSEG_CLEARTEXT_BASENAME="resultsegment-cleartext" +RESULT_SEGMENT_PSEUDO_GS="$(gsExportCsvFilename "$EXPORT_ID" "$RESULTSEG_PSEUDO_BASENAME")" +RESULT_SEGMENT_CLEAR_GS="$(gsExportCsvFilename "$EXPORT_ID" "$RESULTSEG_CLEARTEXT_BASENAME")" +RESULT_SEGMENT_CLEAR="$(importedCsvFilename "$EXPORT_ID" "$TARGET_DIR" "$RESULTSEG_CLEARTEXT_BASENAME")" +RESULT_SEGMENT_SINGLE_COLUMN="$(importedCsvFilename "$EXPORT_ID" "$TARGET_DIR" "$RESULTSEG_CLEARTEXT_BASENAME")" + # Copy the segment pseudo file to gs -RESULT_SEGMENT_PSEUDO_GS="$(gsExportCsvFilename "$EXPORT_ID" "resultsegment-pseudoanonymized")" + gsutil cp $SEGMENT_TMPFILE_PSEUDO $RESULT_SEGMENT_PSEUDO_GS # Then run the script that will convert it into a none-anonymized # file and fetch the results from gs:/ mapPseudosToUserids -RESULT_SEGMENT_CLEAR_GS="$(gsExportCsvFilename "$EXPORT_ID" "resultsegment-cleartext")" -RESULT_SEGMENT_CLEAR="$(importedCsvFilename "$EXPORT_ID" "$TARGET_DIR" "resultsegment-cleartext")" + gsutil cp "$RESULT_SEGMENT_CLEAR_GS" "$RESULT_SEGMENT_CLEAR" echo "Just placed the results in the file $RESULT_SEGMENT_CLEAR" -exit - # Then extract only the column we need (the real userids) -RESULT_SEGMENT_SINGLE_COLUMN="$(importedCsvFilename "$EXPORT_ID" "$TARGET_DIR" "resultsegment-cleartext")" -awk -F, '!/^subscriberId/{print $1'} $RESULT_SEGMENT_CLEAR > $RESULT_SEGMENT_SINGLE_COLUMN - -## Run some script to make sure that we can get deanonumized pseudothing. -## At this point we give the actual content of that file, since we copy it back -## but eventually we may in fact send the URL instead of the actual data, letting - -## the Prime read the dataset from google cloud storage instead. - -## (so we should rally copy back $RESULT_SEGMENT_CLEARTEXT_GS insted of the _PSEUDO_ -## file) - -gsutil cp "$RESULT_SEGMENT_PSEUDO_GS" "$SEGMENT_TMPFILE_CLEAR" +awk -F, '!/^pseudoId/{print $2'} $RESULT_SEGMENT_CLEAR > $RESULT_SEGMENT_SINGLE_COLUMN -exit ## ## Generate the yaml output @@ -316,10 +308,11 @@ EOF # Adding the list of subscribers in clear text (indented six spaces # with a leading "-" as per YAML list syntax. -awk '{print " - " $1}' $SEGMENT_IMPORTFILE_CLEAR >> $IMPORTFILE_YML +awk '{print " - " $1}' $RESULT_SEGMENT_SINGLE_COLUMN >> $IMPORTFILE_YML ## Send it to the importer echo curl --data-binary @$IMPORTFILE_YML $IMPORTER_URL +## And whatever else, some obviousl missing. rm $SEGMENT_TMPFILE_PSEUDO rm $SEGMENT_TMPFILE_CLEAR From 414091263dbb847cf4c79754fb612ec98d36922d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 27 Sep 2018 15:14:39 +0200 Subject: [PATCH 62/85] Enable upload via curl --- sample-agent/sample-agent.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/sample-agent/sample-agent.sh b/sample-agent/sample-agent.sh index fb7a7ee95..443d7fcef 100644 --- a/sample-agent/sample-agent.sh +++ b/sample-agent/sample-agent.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -# IMPORTER_URL=http://127.00.1:8080/importer + # ## Todo: @@ -85,6 +85,15 @@ if [[ -z "$EXPORTER_PODNAME" ]] ; then exit 1 fi +PRIME_PODNAME=$(kubectl get pods | grep prime- | awk '{print $1}') +if [[ -z "$PRIME_PODNAME" ]] ; then + echo "ERROR: Unknown prime podname" + exit 1 +fi + +echo "$0: Assuming that prime is running at $PRIME_PODNAME" +echo "$0: and that you have done" +echo "$0: kubectl port-forward $PRIME_PODNAME 8080:8080" ### @@ -311,7 +320,10 @@ EOF awk '{print " - " $1}' $RESULT_SEGMENT_SINGLE_COLUMN >> $IMPORTFILE_YML ## Send it to the importer -echo curl --data-binary @$IMPORTFILE_YML $IMPORTER_URL + +echo "XXX TODO: Make it so that this thing actually does something useful" +IMPORTER_URL=http://127.0.0.1:8080/importer +curl --data-binary @$IMPORTFILE_YML $IMPORTER_URL ## And whatever else, some obviousl missing. rm $SEGMENT_TMPFILE_PSEUDO From 4a7d8fb56d4e6a88eda830cf5e7896088eb5c9ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 27 Sep 2018 15:26:08 +0200 Subject: [PATCH 63/85] misc cleanup --- sample-agent/sample-agent.sh | 46 +++++++++++++++--------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/sample-agent/sample-agent.sh b/sample-agent/sample-agent.sh index 443d7fcef..c2f683400 100644 --- a/sample-agent/sample-agent.sh +++ b/sample-agent/sample-agent.sh @@ -2,24 +2,10 @@ set -e - -# -## Todo: -## o Refactor using methods, adding comments and structure -## o Make it work (again). -## o add invocation of map_subscribers.sh in the pseudoanonymizer-pod to get -## a result back, using that for generating the yaml. -## o Declare victory with respect to closing the loop, merge the branch, -## then start improving the loop on a daily basis. - - - - ### ### VALIDATING AND PARSING COMMAND LINE PARAMETERS ### - # # Get command line parameter, which should be an existing # directory in which to store the results @@ -38,13 +24,10 @@ if [[ ! -d "$TARGET_DIR" ]] ; then exit 1 fi - - ### ### PRELIMINARIES ### - # Be able to die from inside procedures trap "exit 1" TERM @@ -66,7 +49,6 @@ for dep in $DEPENDENCIES ; do fi done - # # Figure out relevant parts of the environment and check their # sanity. @@ -116,9 +98,12 @@ function runScriptOnExporterPod { local scriptname=$1 local intentDescription=$2 - #TEMPFILE="$(mktemp /tmp/abc-script.XXXXXX)" - # XXX Also should be lowercase + # TEMPFILE="$(mktemp /tmp/abc-script.XXXXXX)" + # XXX The tmpfile is the same thing all the time, bad practice, but + # until I figure out how to make tempfiles dependent on the top + # level process's lifetime, I'll do it this way. TEMPFILE="tmpfile.txt" + [[ -f "$TMPFILE" ]] && rm "$TMPFILE" kubectl exec -it "${EXPORTER_PODNAME}" -- /bin/bash -c "$scriptname" > "$TEMPFILE" @@ -159,8 +144,7 @@ function exportDataFromExporterPod { function mapPseudosToUserids { local tmpfile="$(runScriptOnExporterPod /map_subscribers.sh "mapping pseudoids to subscriber ids")" - # XXX Should doe some more checking here before deleting the file - # rm $tmpfile + [[ -f "$tmpfile" ]] && rm "$tmpfile" } # @@ -189,7 +173,9 @@ function gsExportCsvFilename { } - +# +# Generate a filename +# function importedCsvFilename { if [[ $# -ne 3 ]] ; then echo "$0 ERROR: importedCsvFilename requires exactly three parameters, got $@" @@ -319,12 +305,18 @@ EOF # with a leading "-" as per YAML list syntax. awk '{print " - " $1}' $RESULT_SEGMENT_SINGLE_COLUMN >> $IMPORTFILE_YML +## ## Send it to the importer +## (assuming the kubectl port forwarding is enabled) -echo "XXX TODO: Make it so that this thing actually does something useful" IMPORTER_URL=http://127.0.0.1:8080/importer curl --data-binary @$IMPORTFILE_YML $IMPORTER_URL -## And whatever else, some obviousl missing. -rm $SEGMENT_TMPFILE_PSEUDO -rm $SEGMENT_TMPFILE_CLEAR + +## +## Remove tempfiles +## + +# .... eventually + + From c2befb24491ade8a494a8a5a7dc83354c43b115d Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Thu, 27 Sep 2018 15:43:56 +0200 Subject: [PATCH 64/85] Rename the mcc mnc field. --- analytics-grpc-api/src/main/proto/analytics.proto | 2 +- analytics-grpc-api/src/main/proto/prime_metrics.proto | 2 +- .../prime/analytics/publishers/ActiveUsersPublisher.kt | 2 +- .../main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/analytics-grpc-api/src/main/proto/analytics.proto b/analytics-grpc-api/src/main/proto/analytics.proto index d15a99976..220943f57 100644 --- a/analytics-grpc-api/src/main/proto/analytics.proto +++ b/analytics-grpc-api/src/main/proto/analytics.proto @@ -29,7 +29,7 @@ message AggregatedDataTrafficInfo { message User { string msisdn = 1; string apn = 2; - string mnc_mcc = 3; + string mccMnc = 3; } message ActiveUsersInfo { diff --git a/analytics-grpc-api/src/main/proto/prime_metrics.proto b/analytics-grpc-api/src/main/proto/prime_metrics.proto index 8f76627d9..318e4bdfb 100644 --- a/analytics-grpc-api/src/main/proto/prime_metrics.proto +++ b/analytics-grpc-api/src/main/proto/prime_metrics.proto @@ -20,7 +20,7 @@ message OcsgwAnalyticsReport { message User { string msisdn = 1; string apn = 2; - string mnc_mcc = 3; + string mccMnc = 3; } message OcsgwAnalyticsReply { diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt index 28274211f..69ef2c7fb 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt @@ -35,7 +35,7 @@ object ActiveUsersPublisher : for (user in userList) { val userBuilder = org.ostelco.analytics.api.User.newBuilder() val pseudonym = pseudonymizerService.getSubscriberIdPseudonym(user.msisdn, timestamp).pseudonym - activeUsersInfoBuilder.addUsers(userBuilder.setApn(user.apn).setMncMcc(user.mncMcc).setMsisdn(pseudonym).build()) + activeUsersInfoBuilder.addUsers(userBuilder.setApn(user.apn).setMccMnc(user.mccMnc).setMsisdn(pseudonym).build()) } val pubsubMessage = PubsubMessage.newBuilder() diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java index 4a471e05c..b4898976c 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java @@ -289,8 +289,8 @@ private void updateAnalytics() { sessionIdMap.forEach((msisdn, sessionContext) -> { try { String apn = ccrMap.get(sessionContext.getSessionId()).getCreditControlRequest().getServiceInformation().get(0).getPsInformation().get(0).getCalledStationId(); - String mncMcc = ccrMap.get(sessionContext.getSessionId()).getCreditControlRequest().getServiceInformation().get(0).getPsInformation().get(0).getSgsnMccMnc(); - builder.addUsers(User.newBuilder().setApn(apn).setMncMcc(mncMcc).setMsisdn(msisdn).build()); + String mccMnc = ccrMap.get(sessionContext.getSessionId()).getCreditControlRequest().getServiceInformation().get(0).getPsInformation().get(0).getSgsnMccMnc(); + builder.addUsers(User.newBuilder().setApn(apn).setMccMnc(mccMnc).setMsisdn(msisdn).build()); } catch (Exception e) { LOG.error("Failed to match session info to ccr map", e); } From 40e10969a400544c784297773b40d7d6e401c440 Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Thu, 27 Sep 2018 15:52:37 +0200 Subject: [PATCH 65/85] Fix the usgae of pseudonymiser. --- .../ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt index 69ef2c7fb..204af1015 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt @@ -34,7 +34,7 @@ object ActiveUsersPublisher : val activeUsersInfoBuilder = ActiveUsersInfo.newBuilder().setTimestamp(Timestamps.fromMillis(timestamp)) for (user in userList) { val userBuilder = org.ostelco.analytics.api.User.newBuilder() - val pseudonym = pseudonymizerService.getSubscriberIdPseudonym(user.msisdn, timestamp).pseudonym + val pseudonym = pseudonymizerService.getMsisdnPseudonym(user.msisdn, timestamp).pseudonym activeUsersInfoBuilder.addUsers(userBuilder.setApn(user.apn).setMccMnc(user.mccMnc).setMsisdn(pseudonym).build()) } From 4a1f368f770f54138ed61e34a931961078bdd2e1 Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Wed, 26 Sep 2018 16:04:31 +0200 Subject: [PATCH 66/85] Adds support for removing payment sources --- .../org/ostelco/at/common/StripePayment.kt | 9 -- .../org/ostelco/at/jersey/HttpClientUtil.kt | 11 +++ .../kotlin/org/ostelco/at/jersey/Tests.kt | 98 ++++++++++++------- .../kotlin/org/ostelco/at/okhttp/Tests.kt | 81 ++++++++++----- .../client/api/resources/PaymentResource.kt | 25 +++-- .../prime/client/api/store/SubscriberDAO.kt | 2 + .../client/api/store/SubscriberDAOImpl.kt | 15 +++ .../StripePaymentProcessorTest.kt | 65 +++++++++--- .../StripePaymentProcessor.kt | 11 ++- .../ostelco/prime/apierror/ApiErrorCodes.kt | 1 + .../paymentprocessor/PaymentProcessor.kt | 3 +- prime/infra/dev/prime-client-api.yaml | 22 +++++ prime/infra/prod/prime-client-api.yaml | 22 +++++ 13 files changed, 277 insertions(+), 88 deletions(-) diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt index 8d7562a9a..ea0e5b232 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt @@ -55,15 +55,6 @@ object StripePayment { return token.card.id } - fun getCardIdForSourceId(sourceId: String) : String { - - // https://stripe.com/docs/api/java#create_source - Stripe.apiKey = System.getenv("STRIPE_API_KEY") - - val source = Source.retrieve(sourceId) - return source.id - } - /** * Obtains 'default source' directly from Stripe. Use in tests to * verify that the correspondng 'setDefaultSource' API works as diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/HttpClientUtil.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/HttpClientUtil.kt index 4426da7e1..c8b04a7fc 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/HttpClientUtil.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/HttpClientUtil.kt @@ -54,6 +54,17 @@ inline fun put(execute: HttpRequest.() -> Unit): T { return response.readEntity(object : GenericType() {}) } +/** + * DSL function for DELETE operation + */ +inline fun delete(execute: HttpRequest.() -> Unit): T { + val request = HttpRequest().apply(execute) + val response = HttpClient.send(request.path, request.queryParams, request.headerParams, request.subscriberId) + .delete() + assertEquals(200, response.status) { response.readEntity(String::class.java) } + return response.readEntity(object : GenericType() {}) +} + fun assertEquals(expected: T, actual: T, lazyMessage: () -> String) { var message = "" if (expected != actual) { diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt index 0ece47245..f86a4da6a 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt @@ -290,7 +290,7 @@ class SourceTest { subscriberId = email } - val ids = createdIds.map { getIdFromStripe(it) } + val ids = createdIds.map { getCardIdForTokenFromStripe(it) } assert(sources.isNotEmpty()) { "Expected at least one payment source for profile $email" } assert(sources.map { it.id }.containsAll(ids)) @@ -307,37 +307,6 @@ class SourceTest { } } - private fun getIdFromStripe(tokenId: String): String { - if (tokenId.startsWith("src_")) { - return StripePayment.getCardIdForSourceId(tokenId) - } - return StripePayment.getCardIdForTokenId(tokenId) - } - - private fun createTokenWithStripe(email: String): String { - val tokenId = StripePayment.createPaymentTokenId() - - post { - path = "/paymentSources" - subscriberId = email - queryParams = mapOf("sourceId" to tokenId) - } - - return tokenId - } - - private fun createSourceWithStripe(email: String): String { - val sourceId = StripePayment.createPaymentSourceId() - - post { - path = "/paymentSources" - subscriberId = email - queryParams = mapOf("sourceId" to sourceId) - } - - return sourceId - } - @Test fun `jersey test - PUT source set default`() { @@ -386,6 +355,69 @@ class SourceTest { StripePayment.deleteCustomer(email = email) } } + + @Test + fun `okhttp test - DELETE source`() { + + StripePayment.deleteAllCustomers() + + val email = "purchase-${randomInt()}@test.com" + createProfile(name = "Test Payment Source", email = email) + + Thread.sleep(200) + + val createdIds = listOf(getCardIdForTokenFromStripe(createTokenWithStripe(email)), + createSourceWithStripe(email)) + + val deletedIds = createdIds.map { it -> removeSourceWithStripe(email, it) } + + assert(createdIds.containsAll(deletedIds.toSet())) { + "Failed to delete one or more sources: ${createdIds.toSet() - deletedIds.toSet()}" + } + } + + // Helpers for source handling with Stripe. + + private fun getCardIdForTokenFromStripe(id: String) : String { + if (id.startsWith("tok_")) { + return StripePayment.getCardIdForTokenId(id) + } + return id + } + + private fun createTokenWithStripe(email: String) : String { + val tokenId = StripePayment.createPaymentTokenId() + + post { + path = "/paymentSources" + subscriberId = email + queryParams = mapOf("sourceId" to tokenId) + } + + return tokenId + } + + private fun createSourceWithStripe(email: String) : String { + val sourceId = StripePayment.createPaymentSourceId() + + post { + path = "/paymentSources" + subscriberId = email + queryParams = mapOf("sourceId" to sourceId) + } + + return sourceId + } + + private fun removeSourceWithStripe(email: String, sourceId: String) : String { + val removedSource = delete { + path = "/paymentSources" + subscriberId = email + queryParams = mapOf("sourceId" to sourceId) + } + + return removedSource.id + } } class PurchaseTest { @@ -750,4 +782,4 @@ class ReferralTest { assertEquals(listOf(freeProductForReferred), secondSubscriptionStatus.purchaseRecords.map { it.product }) } -} \ No newline at end of file +} diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt index 07f1115f1..d301304a6 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt @@ -251,7 +251,7 @@ class SourceTest { val sources = client.listSources() - val ids = createdIds.map { getIdFromStripe(it) } + val ids = createdIds.map { getCardIdForTokenFromStripe(it) } assert(sources.isNotEmpty()) { "Expected at least one payment source for profile $email" } assert(sources.map{ it.id }.containsAll(ids)) @@ -268,29 +268,6 @@ class SourceTest { } } - private fun getIdFromStripe(tokenId : String) : String { - if (tokenId.startsWith("src_")) { - return StripePayment.getCardIdForSourceId(tokenId) - } - return StripePayment.getCardIdForTokenId(tokenId) - } - - private fun createTokenWithStripe(client : DefaultApi) : String { - val tokenId = StripePayment.createPaymentTokenId() - - client.createSource(tokenId) - - return tokenId - } - - private fun createSourceWithStripe(client : DefaultApi) : String { - val sourceId = StripePayment.createPaymentSourceId() - - client.createSource(sourceId) - - return sourceId - } - @Test fun `okhttp test - PUT source set default`() { @@ -329,6 +306,60 @@ class SourceTest { StripePayment.deleteCustomer(email = email) } } + + @Test + fun `okhttp test - DELETE source`() { + + StripePayment.deleteAllCustomers() + + val email = "purchase-${randomInt()}@test.com" + createProfile(name = "Test Payment Source", email = email) + + val client = clientForSubject(subject = email) + + Thread.sleep(200) + + val createdIds = listOf(getCardIdForTokenFromStripe(createTokenWithStripe(client)), + createSourceWithStripe(client)) + + val deletedIds = createdIds.map { it -> deleteSourceWithStripe(client, it) } + + assert(createdIds.containsAll(deletedIds.toSet())) { + "Failed to delete one or more sources: ${createdIds.toSet() - deletedIds.toSet()}" + } + } + + // Helpers for source handling with Stripe. + + private fun getCardIdForTokenFromStripe(id: String) : String { + if (id.startsWith("tok_")) { + return StripePayment.getCardIdForTokenId(id) + } + return id + } + + private fun createTokenWithStripe(client: DefaultApi) : String { + val tokenId = StripePayment.createPaymentTokenId() + + client.createSource(tokenId) + + return tokenId + } + + private fun createSourceWithStripe(client: DefaultApi) : String { + val sourceId = StripePayment.createPaymentSourceId() + + client.createSource(sourceId) + + return sourceId + } + + private fun deleteSourceWithStripe(client : DefaultApi, sourceId : String) : String { + + val removedSource = client.removeSource(sourceId) + + return removedSource.id + } } class PurchaseTest { @@ -593,4 +624,4 @@ class ReferralTest { assertEquals(listOf(freeProductForReferred), secondSubscriptionStatus.purchaseRecords.map { it.product }) } -} \ No newline at end of file +} diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/resources/PaymentResource.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/resources/PaymentResource.kt index 737bb3440..ff050e8d7 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/resources/PaymentResource.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/resources/PaymentResource.kt @@ -5,12 +5,7 @@ import org.ostelco.prime.client.api.auth.AccessTokenPrincipal import org.ostelco.prime.client.api.store.SubscriberDAO import org.ostelco.prime.getLogger import javax.validation.constraints.NotNull -import javax.ws.rs.GET -import javax.ws.rs.POST -import javax.ws.rs.PUT -import javax.ws.rs.Path -import javax.ws.rs.Produces -import javax.ws.rs.QueryParam +import javax.ws.rs.* import javax.ws.rs.core.Response /** @@ -72,4 +67,22 @@ class PaymentResource(private val dao: SubscriberDAO) { { sourceInfo -> Response.status(Response.Status.OK).entity(sourceInfo)} ).build() } + + @DELETE + @Produces("application/json") + fun removeSource(@Auth token: AccessTokenPrincipal?, + @NotNull + @QueryParam("sourceId") + sourceId: String): Response { + if (token == null) { + return Response.status(Response.Status.UNAUTHORIZED) + .build() + } + + return dao.removeSource(token.name, sourceId) + .fold( + { apiError -> Response.status(apiError.status).entity(asJson(apiError)) }, + { sourceInfo -> Response.status(Response.Status.OK).entity(sourceInfo)} + ).build() + } } diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAO.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAO.kt index db00add39..39f377634 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAO.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAO.kt @@ -65,6 +65,8 @@ interface SubscriberDAO { fun listSources(subscriberId: String): Either> + fun removeSource(subscriberId: String, sourceId: String): Either + companion object { /** diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt index 801a84084..c8b4a10f9 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt @@ -350,4 +350,19 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu .mapLeft { mapPaymentErrorToApiError("Failed to list sources", ApiErrorCode.FAILED_TO_FETCH_PAYMENT_SOURCES_LIST, it) } } } + + override fun removeSource(subscriberId: String, sourceId: String): Either { + return paymentProcessor.getPaymentProfile(subscriberId) + .fold( + { + paymentProcessor.createPaymentProfile(subscriberId) + .mapLeft { error -> mapPaymentErrorToApiError(error.description, ApiErrorCode.FAILED_TO_SET_DEFAULT_PAYMENT_SOURCE, error) } + }, + { profileInfo -> Either.right(profileInfo) } + ) + .flatMap { profileInfo -> + paymentProcessor.removeSource(profileInfo.id, sourceId) + .mapLeft { mapPaymentErrorToApiError("Failed to remove payment source", ApiErrorCode.FAILED_TO_REMOVE_PAYMENT_SOURCE, it) } + } + } } diff --git a/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt b/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt index 7cc9029f1..4f1987749 100644 --- a/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt +++ b/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt @@ -1,7 +1,10 @@ package org.ostelco.prime.paymentprocessor import arrow.core.getOrElse +import arrow.core.right +import arrow.core.some import com.stripe.Stripe +import com.stripe.model.Source import com.stripe.model.Token import org.junit.After import org.junit.Before @@ -18,19 +21,40 @@ class StripePaymentProcessorTest { private var stripeCustomerId = "" - fun createPaymentSourceId(): String { + private fun createPaymentTokenId() : String { val cardMap = mapOf( "number" to "4242424242424242", "exp_month" to 8, "exp_year" to 2019, "cvc" to "314") - val tokenMap = mapOf("card" to cardMap) + val token = Token.create(tokenMap) return token.id } + private fun createPaymentSourceId() : String { + + val sourceMap = mapOf( + "type" to "card", + "card" to mapOf( + "number" to "4242424242424242", + "exp_month" to 8, + "exp_year" to 2019, + "cvc" to "314"), + "owner" to mapOf( + "address" to mapOf( + "city" to "Oslo", + "country" to "Norway" + ), + "email" to "me@somewhere.com") + ) + + val source = Source.create(sourceMap) + return source.id + } + private fun addCustomer() { val resultAdd = paymentProcessor.createPaymentProfile(testCustomer) assertEquals(true, resultAdd.isRight()) @@ -73,7 +97,7 @@ class StripePaymentProcessorTest { fun ensureSourcesSorted() { run { - paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) // Ensure that not all sources falls within the same second. Thread.sleep(1_001) paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) @@ -84,18 +108,37 @@ class StripePaymentProcessorTest { val createdTimestamps = sources.getOrElse { fail("The 'created' field is missing from the list of sources: ${sources}") - }.map { it.details["created"] as Long } + }.map { it.details["created"] as Long } val createdTimestampsSorted = createdTimestamps.sortedByDescending { it } - assertEquals(createdTimestampsSorted, createdTimestamps, + assertEquals(createdTimestamps, createdTimestampsSorted, "The list of sources is not in descending sorted order by 'created' timestamp: ${sources}") } + @Test + fun addAndRemoveMultipleSources() { + + val sources= listOf( + paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()), + paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + ) + + val sourcesRemoved = sources.map { + paymentProcessor.removeSource(stripeCustomerId, it.getOrElse { + fail("Failed to remove source ${it}") + }.id) + } + + sourcesRemoved.forEach { it -> + assertEquals(true, it.isRight(), "Unexpected failure when removing source ${it}") + } + } + @Test fun addSourceToCustomerAndRemove() { - val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) val resultStoredSources = paymentProcessor.getSavedSources(stripeCustomerId) assertEquals(1, resultStoredSources.fold({ 0 }, { it.size })) @@ -111,8 +154,8 @@ class StripePaymentProcessorTest { } @Test - fun addSourceToCustomerTwise() { - val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + fun addSourceToCustomerTwice() { + val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) val resultStoredSources = paymentProcessor.getSavedSources(stripeCustomerId) assertEquals(1, resultStoredSources.fold({ 0 }, { it.size })) @@ -133,7 +176,7 @@ class StripePaymentProcessorTest { @Test fun addDefaultSourceAndRemove() { - val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) assertEquals(true, resultAddSource.isRight()) val resultAddDefault = paymentProcessor.setDefaultSource(stripeCustomerId, resultAddSource.fold({ "" }, { it.id })) @@ -149,7 +192,7 @@ class StripePaymentProcessorTest { @Test fun createAuthorizeChargeAndRefund() { - val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) assertEquals(true, resultAddSource.isRight()) val resultAuthorizeCharge = paymentProcessor.authorizeCharge(stripeCustomerId, resultAddSource.fold({ "" }, { it.id }), 1000, "nok") @@ -176,7 +219,7 @@ class StripePaymentProcessorTest { @Test fun subscribeAndUnsubscribePlan() { - val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) assertEquals(true, resultAddSource.isRight()) val resultCreateProduct = paymentProcessor.createProduct("TestSku") diff --git a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt index e54a7ac44..eb5e329be 100644 --- a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt +++ b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt @@ -233,9 +233,16 @@ class StripePaymentProcessor : PaymentProcessor { Refund.create(refundParams).charge } - override fun removeSource(customerId: String, sourceId: String): Either = + override fun removeSource(customerId: String, sourceId: String): Either = either("Failed to remove source $sourceId from customer $customerId") { - Customer.retrieve(customerId).sources.retrieve(sourceId).delete().id + val accountInfo = Customer.retrieve(customerId).sources.retrieve(sourceId) + when (accountInfo) { + is Card -> accountInfo.delete() + is Source -> accountInfo.detach() + else -> + Either.left(BadGatewayError("Attempt to remove unsupported account-type ${accountInfo}")) + } + SourceInfo(sourceId) } private fun either(errorDescription: String, action: () -> RETURN): Either { diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiErrorCodes.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiErrorCodes.kt index 4b0950a03..849144266 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiErrorCodes.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiErrorCodes.kt @@ -18,6 +18,7 @@ enum class ApiErrorCode { FAILED_TO_STORE_PAYMENT_SOURCE, FAILED_TO_SET_DEFAULT_PAYMENT_SOURCE, FAILED_TO_FETCH_PAYMENT_SOURCES_LIST, + FAILED_TO_REMOVE_PAYMENT_SOURCE, FAILED_TO_UPDATE_PROFILE, FAILED_TO_FETCH_CONSENT, FAILED_TO_IMPORT_OFFER, diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/PaymentProcessor.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/PaymentProcessor.kt index 098abdb17..eda338db4 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/PaymentProcessor.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/PaymentProcessor.kt @@ -124,6 +124,5 @@ interface PaymentProcessor { * @param sourceId id of the payment source * @return id if removed */ - fun removeSource(customerId: String, sourceId: String): Either - + fun removeSource(customerId: String, sourceId: String): Either } \ No newline at end of file diff --git a/prime/infra/dev/prime-client-api.yaml b/prime/infra/dev/prime-client-api.yaml index 4f258dfa6..4038789bc 100644 --- a/prime/infra/dev/prime-client-api.yaml +++ b/prime/infra/dev/prime-client-api.yaml @@ -151,6 +151,28 @@ paths: description: "User not found." security: - auth0_jwt: [] + delete: + description: "Remove a payment source for user" + produces: + - application/json + operationId: "removeSource" + parameters: + - name: sourceId + in: query + description: "The stripe-id of the source to be removed" + required: true + type: string + responses: + 200: + description: "Successfully removed the source" + schema: + $ref: '#/definitions/PaymentSource' + 400: + description: "The source could not be removed" + 404: + description: "No such source for user" + security: + - auth0_jwt: [] "/products": get: description: "Get all products for the user." diff --git a/prime/infra/prod/prime-client-api.yaml b/prime/infra/prod/prime-client-api.yaml index 2312044bf..023650751 100644 --- a/prime/infra/prod/prime-client-api.yaml +++ b/prime/infra/prod/prime-client-api.yaml @@ -151,6 +151,28 @@ paths: description: "User not found." security: - auth0_jwt: [] + delete: + description: "Remove a payment source for user" + produces: + - application/json + operationId: "removeSource" + parameters: + - name: sourceId + in: query + description: "The stripe-id of the source to be removed" + required: true + type: string + responses: + 200: + description: "Successfully removed the source" + schema: + $ref: '#/definitions/PaymentSource' + 400: + description: "The source could not be removed" + 404: + description: "No such source for user" + security: + - auth0_jwt: [] "/products": get: description: "Get all products for the user." From e1964d03e8b29db79bf650c6580bef737d84ff31 Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Thu, 27 Sep 2018 16:38:32 +0200 Subject: [PATCH 67/85] Update 'remove source' acceptance tests to new Stripe customer management --- .../kotlin/org/ostelco/at/jersey/Tests.kt | 22 ++++++++++-------- .../kotlin/org/ostelco/at/okhttp/Tests.kt | 23 +++++++++++-------- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt index f86a4da6a..810f69a91 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt @@ -359,20 +359,24 @@ class SourceTest { @Test fun `okhttp test - DELETE source`() { - StripePayment.deleteAllCustomers() - val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Payment Source", email = email) - Thread.sleep(200) + try { + + createProfile(name = "Test Payment Source", email = email) + + Thread.sleep(200) - val createdIds = listOf(getCardIdForTokenFromStripe(createTokenWithStripe(email)), - createSourceWithStripe(email)) + val createdIds = listOf(getCardIdForTokenFromStripe(createTokenWithStripe(email)), + createSourceWithStripe(email)) - val deletedIds = createdIds.map { it -> removeSourceWithStripe(email, it) } + val deletedIds = createdIds.map { it -> removeSourceWithStripe(email, it) } - assert(createdIds.containsAll(deletedIds.toSet())) { - "Failed to delete one or more sources: ${createdIds.toSet() - deletedIds.toSet()}" + assert(createdIds.containsAll(deletedIds.toSet())) { + "Failed to delete one or more sources: ${createdIds.toSet() - deletedIds.toSet()}" + } + } finally { + StripePayment.deleteCustomer(email = email) } } diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt index d301304a6..1a33776aa 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt @@ -310,22 +310,25 @@ class SourceTest { @Test fun `okhttp test - DELETE source`() { - StripePayment.deleteAllCustomers() - val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Payment Source", email = email) - val client = clientForSubject(subject = email) + try { + createProfile(name = "Test Payment Source", email = email) - Thread.sleep(200) + val client = clientForSubject(subject = email) - val createdIds = listOf(getCardIdForTokenFromStripe(createTokenWithStripe(client)), - createSourceWithStripe(client)) + Thread.sleep(200) - val deletedIds = createdIds.map { it -> deleteSourceWithStripe(client, it) } + val createdIds = listOf(getCardIdForTokenFromStripe(createTokenWithStripe(client)), + createSourceWithStripe(client)) + + val deletedIds = createdIds.map { it -> deleteSourceWithStripe(client, it) } - assert(createdIds.containsAll(deletedIds.toSet())) { - "Failed to delete one or more sources: ${createdIds.toSet() - deletedIds.toSet()}" + assert(createdIds.containsAll(deletedIds.toSet())) { + "Failed to delete one or more sources: ${createdIds.toSet() - deletedIds.toSet()}" + } + } finally { + StripePayment.deleteCustomer(email = email) } } From 6c2917ac56651ff83068f5bcde1049ff9b561bd5 Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Thu, 27 Sep 2018 16:51:38 +0200 Subject: [PATCH 68/85] Updates the 'new' version of the Swagger client API def. with the 'remove source' API --- prime/infra/new-dev/prime-client-api.yaml | 24 +++++++++++++++++++++- prime/infra/new-prod/prime-client-api.yaml | 24 +++++++++++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/prime/infra/new-dev/prime-client-api.yaml b/prime/infra/new-dev/prime-client-api.yaml index d192203a3..7af2407e1 100644 --- a/prime/infra/new-dev/prime-client-api.yaml +++ b/prime/infra/new-dev/prime-client-api.yaml @@ -151,6 +151,28 @@ paths: description: "User not found." security: - auth0_jwt: [] + delete: + description: "Remove a payment source for user" + produces: + - application/json + operationId: "removeSource" + parameters: + - name: sourceId + in: query + description: "The stripe-id of the source to be removed" + required: true + type: string + responses: + 200: + description: "Successfully removed the source" + schema: + $ref: '#/definitions/PaymentSource' + 400: + description: "The source could not be removed" + 404: + description: "No such source for user" + security: + - auth0_jwt: [] "/products": get: description: "Get all products for the user." @@ -573,4 +595,4 @@ securityDefinitions: type: "oauth2" x-google-issuer: "https://ostelco.eu.auth0.com/" x-google-jwks_uri: "https://ostelco.eu.auth0.com/.well-known/jwks.json" - x-google-audiences: "http://google_api" \ No newline at end of file + x-google-audiences: "http://google_api" diff --git a/prime/infra/new-prod/prime-client-api.yaml b/prime/infra/new-prod/prime-client-api.yaml index 7c705f9ae..09d1d2448 100644 --- a/prime/infra/new-prod/prime-client-api.yaml +++ b/prime/infra/new-prod/prime-client-api.yaml @@ -151,6 +151,28 @@ paths: description: "User not found." security: - auth0_jwt: [] + delete: + description: "Remove a payment source for user" + produces: + - application/json + operationId: "removeSource" + parameters: + - name: sourceId + in: query + description: "The stripe-id of the source to be removed" + required: true + type: string + responses: + 200: + description: "Successfully removed the source" + schema: + $ref: '#/definitions/PaymentSource' + 400: + description: "The source could not be removed" + 404: + description: "No such source for user" + security: + - auth0_jwt: [] "/products": get: description: "Get all products for the user." @@ -573,4 +595,4 @@ securityDefinitions: type: "oauth2" x-google-issuer: "https://ostelco.eu.auth0.com/" x-google-jwks_uri: "https://ostelco.eu.auth0.com/.well-known/jwks.json" - x-google-audiences: "http://google_api" \ No newline at end of file + x-google-audiences: "http://google_api" From c2dcb3941d58004fa8416e3fb5932683e242eb6a Mon Sep 17 00:00:00 2001 From: Martin Cederlof Date: Thu, 27 Sep 2018 19:18:34 +0200 Subject: [PATCH 69/85] Only update analytics on new insert to session map --- .../ocsgw/data/grpc/GrpcDataSource.java | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java index 4a471e05c..0a4ab1860 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java @@ -258,18 +258,12 @@ private void handleGrpcCcrAnswer(CreditControlAnswerInfo answer) { } private void addToSessionMap(CreditControlContext creditControlContext) { - switch (getRequestType(creditControlContext)) { - case INITIAL_REQUEST: - case UPDATE_REQUEST: - case TERMINATION_REQUEST: - sessionIdMap.put(creditControlContext.getCreditControlRequest().getMsisdn(), new SessionContext(creditControlContext.getSessionId(), creditControlContext.getCreditControlRequest().getOriginHost(), creditControlContext.getCreditControlRequest().getOriginRealm())); - updateAnalytics(); - break; - case EVENT_REQUEST: - break; - default: - LOG.warn("Unknown request type"); - break; + + SessionContext sessionContext = new SessionContext(creditControlContext.getSessionId(), + creditControlContext.getCreditControlRequest().getOriginHost(), + creditControlContext.getCreditControlRequest().getOriginRealm()); + if (sessionIdMap.put(creditControlContext.getCreditControlRequest().getMsisdn(), sessionContext) != null) { + updateAnalytics(); } } @@ -283,7 +277,6 @@ private void removeFromSessionMap(CreditControlContext creditControlContext) { private void updateAnalytics() { LOG.info("Number of active sessions is {}", sessionIdMap.size()); - OcsgwAnalyticsReport.Builder builder = OcsgwAnalyticsReport.newBuilder().setActiveSessions(sessionIdMap.size()); builder.setKeepAlive(false); sessionIdMap.forEach((msisdn, sessionContext) -> { From 17653584b729961f4272446d6031a7e72d1be7f4 Mon Sep 17 00:00:00 2001 From: Martin Cederlof Date: Thu, 27 Sep 2018 19:23:29 +0200 Subject: [PATCH 70/85] fix my logic returns null if not in map on insert --- .../main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java index ae3fe522d..0baa9566b 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java @@ -262,7 +262,7 @@ private void addToSessionMap(CreditControlContext creditControlContext) { SessionContext sessionContext = new SessionContext(creditControlContext.getSessionId(), creditControlContext.getCreditControlRequest().getOriginHost(), creditControlContext.getCreditControlRequest().getOriginRealm()); - if (sessionIdMap.put(creditControlContext.getCreditControlRequest().getMsisdn(), sessionContext) != null) { + if (sessionIdMap.put(creditControlContext.getCreditControlRequest().getMsisdn(), sessionContext) == null) { updateAnalytics(); } } From d40adbd47447e350fe840f305616dfccabd60fca Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Thu, 27 Sep 2018 21:30:54 +0200 Subject: [PATCH 71/85] Add docs --- prime/infra/README.md | 22 ++++++++++++++++++++++ prime/infra/raw_activeusers.ddl | 23 +++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 prime/infra/raw_activeusers.ddl diff --git a/prime/infra/README.md b/prime/infra/README.md index 9162e4760..5103ef877 100644 --- a/prime/infra/README.md +++ b/prime/infra/README.md @@ -342,3 +342,25 @@ logName="projects/pantel-2decb/logs/prime" ## Connect using Neo4j Browser Check [docs/NEO4J.md](../docs/NEO4J.md) + +## Deploy dataflow pipeline for raw_activeusers + +```bash +# For dev cluster +gcloud dataflow jobs run active-users-dev \ + --gcs-location gs://dataflow-templates/latest/PubSub_to_BigQuery \ + --region europe-west1 \ + --parameters \ +inputTopic=projects/pantel-2decb/topics/active-users-dev,\ +outputTableSpec=pantel-2decb:ocs_gateway_dev.raw_activeusers + + +# For production cluster +gcloud dataflow jobs run active-users \ + --gcs-location gs://dataflow-templates/latest/PubSub_to_BigQuery \ + --region europe-west1 \ + --parameters \ +inputTopic=projects/pantel-2decb/topics/active-users,\ +outputTableSpec=pantel-2decb:ocs_gateway.raw_activeusers + +``` \ No newline at end of file diff --git a/prime/infra/raw_activeusers.ddl b/prime/infra/raw_activeusers.ddl new file mode 100644 index 000000000..7279d883a --- /dev/null +++ b/prime/infra/raw_activeusers.ddl @@ -0,0 +1,23 @@ +# Table for dev cluster +CREATE TABLE ocs_gateway_dev.raw_activeusers + ( + timestamp TIMESTAMP NOT NULL, + users ARRAY< STRUCT< + msisdn STRING NOT NULL, + apn STRING NOT NULL, + mccMnc STRING NOT NULL + > > +) +PARTITION BY DATE(_PARTITIONTIME) + +# Table for production cluster +CREATE TABLE ocs_gateway.raw_activeusers + ( + timestamp TIMESTAMP NOT NULL, + users ARRAY< STRUCT< + msisdn STRING NOT NULL, + apn STRING NOT NULL, + mccMnc STRING NOT NULL + > > +) +PARTITION BY DATE(_PARTITIONTIME) \ No newline at end of file From 0dc8d437463d05e5b80ca1587576e2d9aa8c25f4 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Thu, 27 Sep 2018 22:12:56 +0200 Subject: [PATCH 72/85] Increased topup timeout to 100 msec --- .../kotlin/org/ostelco/prime/handler/PurchaseRequestHandler.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocs/src/main/kotlin/org/ostelco/prime/handler/PurchaseRequestHandler.kt b/ocs/src/main/kotlin/org/ostelco/prime/handler/PurchaseRequestHandler.kt index f4044b339..2ed28397e 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/handler/PurchaseRequestHandler.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/handler/PurchaseRequestHandler.kt @@ -80,7 +80,7 @@ class PurchaseRequestHandler( val future = CompletableFuture() requestMap[requestId] = future producer.topupDataBundleBalanceEvent(requestId = requestId, bundleId = bundleId, bytes = noOfBytes) - val error = future.get(5, MILLISECONDS) + val error = future.get(100, MILLISECONDS) if (error.isNotBlank()) { return Either.left(error) } From ce5692b3deb8157ce9c0e022aa49b87afd006520 Mon Sep 17 00:00:00 2001 From: Martin Cederlof Date: Fri, 28 Sep 2018 10:55:16 +0200 Subject: [PATCH 73/85] Fix user list pushed from ocsgw --- .../org/ostelco/diameter/model/Model.kt | 4 ++- .../ocsgw/data/grpc/GrpcDataSource.java | 33 ++++++++----------- .../org/ostelco/ocsgw/OcsApplicationTest.java | 4 ++- scripts/deploy-ocsgw.sh | 12 ++++--- 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/diameter-stack/src/main/kotlin/org/ostelco/diameter/model/Model.kt b/diameter-stack/src/main/kotlin/org/ostelco/diameter/model/Model.kt index 5469f6185..bd54c5a0b 100644 --- a/diameter-stack/src/main/kotlin/org/ostelco/diameter/model/Model.kt +++ b/diameter-stack/src/main/kotlin/org/ostelco/diameter/model/Model.kt @@ -210,4 +210,6 @@ enum class UserEquipmentInfoType { data class SessionContext( val sessionId: String, val originHost: String, - val originRealm: String) \ No newline at end of file + val originRealm: String, + val apn: String, + val mccMnc: String) \ No newline at end of file diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java index 0baa9566b..402f0c6fc 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java @@ -35,13 +35,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -258,12 +252,17 @@ private void handleGrpcCcrAnswer(CreditControlAnswerInfo answer) { } private void addToSessionMap(CreditControlContext creditControlContext) { - - SessionContext sessionContext = new SessionContext(creditControlContext.getSessionId(), - creditControlContext.getCreditControlRequest().getOriginHost(), - creditControlContext.getCreditControlRequest().getOriginRealm()); - if (sessionIdMap.put(creditControlContext.getCreditControlRequest().getMsisdn(), sessionContext) == null) { - updateAnalytics(); + try { + SessionContext sessionContext = new SessionContext(creditControlContext.getSessionId(), + creditControlContext.getCreditControlRequest().getOriginHost(), + creditControlContext.getCreditControlRequest().getOriginRealm(), + creditControlContext.getCreditControlRequest().getServiceInformation().get(0).getPsInformation().get(0).getCalledStationId(), + creditControlContext.getCreditControlRequest().getServiceInformation().get(0).getPsInformation().get(0).getSgsnMccMnc()); + if (sessionIdMap.put(creditControlContext.getCreditControlRequest().getMsisdn(), sessionContext) == null) { + updateAnalytics(); + } + } catch (Exception e) { + LOG.error("Failed to update session map"); } } @@ -280,13 +279,7 @@ private void updateAnalytics() { OcsgwAnalyticsReport.Builder builder = OcsgwAnalyticsReport.newBuilder().setActiveSessions(sessionIdMap.size()); builder.setKeepAlive(false); sessionIdMap.forEach((msisdn, sessionContext) -> { - try { - String apn = ccrMap.get(sessionContext.getSessionId()).getCreditControlRequest().getServiceInformation().get(0).getPsInformation().get(0).getCalledStationId(); - String mccMnc = ccrMap.get(sessionContext.getSessionId()).getCreditControlRequest().getServiceInformation().get(0).getPsInformation().get(0).getSgsnMccMnc(); - builder.addUsers(User.newBuilder().setApn(apn).setMccMnc(mccMnc).setMsisdn(msisdn).build()); - } catch (Exception e) { - LOG.error("Failed to match session info to ccr map", e); - } + builder.addUsers(User.newBuilder().setApn(sessionContext.getApn()).setMccMnc(sessionContext.getMccMnc()).setMsisdn(msisdn).build()); }); ocsgwAnalytics.sendAnalytics(builder.build()); } diff --git a/ocsgw/src/test/java/org/ostelco/ocsgw/OcsApplicationTest.java b/ocsgw/src/test/java/org/ostelco/ocsgw/OcsApplicationTest.java index 43362f09d..e4918dd8f 100644 --- a/ocsgw/src/test/java/org/ostelco/ocsgw/OcsApplicationTest.java +++ b/ocsgw/src/test/java/org/ostelco/ocsgw/OcsApplicationTest.java @@ -41,6 +41,8 @@ public class OcsApplicationTest { private static final String OCS_HOST = "ocs"; private static final String PGW_HOST = "testclient"; private static final String PGW_REALM = "loltel"; + private static final String APN = "loltel-test"; + private static final String MCC_MNC = "24201"; private static final String MSISDN = "4790300123"; @@ -162,7 +164,7 @@ public void testReAuthRequest() { client.initRequestTest(); - OcsServer.getInstance().sendReAuthRequest(new SessionContext(session.getSessionId(), PGW_HOST, PGW_REALM)); + OcsServer.getInstance().sendReAuthRequest(new SessionContext(session.getSessionId(), PGW_HOST, PGW_REALM, APN, MCC_MNC)); waitForRequest(); try { AvpSet resultAvps = client.getResultAvps(); diff --git a/scripts/deploy-ocsgw.sh b/scripts/deploy-ocsgw.sh index 30169a00c..31f65f514 100755 --- a/scripts/deploy-ocsgw.sh +++ b/scripts/deploy-ocsgw.sh @@ -14,16 +14,20 @@ if [ "$1" = prod ] ; then variant=prod fi -echo "Starting to deploy OCSGW to $variant" -echo "The last thing this script will do is to look at logs from the ocsgw" -echo "It will continue to do so until terminated by ^C" +echo "Starting update.." +echo "Creating zip files" +gradle pack if [ ! -f build/deploy/ostelco-core-${variant}.zip ]; then echo "build/deploy/ostelco-core-${variant}.zip not found!" - echo "Did you forget gradle pack ?" exit 1 fi +echo "Starting to deploy OCSGW to $variant" +echo "The last thing this script will do is to look at logs from the ocsgw" +echo "It will continue to do so until terminated by ^C" + + scp -oProxyJump=loltel@10.6.101.1 build/deploy/ostelco-core-${variant}.zip ubuntu@${host_ip}:. ssh -A -Jloltel@10.6.101.1 ubuntu@${host_ip} < Date: Fri, 28 Sep 2018 11:00:02 +0200 Subject: [PATCH 74/85] Log error --- .../main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java index 402f0c6fc..a320da85c 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java @@ -262,7 +262,7 @@ private void addToSessionMap(CreditControlContext creditControlContext) { updateAnalytics(); } } catch (Exception e) { - LOG.error("Failed to update session map"); + LOG.error("Failed to update session map", e); } } From 318dec58dbfdbf643cf6c273e29b89a367cb569a Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Thu, 27 Sep 2018 15:17:35 +0200 Subject: [PATCH 75/85] Updated init script --- .../org/ostelco/tools/migration/MainKt.kt | 6 +- .../src/main/resources/init.cypher | 79 +++++++++++++------ 2 files changed, 56 insertions(+), 29 deletions(-) diff --git a/tools/neo4j-admin-tools/src/main/kotlin/org/ostelco/tools/migration/MainKt.kt b/tools/neo4j-admin-tools/src/main/kotlin/org/ostelco/tools/migration/MainKt.kt index bb57ed0bd..d1a32f104 100644 --- a/tools/neo4j-admin-tools/src/main/kotlin/org/ostelco/tools/migration/MainKt.kt +++ b/tools/neo4j-admin-tools/src/main/kotlin/org/ostelco/tools/migration/MainKt.kt @@ -5,8 +5,8 @@ import java.nio.file.Files import java.nio.file.Paths fun main(args: Array) { - neo4jExporterToCypherFile() - // cypherFileToNeo4jImporter() + // neo4jExporterToCypherFile() + cypherFileToNeo4jImporter() } fun neo4jExporterToCypherFile() { @@ -40,7 +40,7 @@ fun cypherFileToNeo4jImporter() { println("Import from file to Neo4j") - importFromCypherFile("src/main/resources/backup.prod.cypher") { query -> + importFromCypherFile("src/main/resources/init.cypher") { query -> txn.run(query) } diff --git a/tools/neo4j-admin-tools/src/main/resources/init.cypher b/tools/neo4j-admin-tools/src/main/resources/init.cypher index d6bbbf15d..d66abcab9 100644 --- a/tools/neo4j-admin-tools/src/main/resources/init.cypher +++ b/tools/neo4j-admin-tools/src/main/resources/init.cypher @@ -1,4 +1,4 @@ -// Create product +// For country:NO CREATE (:Product {`id`: '1GB_249NOK', `presentation/isDefault`: 'true', `presentation/offerLabel`: 'Default Offer', @@ -36,47 +36,74 @@ CREATE (:Product {`id`: '5GB_399NOK', `properties/noOfBytes`: '5_000_000_000', `sku`: '5GB_399NOK'}); -CREATE (:Product {`id`: '100MB_FREE_ON_JOINING', - `presentation/priceLabel`: 'Free', - `presentation/productLabel`: '100MB Welcome Pack', - `price/amount`: '0', - `price/currency`: 'NOK', - `properties/noOfBytes`: '100_000_000', - `sku`: '100MB_FREE_ON_JOINING'}); - -CREATE (:Product {`id`: '1GB_FREE_ON_REFERRED', - `presentation/priceLabel`: 'Free', - `presentation/productLabel`: '1GB Referral Pack', - `price/amount`: '0', - `price/currency`: 'NOK', - `properties/noOfBytes`: '1_000_000_000', - `sku`: '1GB_FREE_ON_REFERRED'}); - -CREATE (:Segment {`id`: 'all'}); +CREATE (:Segment {`id`: 'country-no'}); -CREATE (:Offer {`id`: 'default_offer'}); +CREATE (:Offer {`id`: 'default_offer-no'}); -MATCH (n:Offer {id: 'default_offer'}) +MATCH (n:Offer {id: 'default_offer-no'}) WITH n MATCH (m:Product {id: '1GB_249NOK'}) CREATE (n)-[:OFFER_HAS_PRODUCT]->(m); -MATCH (n:Offer {id: 'default_offer'}) +MATCH (n:Offer {id: 'default_offer-no'}) WITH n MATCH (m:Product {id: '2GB_299NOK'}) CREATE (n)-[:OFFER_HAS_PRODUCT]->(m); -MATCH (n:Offer {id: 'default_offer'}) +MATCH (n:Offer {id: 'default_offer-no'}) WITH n MATCH (m:Product {id: '3GB_349NOK'}) CREATE (n)-[:OFFER_HAS_PRODUCT]->(m); -MATCH (n:Offer {id: 'default_offer'}) +MATCH (n:Offer {id: 'default_offer-no'}) WITH n MATCH (m:Product {id: '5GB_399NOK'}) CREATE (n)-[:OFFER_HAS_PRODUCT]->(m); -MATCH (n:Offer {id: 'default_offer'}) +MATCH (n:Offer {id: 'default_offer-no'}) WITH n -MATCH (m:Segment {id: 'all'}) -CREATE (n)-[:OFFERED_TO_SEGMENT]->(m); \ No newline at end of file +MATCH (m:Segment {id: 'country-no'}) +CREATE (n)-[:OFFERED_TO_SEGMENT]->(m); + +// For country:SG +CREATE (:Product {`id`: '1GB_1SGD', + `presentation/isDefault`: 'true', + `presentation/offerLabel`: 'Default Offer', + `presentation/priceLabel`: '1 SGD', + `presentation/productLabel`: '+1GB', + `price/amount`: '100', + `price/currency`: 'SGD', + `properties/noOfBytes`: '1_000_000_000', + `sku`: '1GB_1SGD'}); + +CREATE (:Segment {`id`: 'country-sg'}); + +CREATE (:Offer {`id`: 'default_offer-sg'}); + +MATCH (n:Offer {id: 'default_offer-sg'}) +WITH n +MATCH (m:Product {id: '1GB_1SGD'}) +CREATE (n)-[:OFFER_HAS_PRODUCT]->(m); + +MATCH (n:Offer {id: 'default_offer-sg'}) +WITH n +MATCH (m:Segment {id: 'country-sg'}) +CREATE (n)-[:OFFERED_TO_SEGMENT]->(m); + +// Generic +CREATE (:Product {`id`: '100MB_FREE_ON_JOINING', + `presentation/priceLabel`: 'Free', + `presentation/productLabel`: '100MB Welcome Pack', + `price/amount`: '0', + `price/currency`: '', + `properties/noOfBytes`: '100_000_000', + `sku`: '100MB_FREE_ON_JOINING'}); + +CREATE (:Product {`id`: '1GB_FREE_ON_REFERRED', + `presentation/priceLabel`: 'Free', + `presentation/productLabel`: '1GB Referral Pack', + `price/amount`: '0', + `price/currency`: '', + `properties/noOfBytes`: '1_000_000_000', + `sku`: '1GB_FREE_ON_REFERRED'}); + From ad978bd33b5315971485a33f6b74b69b9a6bbb6e Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Thu, 27 Sep 2018 15:18:10 +0200 Subject: [PATCH 76/85] Added "Failed to Create Profile" error code --- .../org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt | 6 +++--- .../main/kotlin/org/ostelco/prime/apierror/ApiErrorCodes.kt | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt index 801a84084..e35ac79a1 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt @@ -65,12 +65,12 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu override fun createProfile(subscriberId: String, profile: Subscriber, referredBy: String?): Either { if (!SubscriberDAO.isValidProfile(profile)) { logger.error("Failed to create profile. Invalid profile.") - return Either.left(BadRequestError("Incomplete profile description. Profile must contain name and email", ApiErrorCode.FAILED_TO_CREATE_PAYMENT_PROFILE)) + return Either.left(BadRequestError("Incomplete profile description. Profile must contain name and email", ApiErrorCode.FAILED_TO_CREATE_PROFILE)) } return try { storage.addSubscriber(profile, referredBy) .mapLeft { - mapStorageErrorToApiError("Failed to create profile.", ApiErrorCode.FAILED_TO_CREATE_PAYMENT_PROFILE, it) + mapStorageErrorToApiError("Failed to create profile.", ApiErrorCode.FAILED_TO_CREATE_PROFILE, it) } .flatMap { updateMetricsOnNewSubscriber() @@ -78,7 +78,7 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu } } catch (e: Exception) { logger.error("Failed to create profile for subscriberId $subscriberId", e) - Either.left(BadGatewayError("Failed to create profile", ApiErrorCode.FAILED_TO_CREATE_PAYMENT_PROFILE)) + Either.left(BadGatewayError("Failed to create profile", ApiErrorCode.FAILED_TO_CREATE_PROFILE)) } } diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiErrorCodes.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiErrorCodes.kt index 4b0950a03..7ad7194f7 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiErrorCodes.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiErrorCodes.kt @@ -18,6 +18,7 @@ enum class ApiErrorCode { FAILED_TO_STORE_PAYMENT_SOURCE, FAILED_TO_SET_DEFAULT_PAYMENT_SOURCE, FAILED_TO_FETCH_PAYMENT_SOURCES_LIST, + FAILED_TO_CREATE_PROFILE, FAILED_TO_UPDATE_PROFILE, FAILED_TO_FETCH_CONSENT, FAILED_TO_IMPORT_OFFER, From 53ac7e822b69c43947752b4eb33b2db26cbec126 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Fri, 28 Sep 2018 13:03:28 +0200 Subject: [PATCH 77/85] Minor change in OCS design doc --- ocs/design.puml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocs/design.puml b/ocs/design.puml index 4ee79d418..501724acb 100644 --- a/ocs/design.puml +++ b/ocs/design.puml @@ -6,7 +6,7 @@ [ocsgw] -interface OcsGrpcService +[OcsGrpcService] [OcsGrpcServer] From 911dea2d790a52549d3586987672706665d3fdaf Mon Sep 17 00:00:00 2001 From: Martin Cederlof Date: Fri, 28 Sep 2018 15:22:33 +0200 Subject: [PATCH 78/85] Make difference between roaming and local better in labels --- bq-metrics-extractor/config/config.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/bq-metrics-extractor/config/config.yaml b/bq-metrics-extractor/config/config.yaml index 80f389f08..4434c9273 100644 --- a/bq-metrics-extractor/config/config.yaml +++ b/bq-metrics-extractor/config/config.yaml @@ -93,8 +93,8 @@ bqmetrics: AND timestamp < TIMESTAMP_TRUNC(CURRENT_TIMESTAMP(), DAY) ), 0) as count - type: gauge - name: total_data_used_today_loltel_test - help: Total data used today loltel-test + name: total_data_used_today_local_loltel_test + help: Total data used today local loltel-test resultColumn: count sql: > SELECT COALESCE ( @@ -103,8 +103,8 @@ bqmetrics: AND apn = "loltel-test" AND mccMnc = "24201"), 0) as count - type: gauge - name: total_data_used_yesterday_lotlel_test - help: Total data used yesterday loltel-test + name: total_data_used_yesterday_local_lotlel_test + help: Total data used yesterday local loltel-test resultColumn: count sql: > SELECT COALESCE ( @@ -116,8 +116,8 @@ bqmetrics: - type: gauge - name: total_roaming_data_used_today_loltel_test - help: Total roaming data used today loltel-test + name: total_data_used_today_roaming_loltel_test + help: Total data used today roaming loltel-test resultColumn: count sql: > SELECT COALESCE ( @@ -126,8 +126,8 @@ bqmetrics: AND apn = "loltel-test" AND mccMnc != "24201"), 0) as count - type: gauge - name: total_roaming_data_used_yesterday_lotlel_test - help: Total roaming data used yesterday loltel-test + name: total_data_used_yesterday_roaming_lotlel_test + help: Total data used yesterday roaming loltel-test resultColumn: count sql: > SELECT COALESCE ( From 1969b12d2a2bb0d5735851a34e91838cb6255ba8 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Fri, 28 Sep 2018 21:04:47 +0200 Subject: [PATCH 79/85] For list source and remove source, skipped creating payment profile if it is absent. Other minor refactoring. --- .../client/api/store/SubscriberDAOImpl.kt | 16 +---- .../StripePaymentProcessor.kt | 63 ++++++++++++------- 2 files changed, 43 insertions(+), 36 deletions(-) diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt index 9f0a83650..a57c167b4 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt @@ -338,13 +338,7 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu override fun listSources(subscriberId: String): Either> { return paymentProcessor.getPaymentProfile(subscriberId) - .fold( - { - paymentProcessor.createPaymentProfile(subscriberId) - .mapLeft { error -> mapPaymentErrorToApiError(error.description, ApiErrorCode.FAILED_TO_FETCH_PAYMENT_SOURCES_LIST, error) } - }, - { profileInfo -> Either.right(profileInfo) } - ) + .mapLeft { error -> mapPaymentErrorToApiError(error.description, ApiErrorCode.FAILED_TO_FETCH_PAYMENT_SOURCES_LIST, error) } .flatMap { profileInfo -> paymentProcessor.getSavedSources(profileInfo.id) .mapLeft { mapPaymentErrorToApiError("Failed to list sources", ApiErrorCode.FAILED_TO_FETCH_PAYMENT_SOURCES_LIST, it) } @@ -353,13 +347,7 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu override fun removeSource(subscriberId: String, sourceId: String): Either { return paymentProcessor.getPaymentProfile(subscriberId) - .fold( - { - paymentProcessor.createPaymentProfile(subscriberId) - .mapLeft { error -> mapPaymentErrorToApiError(error.description, ApiErrorCode.FAILED_TO_SET_DEFAULT_PAYMENT_SOURCE, error) } - }, - { profileInfo -> Either.right(profileInfo) } - ) + .mapLeft { error -> mapPaymentErrorToApiError(error.description, ApiErrorCode.FAILED_TO_REMOVE_PAYMENT_SOURCE, error) } .flatMap { profileInfo -> paymentProcessor.removeSource(profileInfo.id, sourceId) .mapLeft { mapPaymentErrorToApiError("Failed to remove payment source", ApiErrorCode.FAILED_TO_REMOVE_PAYMENT_SOURCE, it) } diff --git a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt index eb5e329be..af56dbe54 100644 --- a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt +++ b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt @@ -2,11 +2,32 @@ package org.ostelco.prime.paymentprocessor import arrow.core.Either import arrow.core.flatMap -import com.stripe.exception.* -import org.ostelco.prime.getLogger -import com.stripe.model.* -import org.ostelco.prime.paymentprocessor.core.* +import com.stripe.exception.ApiConnectionException +import com.stripe.exception.AuthenticationException +import com.stripe.exception.CardException +import com.stripe.exception.InvalidRequestException +import com.stripe.exception.RateLimitException +import com.stripe.exception.StripeException +import com.stripe.model.Card +import com.stripe.model.Charge import com.stripe.model.Customer +import com.stripe.model.ExternalAccount +import com.stripe.model.Plan +import com.stripe.model.Product +import com.stripe.model.Refund +import com.stripe.model.Source +import com.stripe.model.Subscription +import org.ostelco.prime.getLogger +import org.ostelco.prime.paymentprocessor.core.BadGatewayError +import org.ostelco.prime.paymentprocessor.core.ForbiddenError +import org.ostelco.prime.paymentprocessor.core.NotFoundError +import org.ostelco.prime.paymentprocessor.core.PaymentError +import org.ostelco.prime.paymentprocessor.core.PlanInfo +import org.ostelco.prime.paymentprocessor.core.ProductInfo +import org.ostelco.prime.paymentprocessor.core.ProfileInfo +import org.ostelco.prime.paymentprocessor.core.SourceDetailsInfo +import org.ostelco.prime.paymentprocessor.core.SourceInfo +import org.ostelco.prime.paymentprocessor.core.SubscriptionInfo class StripePaymentProcessor : PaymentProcessor { @@ -20,15 +41,15 @@ class StripePaymentProcessor : PaymentProcessor { val details = getAccountDetails(it) SourceDetailsInfo(it.id, getAccountType(details), details) } - sources.sortedByDescending { it.details.get("created") as Long } + sources.sortedByDescending { it.details["created"] as Long } } private fun getAccountType(details: Map) : String { - return details.get("type").toString() + return details["type"].toString() } /* Returns detailed 'account details' for the given Stripe source/account. - Note that including the fields 'id', 'type' and 'created' are manadatory. */ + Note that including the fields 'id', 'type' and 'created' are mandatory. */ private fun getAccountDetails(accountInfo: ExternalAccount) : Map { when (accountInfo) { is Card -> { @@ -75,7 +96,7 @@ class StripePaymentProcessor : PaymentProcessor { returns stored metadata values as strings, even if they where stored using an another type. Needs to be verified.) */ private fun getCreatedTimestampFromMetadata(id: String, metadata: Map) : Long { - val created: String? = metadata.get("created") as? String + val created: String? = metadata["created"] as? String return created?.toLongOrNull() ?: run { logger.warn("No 'created' timestamp found in metadata for Stripe account {}", id) @@ -99,13 +120,11 @@ class StripePaymentProcessor : PaymentProcessor { "limit" to "1", "email" to userEmail) val customerList = Customer.list(customerParams) - if (customerList.data.isEmpty()) { - return Either.left(NotFoundError("Could not find a payment profile for user $userEmail")) - } else if (customerList.data.size > 1){ - return Either.left(NotFoundError("Multiple profiles for user $userEmail found")) - } else { - return Either.right(ProfileInfo(customerList.data.first().id)) - } + return when { + customerList.data.isEmpty() -> Either.left(NotFoundError("Could not find a payment profile for user $userEmail")) + customerList.data.size > 1 -> Either.left(NotFoundError("Multiple profiles for user $userEmail found")) + else -> Either.right(ProfileInfo(customerList.data.first().id)) + } } override fun createPlan(productId: String, amount: Int, currency: String, interval: PaymentProcessor.Interval): Either = @@ -240,7 +259,7 @@ class StripePaymentProcessor : PaymentProcessor { is Card -> accountInfo.delete() is Source -> accountInfo.detach() else -> - Either.left(BadGatewayError("Attempt to remove unsupported account-type ${accountInfo}")) + Either.left(BadGatewayError("Attempt to remove unsupported account-type $accountInfo")) } SourceInfo(sourceId) } @@ -250,28 +269,28 @@ class StripePaymentProcessor : PaymentProcessor { Either.right(action()) } catch (e: CardException) { // If something is decline with a card purchase, CardException will be caught - logger.warn("Payment error : $errorDescription , Stripe Error Code: ${e.getCode()}", e) + logger.warn("Payment error : $errorDescription , Stripe Error Code: ${e.code}", e) Either.left(ForbiddenError(errorDescription, e.message)) } catch (e: RateLimitException) { // Too many requests made to the API too quickly - logger.warn("Payment error : $errorDescription , Stripe Error Code: ${e.getCode()}", e) + logger.warn("Payment error : $errorDescription , Stripe Error Code: ${e.code}", e) Either.left(BadGatewayError(errorDescription, e.message)) } catch (e: InvalidRequestException) { // Invalid parameters were supplied to Stripe's API - logger.warn("Payment error : $errorDescription , Stripe Error Code: ${e.getCode()}", e) + logger.warn("Payment error : $errorDescription , Stripe Error Code: ${e.code}", e) Either.left(NotFoundError(errorDescription, e.message)) } catch (e: AuthenticationException) { // Authentication with Stripe's API failed // (maybe you changed API keys recently) - logger.warn("Payment error : $errorDescription , Stripe Error Code: ${e.getCode()}", e) + logger.warn("Payment error : $errorDescription , Stripe Error Code: ${e.code}", e) Either.left(BadGatewayError(errorDescription)) } catch (e: ApiConnectionException) { // Network communication with Stripe failed - logger.warn("Payment error : $errorDescription , Stripe Error Code: ${e.getCode()}", e) + logger.warn("Payment error : $errorDescription , Stripe Error Code: ${e.code}", e) Either.left(BadGatewayError(errorDescription)) } catch (e: StripeException) { // Unknown Stripe error - logger.error("Payment error : $errorDescription , Stripe Error Code: ${e.getCode()}", e) + logger.error("Payment error : $errorDescription , Stripe Error Code: ${e.code}", e) Either.left(BadGatewayError(errorDescription)) } catch (e: Exception) { // Something else happened, could be completely unrelated to Stripe From 811f66f7478fda263ff602d9d08cd18fd2fe4153 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Sun, 12 Aug 2018 18:38:08 +0200 Subject: [PATCH 80/85] Updated to Java 11 --- .circleci/config.yml | 2 +- acceptance-tests/Dockerfile | 2 +- analytics-grpc-api/build.gradle | 1 + auth-server/Dockerfile | 2 +- auth-server/build.gradle | 3 +++ bq-metrics-extractor/Dockerfile | 2 +- bq-metrics-extractor/Dockerfile.test | 2 +- build.gradle | 8 ++++++-- client-api/build.gradle | 3 +++ dataflow-pipelines/Dockerfile | 2 +- ext-auth-provider/Dockerfile | 2 +- ext-auth-provider/build.gradle | 3 +++ ocs-grpc-api/build.gradle | 1 + ocsgw/Dockerfile | 2 +- ocsgw/build.gradle | 3 +++ prime-client-api/build.gradle | 2 ++ prime/Dockerfile | 2 +- prime/Dockerfile.test | 2 +- prime/cloudbuild.dev.yaml | 2 +- prime/cloudbuild.yaml | 2 +- 20 files changed, 34 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f155d7d1e..9e2e0dc11 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -78,7 +78,7 @@ jobs: CODACY_MODULE: com.codacy.CodacyCoverageReporter docker: - - image: circleci/openjdk:8u171-jdk + - image: circleci/openjdk:11-jdk-sid steps: - run: diff --git a/acceptance-tests/Dockerfile b/acceptance-tests/Dockerfile index 35e8792f4..d695fff0f 100644 --- a/acceptance-tests/Dockerfile +++ b/acceptance-tests/Dockerfile @@ -1,4 +1,4 @@ -FROM azul/zulu-openjdk:8u181-8.31.0.1 +FROM openjdk:11 MAINTAINER CSI "csi@telenordigital.com" diff --git a/analytics-grpc-api/build.gradle b/analytics-grpc-api/build.gradle index 1a37a301e..fb2ba62f8 100644 --- a/analytics-grpc-api/build.gradle +++ b/analytics-grpc-api/build.gradle @@ -9,6 +9,7 @@ dependencies { api "io.grpc:grpc-protobuf:$grpcVersion" api "io.grpc:grpc-stub:$grpcVersion" api "io.grpc:grpc-core:$grpcVersion" + implementation 'javax.annotation:javax.annotation-api:1.3.2' } protobuf { diff --git a/auth-server/Dockerfile b/auth-server/Dockerfile index 50a1964b9..99c77878a 100644 --- a/auth-server/Dockerfile +++ b/auth-server/Dockerfile @@ -1,4 +1,4 @@ -FROM azul/zulu-openjdk:8u181-8.31.0.1 +FROM openjdk:11 MAINTAINER CSI "csi@telenordigital.com" diff --git a/auth-server/build.gradle b/auth-server/build.gradle index a1c13fb54..648091699 100644 --- a/auth-server/build.gradle +++ b/auth-server/build.gradle @@ -9,6 +9,9 @@ dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion" implementation "io.dropwizard:dropwizard-core:$dropwizardVersion" implementation project(":firebase-extensions") + + implementation 'javax.xml.bind:jaxb-api:2.3.0' + implementation 'javax.activation:activation:1.1.1' testImplementation "io.dropwizard:dropwizard-testing:$dropwizardVersion" testImplementation "org.jetbrains.kotlin:kotlin-test-junit:$kotlinVersion" diff --git a/bq-metrics-extractor/Dockerfile b/bq-metrics-extractor/Dockerfile index ab899b783..ed8144f51 100644 --- a/bq-metrics-extractor/Dockerfile +++ b/bq-metrics-extractor/Dockerfile @@ -1,4 +1,4 @@ -FROM azul/zulu-openjdk:8u181-8.31.0.1 +FROM openjdk:11 MAINTAINER CSI "csi@telenordigital.com" diff --git a/bq-metrics-extractor/Dockerfile.test b/bq-metrics-extractor/Dockerfile.test index a9e7587fe..fda30d0b5 100644 --- a/bq-metrics-extractor/Dockerfile.test +++ b/bq-metrics-extractor/Dockerfile.test @@ -1,4 +1,4 @@ -FROM azul/zulu-openjdk:8u181-8.31.0.1 +FROM openjdk:11 MAINTAINER CSI "csi@telenordigital.com" diff --git a/build.gradle b/build.gradle index e319235f3..714bc2027 100644 --- a/build.gradle +++ b/build.gradle @@ -19,11 +19,15 @@ allprojects { maven { url = "https://maven.repository.redhat.com/ga/" } maven { url = "http://clojars.org/repo/" } } + + jacoco { + toolVersion = "0.8.2" + } } subprojects { - sourceCompatibility = 1.8 - targetCompatibility = 1.8 + sourceCompatibility = 11 + targetCompatibility = 11 tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } diff --git a/client-api/build.gradle b/client-api/build.gradle index f0e158a68..2dc95425d 100644 --- a/client-api/build.gradle +++ b/client-api/build.gradle @@ -14,6 +14,9 @@ dependencies { implementation "com.google.guava:guava:$guavaVersion" implementation 'io.jsonwebtoken:jjwt:0.9.1' + implementation 'javax.xml.bind:jaxb-api:2.3.0' + implementation 'javax.activation:activation:1.1.1' + testImplementation "io.dropwizard:dropwizard-client:$dropwizardVersion" testImplementation "io.dropwizard:dropwizard-testing:$dropwizardVersion" testImplementation "com.fasterxml.jackson.module:jackson-module-kotlin:$jacksonVersion" diff --git a/dataflow-pipelines/Dockerfile b/dataflow-pipelines/Dockerfile index 4827fd238..d0cd99b27 100644 --- a/dataflow-pipelines/Dockerfile +++ b/dataflow-pipelines/Dockerfile @@ -1,4 +1,4 @@ -FROM azul/zulu-openjdk:8u181-8.31.0.1 +FROM openjdk:11 MAINTAINER CSI "csi@telenordigital.com" diff --git a/ext-auth-provider/Dockerfile b/ext-auth-provider/Dockerfile index 78e784b6d..a69b2b1a3 100644 --- a/ext-auth-provider/Dockerfile +++ b/ext-auth-provider/Dockerfile @@ -1,4 +1,4 @@ -FROM azul/zulu-openjdk:8u181-8.31.0.1 +FROM openjdk:11 MAINTAINER CSI "csi@telenordigital.com" diff --git a/ext-auth-provider/build.gradle b/ext-auth-provider/build.gradle index 225fb0949..69139c60d 100644 --- a/ext-auth-provider/build.gradle +++ b/ext-auth-provider/build.gradle @@ -10,6 +10,9 @@ dependencies { implementation "io.dropwizard:dropwizard-core:$dropwizardVersion" implementation 'io.jsonwebtoken:jjwt:0.9.1' + implementation 'javax.xml.bind:jaxb-api:2.3.0' + implementation 'javax.activation:activation:1.1.1' + testImplementation "io.dropwizard:dropwizard-testing:$dropwizardVersion" testImplementation "org.jetbrains.kotlin:kotlin-test:$kotlinVersion" testImplementation "org.jetbrains.kotlin:kotlin-test-junit:$kotlinVersion" diff --git a/ocs-grpc-api/build.gradle b/ocs-grpc-api/build.gradle index 1a37a301e..fb2ba62f8 100644 --- a/ocs-grpc-api/build.gradle +++ b/ocs-grpc-api/build.gradle @@ -9,6 +9,7 @@ dependencies { api "io.grpc:grpc-protobuf:$grpcVersion" api "io.grpc:grpc-stub:$grpcVersion" api "io.grpc:grpc-core:$grpcVersion" + implementation 'javax.annotation:javax.annotation-api:1.3.2' } protobuf { diff --git a/ocsgw/Dockerfile b/ocsgw/Dockerfile index 2e2ffd740..b6b5264ee 100644 --- a/ocsgw/Dockerfile +++ b/ocsgw/Dockerfile @@ -1,4 +1,4 @@ -FROM azul/zulu-openjdk:8u181-8.31.0.1 +FROM openjdk:11 MAINTAINER CSI "csi@telenordigital.com" diff --git a/ocsgw/build.gradle b/ocsgw/build.gradle index 75bd6fd43..ef85ce838 100644 --- a/ocsgw/build.gradle +++ b/ocsgw/build.gradle @@ -12,6 +12,9 @@ dependencies { implementation project(':diameter-stack') implementation "com.google.cloud:google-cloud-core-grpc:$googleCloudVersion" + implementation 'javax.xml.bind:jaxb-api:2.3.0' + implementation 'javax.activation:activation:1.1.1' + implementation 'ch.qos.logback:logback-classic:1.2.3' // log to gcp stack-driver diff --git a/prime-client-api/build.gradle b/prime-client-api/build.gradle index ba58c49f1..e711a3145 100644 --- a/prime-client-api/build.gradle +++ b/prime-client-api/build.gradle @@ -41,6 +41,8 @@ dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion" swaggerCodegen 'io.swagger:swagger-codegen-cli:2.3.1' + implementation 'javax.annotation:javax.annotation-api:1.3.2' + // taken from build/swagger-code-java-client/build.gradle implementation 'io.swagger:swagger-annotations:1.5.21' implementation 'com.google.code.gson:gson:2.8.5' diff --git a/prime/Dockerfile b/prime/Dockerfile index a7903c099..c89b95728 100644 --- a/prime/Dockerfile +++ b/prime/Dockerfile @@ -1,4 +1,4 @@ -FROM azul/zulu-openjdk:8u181-8.31.0.1 +FROM openjdk:11 MAINTAINER CSI "csi@telenordigital.com" diff --git a/prime/Dockerfile.test b/prime/Dockerfile.test index 0c542a2b4..7e088682f 100644 --- a/prime/Dockerfile.test +++ b/prime/Dockerfile.test @@ -1,6 +1,6 @@ # This Dockerfile is used when running locally using docker-compose for Acceptance Testing. -FROM azul/zulu-openjdk:8u181-8.31.0.1 +FROM openjdk:11 MAINTAINER CSI "csi@telenordigital.com" diff --git a/prime/cloudbuild.dev.yaml b/prime/cloudbuild.dev.yaml index 88c3dceca..3394a66cf 100644 --- a/prime/cloudbuild.dev.yaml +++ b/prime/cloudbuild.dev.yaml @@ -50,7 +50,7 @@ steps: path: /root/out_zip # Build docker images - name: gcr.io/cloud-builders/docker - args: ['build', '--tag=eu.gcr.io/$PROJECT_ID/prime:$SHORT_SHA', '--cache-from', 'azul/zulu-openjdk:8u181-8.31.0.1', 'prime'] + args: ['build', '--tag=eu.gcr.io/$PROJECT_ID/prime:$SHORT_SHA', '--cache-from', 'openjdk:11', 'prime'] timeout: 120s # Deploy new docker image to Google Kubernetes Engine (GKE) - name: ubuntu diff --git a/prime/cloudbuild.yaml b/prime/cloudbuild.yaml index d4cba67b0..ff9d63966 100644 --- a/prime/cloudbuild.yaml +++ b/prime/cloudbuild.yaml @@ -50,7 +50,7 @@ steps: path: /root/out_zip # Build docker images - name: gcr.io/cloud-builders/docker - args: ['build', '--tag=eu.gcr.io/$PROJECT_ID/prime:$TAG_NAME', '--cache-from', 'azul/zulu-openjdk:8u181-8.31.0.1', 'prime'] + args: ['build', '--tag=eu.gcr.io/$PROJECT_ID/prime:$TAG_NAME', '--cache-from', 'openjdk:11', 'prime'] timeout: 120s # Deploy new docker image to Google Kubernetes Engine (GKE) - name: ubuntu From 31dc071222a05cc6c1f0361b0cd61c245703cb9f Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Sat, 29 Sep 2018 13:26:53 +0200 Subject: [PATCH 81/85] Updated dependencies --- README.md | 2 +- acceptance-tests/build.gradle | 4 ++-- admin-api/build.gradle | 2 +- analytics-module/build.gradle | 2 +- app-notifier/build.gradle | 2 +- auth-server/build.gradle | 4 ++-- bq-metrics-extractor/build.gradle | 4 ++-- build.gradle | 7 ++++--- client-api/build.gradle | 2 +- dataflow-pipelines/build.gradle | 8 ++++---- diameter-stack/build.gradle | 2 +- diameter-test/build.gradle | 2 +- ext-auth-provider/build.gradle | 4 ++-- firebase-extensions/build.gradle | 2 +- firebase-store/build.gradle | 2 +- model/build.gradle | 2 +- neo4j-store/build.gradle | 2 +- ocs/build.gradle | 4 ++-- ocsgw/build.gradle | 17 +++++++++-------- payment-processor/build.gradle | 2 +- prime-client-api/build.gradle | 2 +- prime-modules/build.gradle | 2 +- prime/build.gradle | 6 +++--- pseudonym-server/build.gradle | 2 +- tools/neo4j-admin-tools/build.gradle | 4 ++-- 25 files changed, 47 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 8f5acb09d..5ba31bb3c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -[![Kotlin version badge](https://img.shields.io/badge/kotlin-1.2.70-blue.svg)](http://kotlinlang.org/) +[![Kotlin version badge](https://img.shields.io/badge/kotlin-1.2.71-blue.svg)](http://kotlinlang.org/) [![Prime version](https://img.shields.io/github/tag/ostelco/ostelco-core.svg)](https://github.com/ostelco/ostelco-core/tags) [![GitHub license](https://img.shields.io/github/license/ostelco/ostelco-core.svg)](https://github.com/ostelco/ostelco-core/blob/master/LICENSE) diff --git a/acceptance-tests/build.gradle b/acceptance-tests/build.gradle index 476060dc8..6c934663e 100644 --- a/acceptance-tests/build.gradle +++ b/acceptance-tests/build.gradle @@ -1,7 +1,7 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "application" - id "com.github.johnrengelman.shadow" version "2.0.4" + id "com.github.johnrengelman.shadow" version "4.0.0" } dependencies { diff --git a/admin-api/build.gradle b/admin-api/build.gradle index 75e6f1408..70f917bf0 100644 --- a/admin-api/build.gradle +++ b/admin-api/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" } diff --git a/analytics-module/build.gradle b/analytics-module/build.gradle index fea4ea378..b02d49302 100644 --- a/analytics-module/build.gradle +++ b/analytics-module/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" } diff --git a/app-notifier/build.gradle b/app-notifier/build.gradle index 94b971821..576935bca 100644 --- a/app-notifier/build.gradle +++ b/app-notifier/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" } diff --git a/auth-server/build.gradle b/auth-server/build.gradle index 648091699..82c788ffc 100644 --- a/auth-server/build.gradle +++ b/auth-server/build.gradle @@ -1,7 +1,7 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "application" - id "com.github.johnrengelman.shadow" version "2.0.4" + id "com.github.johnrengelman.shadow" version "4.0.0" id "idea" } diff --git a/bq-metrics-extractor/build.gradle b/bq-metrics-extractor/build.gradle index a5155aa67..c01398491 100644 --- a/bq-metrics-extractor/build.gradle +++ b/bq-metrics-extractor/build.gradle @@ -1,7 +1,7 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "application" - id "com.github.johnrengelman.shadow" version "2.0.4" + id "com.github.johnrengelman.shadow" version "4.0.0" id "idea" } diff --git a/build.gradle b/build.gradle index 714bc2027..1b9144677 100644 --- a/build.gradle +++ b/build.gradle @@ -32,15 +32,16 @@ subprojects { options.encoding = 'UTF-8' } ext { - kotlinVersion = "1.2.70" + kotlinVersion = "1.2.71" dropwizardVersion = "1.3.5" - googleCloudVersion = "1.45.0" + googleCloudVersion = "1.46.0" jacksonVersion = "2.9.7" - stripeVersion = "6.12.0" + stripeVersion = "7.0.0" guavaVersion = "26.0-jre" assertJVersion = "3.11.1" mockitoVersion = "2.22.0" firebaseVersion = "6.5.0" + beamVersion = "2.6.0" // Keeping it version 1.15.0 to be consistent with grpc via PubSub client lib // Keeping it version 1.15.0 to be consistent with netty via Firebase lib grpcVersion = "1.15.0" diff --git a/client-api/build.gradle b/client-api/build.gradle index 2dc95425d..b0519c812 100644 --- a/client-api/build.gradle +++ b/client-api/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" } diff --git a/dataflow-pipelines/build.gradle b/dataflow-pipelines/build.gradle index 5bb20659a..b075ea6b4 100644 --- a/dataflow-pipelines/build.gradle +++ b/dataflow-pipelines/build.gradle @@ -1,7 +1,7 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "application" - id "com.github.johnrengelman.shadow" version "2.0.4" + id "com.github.johnrengelman.shadow" version "4.0.0" id "idea" } @@ -13,8 +13,8 @@ dependencies { implementation "com.google.cloud:google-cloud-pubsub:$googleCloudVersion" - implementation 'com.google.cloud.dataflow:google-cloud-dataflow-java-sdk-all:2.5.0' - runtimeOnly 'org.apache.beam:beam-runners-google-cloud-dataflow-java:2.5.0' + implementation "org.apache.beam:beam-sdks-java-core:$beamVersion" + implementation "org.apache.beam:beam-runners-google-cloud-dataflow-java:$beamVersion" implementation 'ch.qos.logback:logback-classic:1.2.3' diff --git a/diameter-stack/build.gradle b/diameter-stack/build.gradle index 9060d9b76..4b2846403 100644 --- a/diameter-stack/build.gradle +++ b/diameter-stack/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" id "signing" id "maven" diff --git a/diameter-test/build.gradle b/diameter-test/build.gradle index 1e33e6c78..c6d970380 100644 --- a/diameter-test/build.gradle +++ b/diameter-test/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" id "signing" id "maven" diff --git a/ext-auth-provider/build.gradle b/ext-auth-provider/build.gradle index 69139c60d..2917c7371 100644 --- a/ext-auth-provider/build.gradle +++ b/ext-auth-provider/build.gradle @@ -1,7 +1,7 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "application" - id "com.github.johnrengelman.shadow" version "2.0.4" + id "com.github.johnrengelman.shadow" version "4.0.0" } dependencies { diff --git a/firebase-extensions/build.gradle b/firebase-extensions/build.gradle index d741c6479..c39c5cc27 100644 --- a/firebase-extensions/build.gradle +++ b/firebase-extensions/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" } diff --git a/firebase-store/build.gradle b/firebase-store/build.gradle index 14a377823..3c30ffebe 100644 --- a/firebase-store/build.gradle +++ b/firebase-store/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" } diff --git a/model/build.gradle b/model/build.gradle index 3c2a0a6cd..73d0f5aef 100644 --- a/model/build.gradle +++ b/model/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" } diff --git a/neo4j-store/build.gradle b/neo4j-store/build.gradle index 70ed8509f..a9161fc15 100644 --- a/neo4j-store/build.gradle +++ b/neo4j-store/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" } diff --git a/ocs/build.gradle b/ocs/build.gradle index aef0024c8..9444e7e74 100644 --- a/ocs/build.gradle +++ b/ocs/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" } @@ -13,7 +13,7 @@ dependencies { implementation project(':prime-modules') implementation 'com.lmax:disruptor:3.4.2' - implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:0.26.1" + // implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:0.30.0" testImplementation "org.jetbrains.kotlin:kotlin-test:$kotlinVersion" testImplementation "org.jetbrains.kotlin:kotlin-test-junit:$kotlinVersion" diff --git a/ocsgw/build.gradle b/ocsgw/build.gradle index ef85ce838..75daf2ccc 100644 --- a/ocsgw/build.gradle +++ b/ocsgw/build.gradle @@ -1,5 +1,6 @@ plugins { id "application" + // FIXME: unable to update to 4.0.0 id "com.github.johnrengelman.shadow" version "2.0.4" } @@ -18,7 +19,7 @@ dependencies { implementation 'ch.qos.logback:logback-classic:1.2.3' // log to gcp stack-driver - implementation 'com.google.cloud:google-cloud-logging-logback:0.63.0-alpha' + implementation 'com.google.cloud:google-cloud-logging-logback:0.64.0-alpha' testImplementation project(':diameter-test') testImplementation "org.junit.jupiter:junit-jupiter-api:$junit5Version" @@ -28,6 +29,13 @@ dependencies { testRuntimeOnly "org.junit.vintage:junit-vintage-engine:$junit5Version" } +shadowJar { + mainClassName = 'org.ostelco.ocsgw.OcsApplication' + mergeServiceFiles() + classifier = "uber" + version = null +} + test { // native support to Junit5 in Gradle 4.6+ useJUnitPlatform { @@ -39,13 +47,6 @@ test { } } -shadowJar { - mainClassName = 'org.ostelco.ocsgw.OcsApplication' - mergeServiceFiles() - classifier = "uber" - version = null -} - task pack(dependsOn: ['packDev', 'packProd']) task packProd(type: Zip, dependsOn: 'shadowJar') { diff --git a/payment-processor/build.gradle b/payment-processor/build.gradle index e47629b26..9c2069846 100644 --- a/payment-processor/build.gradle +++ b/payment-processor/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" id "idea" } diff --git a/prime-client-api/build.gradle b/prime-client-api/build.gradle index e711a3145..09a55d0d2 100644 --- a/prime-client-api/build.gradle +++ b/prime-client-api/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id 'java-library' id 'org.hidetake.swagger.generator' version '2.13.0' id "idea" diff --git a/prime-modules/build.gradle b/prime-modules/build.gradle index a302cb169..43c9bbe4e 100644 --- a/prime-modules/build.gradle +++ b/prime-modules/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" } diff --git a/prime/build.gradle b/prime/build.gradle index 42fc01c4e..6e73050c9 100644 --- a/prime/build.gradle +++ b/prime/build.gradle @@ -1,7 +1,7 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "application" - id "com.github.johnrengelman.shadow" version "2.0.4" + id "com.github.johnrengelman.shadow" version "4.0.0" id "idea" } @@ -18,7 +18,7 @@ sourceSets { } } -version = "1.15.0" +version = "1.16.0" repositories { maven { diff --git a/pseudonym-server/build.gradle b/pseudonym-server/build.gradle index eb1a06383..a6ceecd35 100644 --- a/pseudonym-server/build.gradle +++ b/pseudonym-server/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" } diff --git a/tools/neo4j-admin-tools/build.gradle b/tools/neo4j-admin-tools/build.gradle index 017a5f34c..c22bf1e04 100644 --- a/tools/neo4j-admin-tools/build.gradle +++ b/tools/neo4j-admin-tools/build.gradle @@ -1,7 +1,7 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "application" - id "com.github.johnrengelman.shadow" version "2.0.4" + id "com.github.johnrengelman.shadow" version "4.0.0" id "idea" } From 91a24a92361b03debd951cc35203fba2df645825 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Sat, 29 Sep 2018 15:41:32 +0200 Subject: [PATCH 82/85] Ignore dataflow beam test since it does not run on jdk11 --- build.gradle | 3 ++- dataflow-pipelines/build.gradle | 16 +++++++++++++++- .../pipelines/ConsumptionPerMsisdnTest.kt | 4 +++- ocsgw/build.gradle | 2 -- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/build.gradle b/build.gradle index 1b9144677..0211da753 100644 --- a/build.gradle +++ b/build.gradle @@ -38,10 +38,11 @@ subprojects { jacksonVersion = "2.9.7" stripeVersion = "7.0.0" guavaVersion = "26.0-jre" + junit5Version = "5.3.1" assertJVersion = "3.11.1" mockitoVersion = "2.22.0" firebaseVersion = "6.5.0" - beamVersion = "2.6.0" + beamVersion = "2.7.0" // Keeping it version 1.15.0 to be consistent with grpc via PubSub client lib // Keeping it version 1.15.0 to be consistent with netty via Firebase lib grpcVersion = "1.15.0" diff --git a/dataflow-pipelines/build.gradle b/dataflow-pipelines/build.gradle index b075ea6b4..54b887a52 100644 --- a/dataflow-pipelines/build.gradle +++ b/dataflow-pipelines/build.gradle @@ -18,8 +18,11 @@ dependencies { implementation 'ch.qos.logback:logback-classic:1.2.3' - testImplementation "org.jetbrains.kotlin:kotlin-test-junit:$kotlinVersion" testRuntimeOnly 'org.hamcrest:hamcrest-all:1.3' + testRuntimeOnly "org.apache.beam:beam-runners-direct-java:$beamVersion" + + testImplementation "org.junit.jupiter:junit-jupiter-api:$junit5Version" + testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:$junit5Version" } shadowJar { @@ -29,4 +32,15 @@ shadowJar { version = null } +test { + // native support to Junit5 in Gradle 4.6+ + useJUnitPlatform { + includeEngines 'junit-jupiter' + } + testLogging { + exceptionFormat = 'full' + events "PASSED", "FAILED", "SKIPPED" + } +} + apply from: '../jacoco.gradle' \ No newline at end of file diff --git a/dataflow-pipelines/src/test/kotlin/org/ostelco/dataflow/pipelines/ConsumptionPerMsisdnTest.kt b/dataflow-pipelines/src/test/kotlin/org/ostelco/dataflow/pipelines/ConsumptionPerMsisdnTest.kt index 0a785c668..0ac637e44 100644 --- a/dataflow-pipelines/src/test/kotlin/org/ostelco/dataflow/pipelines/ConsumptionPerMsisdnTest.kt +++ b/dataflow-pipelines/src/test/kotlin/org/ostelco/dataflow/pipelines/ConsumptionPerMsisdnTest.kt @@ -12,6 +12,8 @@ import org.joda.time.Instant import org.junit.Rule import org.junit.Test import org.junit.experimental.categories.Category +import org.junit.jupiter.api.condition.EnabledOnJre +import org.junit.jupiter.api.condition.JRE.JAVA_8 import org.ostelco.analytics.api.AggregatedDataTrafficInfo import org.ostelco.analytics.api.DataTrafficInfo import org.ostelco.dataflow.pipelines.definitions.consumptionPerMsisdn @@ -19,11 +21,11 @@ import org.ostelco.dataflow.pipelines.definitions.consumptionPerMsisdn class ConsumptionPerMsisdnTest { @Rule - @Transient @JvmField val pipeline: TestPipeline? = TestPipeline.create() @Test + @EnabledOnJre(JAVA_8) @Category(NeedsRunner::class) fun testPipeline() { diff --git a/ocsgw/build.gradle b/ocsgw/build.gradle index 75daf2ccc..021f7dab9 100644 --- a/ocsgw/build.gradle +++ b/ocsgw/build.gradle @@ -4,8 +4,6 @@ plugins { id "com.github.johnrengelman.shadow" version "2.0.4" } -ext.junit5Version = "5.3.1" - dependencies { implementation project(':ocs-grpc-api') implementation project(':analytics-grpc-api') From 6159ceb8fb50845bc0c2ff7feadca4b48aa80f17 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Sat, 29 Sep 2018 16:31:20 +0200 Subject: [PATCH 83/85] Multiple jdk building in travis --- .circleci/config.yml | 9 +++++---- docker-compose.override.yaml | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9e2e0dc11..68374faf5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,8 +17,8 @@ jobs: git checkout ${CIRCLE_BRANCH} git checkout develop git merge ${CIRCLE_BRANCH} -m "Merging ${CIRCLE_BRANCH} into develop." - # Show the java version installed. - - run: java -version + # Show the javac version installed. + - run: javac -version - run: name: Pulling Gradle cache @@ -103,8 +103,9 @@ jobs: ### JOBS FOR on-PR-merge-to-dev PIPELINE build-code: - machine: - enabled: true + + docker: + - image: circleci/openjdk:11-jdk-sid steps: - checkout diff --git a/docker-compose.override.yaml b/docker-compose.override.yaml index 16b2785f0..f920554c9 100644 --- a/docker-compose.override.yaml +++ b/docker-compose.override.yaml @@ -114,7 +114,7 @@ services: datastore-emulator: container_name: datastore-emulator - image: google/cloud-sdk:206.0.0 + image: google/cloud-sdk:218.0.0 expose: - "8081" environment: From db19f6c97fbbaffbf7057b801f39747b5c57aae1 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Sat, 29 Sep 2018 22:06:55 +0200 Subject: [PATCH 84/85] Added metrics grpc endpoint to prime in prod --- prime/infra/README.md | 2 ++ prime/infra/dev/metrics-api.yaml | 2 +- prime/infra/prod/metrics-api.yaml | 30 ++++++++++++++++++++++ prime/infra/prod/prime.yaml | 42 ++++++++++++++++++++++++++++++- 4 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 prime/infra/prod/metrics-api.yaml diff --git a/prime/infra/README.md b/prime/infra/README.md index 5103ef877..e1d3d696d 100644 --- a/prime/infra/README.md +++ b/prime/infra/README.md @@ -83,6 +83,8 @@ Reference: Generate self-contained protobuf descriptor file - `ocs_descriptor.pb` & `metrics_descriptor.pb` ```bash +pyenv versions +pyenv local 3.5.2 pip install grpcio grpcio-tools python -m grpc_tools.protoc \ diff --git a/prime/infra/dev/metrics-api.yaml b/prime/infra/dev/metrics-api.yaml index 750adc869..fafd2ff79 100644 --- a/prime/infra/dev/metrics-api.yaml +++ b/prime/infra/dev/metrics-api.yaml @@ -27,4 +27,4 @@ authentication: rules: - selector: "*" requirements: - - provider_id: google_service_account + - provider_id: google_service_account \ No newline at end of file diff --git a/prime/infra/prod/metrics-api.yaml b/prime/infra/prod/metrics-api.yaml new file mode 100644 index 000000000..7badcd6a5 --- /dev/null +++ b/prime/infra/prod/metrics-api.yaml @@ -0,0 +1,30 @@ +type: google.api.Service + +config_version: 3 + +name: metrics.ostelco.org + +title: Prime Metrics Reporter Service gRPC API + +apis: + - name: org.ostelco.prime.metrics.api.OcsgwAnalyticsService + +usage: + rules: + # All methods can be called without an API Key. + - selector: "*" + allow_unregistered_calls: true + +authentication: + providers: + - id: google_service_account + issuer: prime-service-account@pantel-2decb.iam.gserviceaccount.com + jwks_uri: https://www.googleapis.com/robot/v1/metadata/x509/prime-service-account@pantel-2decb.iam.gserviceaccount.com + audiences: > + https://metrics.ostelco.org/org.ostelco.prime.metrics.api.OcsgwAnalyticsService, + metrics.ostelco.org/org.ostelco.prime.metrics.api.OcsgwAnalyticsService, + metrics.ostelco.org + rules: + - selector: "*" + requirements: + - provider_id: google_service_account \ No newline at end of file diff --git a/prime/infra/prod/prime.yaml b/prime/infra/prod/prime.yaml index 2dfc8a11a..54691ae8a 100644 --- a/prime/infra/prod/prime.yaml +++ b/prime/infra/prod/prime.yaml @@ -37,6 +37,25 @@ spec: --- apiVersion: v1 kind: Service +metadata: + name: prime-metrics + labels: + app: prime + tier: backend +spec: + type: LoadBalancer + loadBalancerIP: 35.240.23.167 + ports: + - name: grpc + port: 443 + targetPort: 9443 + protocol: TCP + selector: + app: prime + tier: backend +--- +apiVersion: v1 +kind: Service metadata: name: pseudonym-server-service labels: @@ -71,7 +90,7 @@ spec: prometheus.io/port: '8081' spec: containers: - - name: esp + - name: ocs-esp image: gcr.io/endpoints-release/endpoints-runtime:1 args: [ "--http2_port=9000", @@ -105,6 +124,23 @@ spec: - mountPath: /etc/nginx/ssl name: api-ostelco-ssl readOnly: true + - name: metrics-esp + image: gcr.io/endpoints-release/endpoints-runtime:1 + args: [ + "--http2_port=9004", + "--ssl_port=9443", + "--status_port=8094", + "--service=metrics.ostelco.org", + "--rollout_strategy=managed", + "--backend=grpc://127.0.0.1:8083" + ] + ports: + - containerPort: 9004 + - containerPort: 9443 + volumeMounts: + - mountPath: /etc/nginx/ssl + name: metrics-ostelco-ssl + readOnly: true - name: prime image: eu.gcr.io/pantel-2decb/prime:PRIME_VERSION imagePullPolicy: Always @@ -132,6 +168,7 @@ spec: - containerPort: 8080 - containerPort: 8081 - containerPort: 8082 + - containerPort: 8083 volumes: - name: secret-config secret: @@ -142,3 +179,6 @@ spec: - name: ocs-ostelco-ssl secret: secretName: ocs-ostelco-ssl + - name: metrics-ostelco-ssl + secret: + secretName: metrics-ostelco-ssl From 17363403456a190b0405ad436d3becd14656a596 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Sat, 29 Sep 2018 22:21:13 +0200 Subject: [PATCH 85/85] Fixed issue with wildcard certificates --- certs/dev.ostelco.org/.gitignore | 2 -- certs/metrics.ostelco.org/.gitignore | 2 -- certs/ocs.ostelco.org/.gitignore | 2 -- ocsgw/build.gradle | 14 -------------- 4 files changed, 20 deletions(-) delete mode 100644 certs/dev.ostelco.org/.gitignore delete mode 100644 certs/metrics.ostelco.org/.gitignore delete mode 100644 certs/ocs.ostelco.org/.gitignore diff --git a/certs/dev.ostelco.org/.gitignore b/certs/dev.ostelco.org/.gitignore deleted file mode 100644 index 47c805dfe..000000000 --- a/certs/dev.ostelco.org/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -nginx.crt -nginx.key \ No newline at end of file diff --git a/certs/metrics.ostelco.org/.gitignore b/certs/metrics.ostelco.org/.gitignore deleted file mode 100644 index 47c805dfe..000000000 --- a/certs/metrics.ostelco.org/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -nginx.crt -nginx.key \ No newline at end of file diff --git a/certs/ocs.ostelco.org/.gitignore b/certs/ocs.ostelco.org/.gitignore deleted file mode 100644 index 47c805dfe..000000000 --- a/certs/ocs.ostelco.org/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -nginx.crt -nginx.key \ No newline at end of file diff --git a/ocsgw/build.gradle b/ocsgw/build.gradle index 021f7dab9..0c92891f4 100644 --- a/ocsgw/build.gradle +++ b/ocsgw/build.gradle @@ -95,20 +95,6 @@ task packDev(type: Zip, dependsOn: 'shadowJar') { fileName.replace('dev.', '') } } - // TODO vihang: figure out why wild-card certs fail to verify - from ('../certs/dev.ostelco.org/nginx.crt') { - into (project.name + '/config/') - rename { String fileName -> - fileName.replace('nginx', 'ocs') - } - } - from ('../certs/dev.ostelco.org/nginx.crt') { - into (project.name + '/config/') - rename { String fileName -> - fileName.replace('nginx', 'metrics') - } - } - // END of certs from ('config/pantel-prod.json') { into (project.name + '/config/') }