diff --git a/.circleci/config.yml b/.circleci/config.yml index bb33aa710..68374faf5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,8 +17,8 @@ jobs: git checkout ${CIRCLE_BRANCH} git checkout develop git merge ${CIRCLE_BRANCH} -m "Merging ${CIRCLE_BRANCH} into develop." - # Show the java version installed. - - run: java -version + # Show the javac version installed. + - run: javac -version - run: name: Pulling Gradle cache @@ -39,7 +39,13 @@ jobs: - run: name: Build entire repo command: ./gradlew clean build -info -s -x :neo4j-store:test - + - run: + name: Push Gradle cache + command: | + rm -f ~/.gradle/caches/modules-2/modules-2.lock + rm -fr ~/.gradle/caches/*/plugin-resolution/ + tar -czvf ~/caches.tar.gz -C ~/.gradle/caches . + gsutil cp ~/caches.tar.gz gs://pi-ostelco-core-gradle-cache # persisting the entire project with its generated artifacts. They are needed in the build-image job below. # the default working directory in circleci is ~/project/ - persist_to_workspace: @@ -72,7 +78,7 @@ jobs: CODACY_MODULE: com.codacy.CodacyCoverageReporter docker: - - image: circleci/openjdk:8u171-jdk + - image: circleci/openjdk:11-jdk-sid steps: - run: @@ -97,8 +103,9 @@ jobs: ### JOBS FOR on-PR-merge-to-dev PIPELINE build-code: - machine: - enabled: true + + docker: + - image: circleci/openjdk:11-jdk-sid steps: - checkout @@ -342,9 +349,9 @@ workflows: - deploy-to-dev: requires: - update-dev-endpoints - - create-PR-to-master: - requires: - - deploy-to-dev +# - create-PR-to-master: +# requires: +# - deploy-to-dev deploy-to-prod: jobs: diff --git a/.circleci/prime-dev-values.yaml b/.circleci/prime-dev-values.yaml index af7ba5042..7d9c67148 100644 --- a/.circleci/prime-dev-values.yaml +++ b/.circleci/prime-dev-values.yaml @@ -16,6 +16,7 @@ prime: STRIPE_API_KEY: "" DATA_TRAFFIC_TOPIC: "data-traffic" PURCHASE_INFO_TOPIC: "purchase-info" + ACTIVE_USERS_TOPIC: "active-users" ports: - 8080 - 8081 diff --git a/.circleci/prime-prod-values-template.yaml b/.circleci/prime-prod-values-template.yaml index 1664a9e9c..59615a8c3 100644 --- a/.circleci/prime-prod-values-template.yaml +++ b/.circleci/prime-prod-values-template.yaml @@ -15,6 +15,7 @@ prime: STRIPE_API_KEY: "" DATA_TRAFFIC_TOPIC: "data-traffic" PURCHASE_INFO_TOPIC: "purchase-info" + ACTIVE_USERS_TOPIC: "active-users" ports: - 8080 - 8081 diff --git a/.circleci/prime-prod-values.yaml b/.circleci/prime-prod-values.yaml index 12a906c71..19bc9752e 100644 --- a/.circleci/prime-prod-values.yaml +++ b/.circleci/prime-prod-values.yaml @@ -15,6 +15,7 @@ prime: STRIPE_API_KEY: "" DATA_TRAFFIC_TOPIC: "data-traffic" PURCHASE_INFO_TOPIC: "purchase-info" + ACTIVE_USERS_TOPIC: "active-users" ports: - 8080 diff --git a/README.md b/README.md index 8f5acb09d..5ba31bb3c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -[![Kotlin version badge](https://img.shields.io/badge/kotlin-1.2.70-blue.svg)](http://kotlinlang.org/) +[![Kotlin version badge](https://img.shields.io/badge/kotlin-1.2.71-blue.svg)](http://kotlinlang.org/) [![Prime version](https://img.shields.io/github/tag/ostelco/ostelco-core.svg)](https://github.com/ostelco/ostelco-core/tags) [![GitHub license](https://img.shields.io/github/license/ostelco/ostelco-core.svg)](https://github.com/ostelco/ostelco-core/blob/master/LICENSE) diff --git a/acceptance-tests/Dockerfile b/acceptance-tests/Dockerfile index 35e8792f4..d695fff0f 100644 --- a/acceptance-tests/Dockerfile +++ b/acceptance-tests/Dockerfile @@ -1,4 +1,4 @@ -FROM azul/zulu-openjdk:8u181-8.31.0.1 +FROM openjdk:11 MAINTAINER CSI "csi@telenordigital.com" diff --git a/acceptance-tests/build.gradle b/acceptance-tests/build.gradle index 476060dc8..6c934663e 100644 --- a/acceptance-tests/build.gradle +++ b/acceptance-tests/build.gradle @@ -1,7 +1,7 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "application" - id "com.github.johnrengelman.shadow" version "2.0.4" + id "com.github.johnrengelman.shadow" version "4.0.0" } dependencies { diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt index 5e933ac13..ea0e5b232 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt @@ -55,15 +55,6 @@ object StripePayment { return token.card.id } - fun getCardIdForSourceId(sourceId: String) : String { - - // https://stripe.com/docs/api/java#create_source - Stripe.apiKey = System.getenv("STRIPE_API_KEY") - - val source = Source.retrieve(sourceId) - return source.id - } - /** * Obtains 'default source' directly from Stripe. Use in tests to * verify that the correspondng 'setDefaultSource' API works as @@ -91,15 +82,11 @@ object StripePayment { return customers.filter { it.email.equals(email) }.first().id } - fun deleteAllCustomers() { + fun deleteCustomer(email: String) { // https://stripe.com/docs/api/java#create_card_token Stripe.apiKey = System.getenv("STRIPE_API_KEY") - - do { - val customers = Customer.list(emptyMap()).data - customers.forEach { customer -> - customer.delete() - } - } while (customers.isNotEmpty()) + val customers = Customer.list(emptyMap()).data + customers.filter { it.email == email } + .forEach { it.delete() } } } \ No newline at end of file diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/common/TestUser.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/common/TestUser.kt index d55a70598..3a278d77b 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/common/TestUser.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/common/TestUser.kt @@ -12,7 +12,7 @@ fun createProfile(name: String, email: String) { .name(name) .address("") .city("") - .country("") + .country("NO") .postCode("") .referralId("") diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/HttpClientUtil.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/HttpClientUtil.kt index 4426da7e1..c8b04a7fc 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/HttpClientUtil.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/HttpClientUtil.kt @@ -54,6 +54,17 @@ inline fun put(execute: HttpRequest.() -> Unit): T { return response.readEntity(object : GenericType() {}) } +/** + * DSL function for DELETE operation + */ +inline fun delete(execute: HttpRequest.() -> Unit): T { + val request = HttpRequest().apply(execute) + val response = HttpClient.send(request.path, request.queryParams, request.headerParams, request.subscriberId) + .delete() + assertEquals(200, response.status) { response.readEntity(String::class.java) } + return response.readEntity(object : GenericType() {}) +} + fun assertEquals(expected: T, actual: T, lazyMessage: () -> String) { var message = "" if (expected != actual) { diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt index 8ada341c8..86cbc9f0b 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt @@ -24,6 +24,7 @@ import java.time.Instant import java.util.* import kotlin.test.assertEquals import kotlin.test.assertFails +import kotlin.test.assertFailsWith import kotlin.test.assertFalse import kotlin.test.assertNotNull import kotlin.test.assertNull @@ -42,7 +43,7 @@ class ProfileTest { .name("Test Profile User") .address("") .city("") - .country("") + .country("NO") .postCode("") .referralId("") @@ -88,7 +89,6 @@ class ProfileTest { .address("") .postCode("") .city("") - .country("") val clearedProfile: Profile = put { path = "/profile" @@ -101,7 +101,18 @@ class ProfileTest { assertEquals("", clearedProfile.address, "Incorrect 'address' in response after clearing profile") assertEquals("", clearedProfile.postCode, "Incorrect 'postcode' in response after clearing profile") assertEquals("", clearedProfile.city, "Incorrect 'city' in response after clearing profile") - assertEquals("", clearedProfile.country, "Incorrect 'country' in response after clearing profile") + + updatedProfile.country("") + + // A test in 'HttpClientUtil' checks for status code 200 while the + // expected status code is actually 400. + assertFailsWith(AssertionError::class, "Incorrectly accepts that 'country' is cleared/not set") { + put { + path = "/profile" + body = updatedProfile + subscriberId = email + } + } } @Test @@ -231,71 +242,152 @@ class SourceTest { @Test fun `jersey test - POST source create`() { - StripePayment.deleteAllCustomers() - val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Payment Source", email = email) + try { - val tokenId = StripePayment.createPaymentTokenId() + createProfile(name = "Test Payment Source", email = email) - // Ties source with user profile both local and with Stripe - post { - path = "/paymentSources" - subscriberId = email - queryParams = mapOf("sourceId" to tokenId) - } + val tokenId = StripePayment.createPaymentTokenId() - Thread.sleep(200) + // Ties source with user profile both local and with Stripe + post { + path = "/paymentSources" + subscriberId = email + queryParams = mapOf("sourceId" to tokenId) + } - val sources: PaymentSourceList = get { - path = "/paymentSources" - subscriberId = email - } - assert(sources.isNotEmpty()) { "Expected at least one payment source for profile $email" } + Thread.sleep(200) + + val sources: PaymentSourceList = get { + path = "/paymentSources" + subscriberId = email + } + assert(sources.isNotEmpty()) { "Expected at least one payment source for profile $email" } - val cardId = StripePayment.getCardIdForTokenId(tokenId) - assertNotNull(sources.first { it.id == cardId }, "Expected card $cardId in list of payment sources for profile $email") + val cardId = StripePayment.getCardIdForTokenId(tokenId) + assertNotNull(sources.first { it.id == cardId }, "Expected card $cardId in list of payment sources for profile $email") + } finally { + StripePayment.deleteCustomer(email = email) + } } @Test fun `jersey test - GET list sources`() { - StripePayment.deleteAllCustomers() + val email = "purchase-${randomInt()}@test.com" + + try { + createProfile(name = "Test Payment Source", email = email) + + Thread.sleep(200) + + val createdIds = listOf(createTokenWithStripe(email), + createSourceWithStripe(email), + createTokenWithStripe(email), + createSourceWithStripe(email)) + + val sources: PaymentSourceList = get { + path = "/paymentSources" + subscriberId = email + } + + val ids = createdIds.map { getCardIdForTokenFromStripe(it) } + + assert(sources.isNotEmpty()) { "Expected at least one payment source for profile $email" } + assert(sources.map { it.id }.containsAll(ids)) + { "Expected to find all of $ids in list of sources for profile $email" } + + sources.forEach { + assert(it.id.isNotEmpty()) { "Expected 'id' to be set in source account details for profile $email" } + assert(arrayOf("card", "source").contains(it.type)) { + "Unexpected source account type ${it.type} for profile $email" + } + } + } finally { + StripePayment.deleteCustomer(email = email) + } + } + + @Test + fun `jersey test - PUT source set default`() { val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Payment Source", email = email) + try { + createProfile(name = "Test Payment Source", email = email) + + val tokenId = StripePayment.createPaymentTokenId() + val cardId = StripePayment.getCardIdForTokenId(tokenId) + + // Ties source with user profile both local and with Stripe + post { + path = "/paymentSources" + subscriberId = email + queryParams = mapOf("sourceId" to tokenId) + } - Thread.sleep(200) + Thread.sleep(200) - val createdIds = listOf(createTokenWithStripe(email), - createSourceWithStripe(email), - createTokenWithStripe(email), - createSourceWithStripe(email)) + val newTokenId = StripePayment.createPaymentTokenId() + val newCardId = StripePayment.getCardIdForTokenId(newTokenId) - val sources : PaymentSourceList = get { - path = "/paymentSources" - subscriberId = email + post { + path = "/paymentSources" + subscriberId = email + queryParams = mapOf("sourceId" to newTokenId) + } + + // TODO: Update to fetch the Stripe customerId from 'admin' API when ready. + val customerId = StripePayment.getCustomerIdForEmail(email) + + // Verify that original 'sourceId/card' is default. + assertEquals(cardId, StripePayment.getDefaultSourceForCustomer(customerId), + "Expected $cardId to be default source for $customerId") + + // Set new default card. + put { + path = "/paymentSources" + subscriberId = email + queryParams = mapOf("sourceId" to newCardId) + } + + assertEquals(newCardId, StripePayment.getDefaultSourceForCustomer(customerId), + "Expected $newCardId to be default source for $customerId") + } finally { + StripePayment.deleteCustomer(email = email) } + } + + @Test + fun `okhttp test - DELETE source`() { + + val email = "purchase-${randomInt()}@test.com" + + try { - val ids = createdIds.map { getIdFromStripe(it) } + createProfile(name = "Test Payment Source", email = email) - assert(sources.isNotEmpty()) { "Expected at least one payment source for profile $email" } - assert(sources.map{ it.id }.containsAll(ids)) - { "Expected to find all of $ids in list of sources for profile $email" } + Thread.sleep(200) - sources.forEach { - assert(it.id.isNotEmpty()) { "Expected 'id' to be set in source account details for profile $email" } - assert(arrayOf("card", "source").contains(it.type)) { - "Unexpected source account type ${it.type} for profile $email" + val createdIds = listOf(getCardIdForTokenFromStripe(createTokenWithStripe(email)), + createSourceWithStripe(email)) + + val deletedIds = createdIds.map { it -> removeSourceWithStripe(email, it) } + + assert(createdIds.containsAll(deletedIds.toSet())) { + "Failed to delete one or more sources: ${createdIds.toSet() - deletedIds.toSet()}" } + } finally { + StripePayment.deleteCustomer(email = email) } } - private fun getIdFromStripe(tokenId : String) : String { - if (tokenId.startsWith("src_")) { - return StripePayment.getCardIdForSourceId(tokenId) + // Helpers for source handling with Stripe. + + private fun getCardIdForTokenFromStripe(id: String) : String { + if (id.startsWith("tok_")) { + return StripePayment.getCardIdForTokenId(id) } - return StripePayment.getCardIdForTokenId(tokenId) + return id } private fun createTokenWithStripe(email: String) : String { @@ -322,51 +414,14 @@ class SourceTest { return sourceId } - @Test - fun `jersey test - PUT source set default`() { - - StripePayment.deleteAllCustomers() - - val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Payment Source", email = email) - - val tokenId = StripePayment.createPaymentTokenId() - val cardId = StripePayment.getCardIdForTokenId(tokenId) - - // Ties source with user profile both local and with Stripe - post { + private fun removeSourceWithStripe(email: String, sourceId: String) : String { + val removedSource = delete { path = "/paymentSources" subscriberId = email - queryParams = mapOf("sourceId" to tokenId) - } - - Thread.sleep(200) - - val newTokenId = StripePayment.createPaymentTokenId() - val newCardId = StripePayment.getCardIdForTokenId(newTokenId) - - post { - path = "/paymentSources" - subscriberId = email - queryParams = mapOf("sourceId" to newTokenId) - } - - // TODO: Update to fetch the Stripe customerId from 'admin' API when ready. - val customerId = StripePayment.getCustomerIdForEmail(email) - - // Verify that original 'sourceId/card' is default. - assertEquals(cardId, StripePayment.getDefaultSourceForCustomer(customerId), - "Expected $cardId to be default source for $customerId") - - // Set new default card. - put { - path = "/paymentSources" - subscriberId = email - queryParams = mapOf("sourceId" to newCardId) + queryParams = mapOf("sourceId" to sourceId) } - assertEquals(newCardId, StripePayment.getDefaultSourceForCustomer(customerId), - "Expected $newCardId to be default source for $customerId") + return removedSource.id } } @@ -375,151 +430,156 @@ class PurchaseTest { @Test fun `jersey test - POST products purchase`() { - StripePayment.deleteAllCustomers() - val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Purchase User", email = email) + try { + createProfile(name = "Test Purchase User", email = email) - val balanceBefore = get> { - path = "/bundles" - subscriberId = email - }.first().balance + val balanceBefore = get> { + path = "/bundles" + subscriberId = email + }.first().balance - val productSku = "1GB_249NOK" - val sourceId = StripePayment.createPaymentTokenId() + val productSku = "1GB_249NOK" + val sourceId = StripePayment.createPaymentTokenId() - post { - path = "/products/$productSku/purchase" - subscriberId = email - queryParams = mapOf("sourceId" to sourceId) - } + post { + path = "/products/$productSku/purchase" + subscriberId = email + queryParams = mapOf("sourceId" to sourceId) + } - Thread.sleep(100) // wait for 100 ms for balance to be updated in db + Thread.sleep(100) // wait for 100 ms for balance to be updated in db - val balanceAfter = get> { - path = "/bundles" - subscriberId = email - }.first().balance + val balanceAfter = get> { + path = "/bundles" + subscriberId = email + }.first().balance - assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") + assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") - val purchaseRecords: PurchaseRecordList = get { - path = "/purchases" - subscriberId = email - } + val purchaseRecords: PurchaseRecordList = get { + path = "/purchases" + subscriberId = email + } - purchaseRecords.sortBy { it.timestamp } + purchaseRecords.sortBy { it.timestamp } - assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } - assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } + assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + } finally { + StripePayment.deleteCustomer(email = email) + } } @Test fun `jersey test - POST products purchase using default source`() { - StripePayment.deleteAllCustomers() - val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Purchase User with Default Payment Source", email = email) + try { + createProfile(name = "Test Purchase User with Default Payment Source", email = email) - val sourceId = StripePayment.createPaymentTokenId() + val sourceId = StripePayment.createPaymentTokenId() - val paymentSource: PaymentSource = post { - path = "/paymentSources" - subscriberId = email - queryParams = mapOf("sourceId" to sourceId) - } + val paymentSource: PaymentSource = post { + path = "/paymentSources" + subscriberId = email + queryParams = mapOf("sourceId" to sourceId) + } - assertNotNull(paymentSource.id, message = "Failed to create payment source") + assertNotNull(paymentSource.id, message = "Failed to create payment source") - val balanceBefore = get> { - path = "/bundles" - subscriberId = email - }.first().balance + val balanceBefore = get> { + path = "/bundles" + subscriberId = email + }.first().balance - val productSku = "1GB_249NOK" + val productSku = "1GB_249NOK" - post { - path = "/products/$productSku/purchase" - subscriberId = email - } + post { + path = "/products/$productSku/purchase" + subscriberId = email + } - Thread.sleep(100) // wait for 100 ms for balance to be updated in db + Thread.sleep(100) // wait for 100 ms for balance to be updated in db - val balanceAfter = get> { - path = "/bundles" - subscriberId = email - }.first().balance + val balanceAfter = get> { + path = "/bundles" + subscriberId = email + }.first().balance - assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") + assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") - val purchaseRecords: PurchaseRecordList = get { - path = "/purchases" - subscriberId = email - } + val purchaseRecords: PurchaseRecordList = get { + path = "/purchases" + subscriberId = email + } - purchaseRecords.sortBy { it.timestamp } + purchaseRecords.sortBy { it.timestamp } - assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } - assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } + assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + } finally { + StripePayment.deleteCustomer(email = email) + } } @Test fun `jersey test - POST products purchase add source then pay with it`() { - StripePayment.deleteAllCustomers() - val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Purchase User with Default Payment Source", email = email) + try { + createProfile(name = "Test Purchase User with Default Payment Source", email = email) - val sourceId = StripePayment.createPaymentTokenId() + val sourceId = StripePayment.createPaymentTokenId() - val paymentSource: PaymentSource = post { - path = "/paymentSources" - subscriberId = email - queryParams = mapOf("sourceId" to sourceId) - } + val paymentSource: PaymentSource = post { + path = "/paymentSources" + subscriberId = email + queryParams = mapOf("sourceId" to sourceId) + } - assertNotNull(paymentSource.id, message = "Failed to create payment source") + assertNotNull(paymentSource.id, message = "Failed to create payment source") - val subscriptionStatusBefore: SubscriptionStatus = get { - path = "/subscription/status" - subscriberId = email - } - val balanceBefore = subscriptionStatusBefore.remaining + val subscriptionStatusBefore: SubscriptionStatus = get { + path = "/subscription/status" + subscriberId = email + } + val balanceBefore = subscriptionStatusBefore.remaining - val productSku = "1GB_249NOK" + val productSku = "1GB_249NOK" - post { - path = "/products/$productSku/purchase" - subscriberId = email - queryParams = mapOf("sourceId" to paymentSource.id) - } + post { + path = "/products/$productSku/purchase" + subscriberId = email + queryParams = mapOf("sourceId" to paymentSource.id) + } - Thread.sleep(100) // wait for 100 ms for balance to be updated in db + Thread.sleep(100) // wait for 100 ms for balance to be updated in db - val subscriptionStatusAfter: SubscriptionStatus = get { - path = "/subscription/status" - subscriberId = email - } - val balanceAfter = subscriptionStatusAfter.remaining + val subscriptionStatusAfter: SubscriptionStatus = get { + path = "/subscription/status" + subscriberId = email + } + val balanceAfter = subscriptionStatusAfter.remaining - assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") + assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") - val purchaseRecords: PurchaseRecordList = get { - path = "/purchases" - subscriberId = email - } + val purchaseRecords: PurchaseRecordList = get { + path = "/purchases" + subscriberId = email + } - purchaseRecords.sortBy { it.timestamp } + purchaseRecords.sortBy { it.timestamp } - assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } - assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } + assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + } finally { + StripePayment.deleteCustomer(email = email) + } } - @Test fun `jersey test - POST products purchase without payment`() { @@ -727,4 +787,4 @@ class ReferralTest { assertEquals(listOf(freeProductForReferred), secondSubscriptionStatus.purchaseRecords.map { it.product }) } -} \ No newline at end of file +} diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt index 715353b6c..1a33776aa 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt @@ -8,6 +8,7 @@ import org.ostelco.at.common.expectedProducts import org.ostelco.at.common.getLogger import org.ostelco.at.common.randomInt import org.ostelco.at.okhttp.ClientFactory.clientForSubject +import org.ostelco.prime.client.ApiException import org.ostelco.prime.client.api.DefaultApi import org.ostelco.prime.client.model.ApplicationToken import org.ostelco.prime.client.model.Consent @@ -22,6 +23,7 @@ import java.time.Instant import java.util.* import kotlin.test.assertEquals import kotlin.test.assertFails +import kotlin.test.assertFailsWith import kotlin.test.assertNotNull import kotlin.test.assertNull @@ -39,7 +41,7 @@ class ProfileTest { .name("Test Profile User") .address("") .city("") - .country("") + .country("NO") .postCode("") .referralId("") @@ -70,7 +72,6 @@ class ProfileTest { .address("") .postCode("") .city("") - .country("") val clearedProfile: Profile = client.updateProfile(updatedProfile) @@ -79,7 +80,12 @@ class ProfileTest { assertEquals("", clearedProfile.address, "Incorrect 'address' in response after clearing profile") assertEquals("", clearedProfile.postCode, "Incorrect 'postcode' in response after clearing profile") assertEquals("", clearedProfile.city, "Incorrect 'city' in response after clearing profile") - assertEquals("", clearedProfile.country, "Incorrect 'country' in response after clearing profile") + + updatedProfile.country("") + + assertFailsWith(ApiException::class, "Incorrectly accepts that 'country' is cleared/not set") { + client.updateProfile(updatedProfile) + } } @Test @@ -203,119 +209,159 @@ class SourceTest { @Test fun `okhttp test - POST source create`() { - StripePayment.deleteAllCustomers() - val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Payment Source", email = email) + try { + createProfile(name = "Test Payment Source", email = email) - val client = clientForSubject(subject = email) + val client = clientForSubject(subject = email) - val tokenId = StripePayment.createPaymentTokenId() - val cardId = StripePayment.getCardIdForTokenId(tokenId) + val tokenId = StripePayment.createPaymentTokenId() + val cardId = StripePayment.getCardIdForTokenId(tokenId) - // Ties source with user profile both local and with Stripe - client.createSource(tokenId) + // Ties source with user profile both local and with Stripe + client.createSource(tokenId) - Thread.sleep(200) + Thread.sleep(200) - val sources = client.listSources() + val sources = client.listSources() - assert(sources.isNotEmpty()) { "Expected at least one payment source for profile $email" } - assertNotNull(sources.first { it.id == cardId }, - "Expected card $cardId in list of payment sources for profile $email") + assert(sources.isNotEmpty()) { "Expected at least one payment source for profile $email" } + assertNotNull(sources.first { it.id == cardId }, + "Expected card $cardId in list of payment sources for profile $email") + } finally { + StripePayment.deleteCustomer(email = email) + } } @Test fun `okhttp test - GET list sources`() { - StripePayment.deleteAllCustomers() - val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Payment Source", email = email) + try { + createProfile(name = "Test Payment Source", email = email) - val client = clientForSubject(subject = email) + val client = clientForSubject(subject = email) - Thread.sleep(200) + Thread.sleep(200) - val createdIds = listOf(createTokenWithStripe(client), - createSourceWithStripe(client), - createTokenWithStripe(client), - createSourceWithStripe(client)) + val createdIds = listOf(createTokenWithStripe(client), + createSourceWithStripe(client), + createTokenWithStripe(client), + createSourceWithStripe(client)) - val sources = client.listSources() + val sources = client.listSources() - val ids = createdIds.map { getIdFromStripe(it) } + val ids = createdIds.map { getCardIdForTokenFromStripe(it) } - assert(sources.isNotEmpty()) { "Expected at least one payment source for profile $email" } - assert(sources.map{ it.id }.containsAll(ids)) - { "Expected to find all of $ids in list of sources for profile $email" } + assert(sources.isNotEmpty()) { "Expected at least one payment source for profile $email" } + assert(sources.map{ it.id }.containsAll(ids)) + { "Expected to find all of $ids in list of sources for profile $email" } - sources.forEach { - assert(it.id.isNotEmpty()) { "Expected 'id' to be set in source account details for profile $email" } - assert(arrayOf("card", "source").contains(it.type)) { - "Unexpected source account type ${it.type} for profile $email" + sources.forEach { + assert(it.id.isNotEmpty()) { "Expected 'id' to be set in source account details for profile $email" } + assert(arrayOf("card", "source").contains(it.type)) { + "Unexpected source account type ${it.type} for profile $email" + } } + } finally { + StripePayment.deleteCustomer(email = email) } } - private fun getIdFromStripe(tokenId : String) : String { - if (tokenId.startsWith("src_")) { - return StripePayment.getCardIdForSourceId(tokenId) - } - return StripePayment.getCardIdForTokenId(tokenId) - } + @Test + fun `okhttp test - PUT source set default`() { - private fun createTokenWithStripe(client : DefaultApi) : String { - val tokenId = StripePayment.createPaymentTokenId() + val email = "purchase-${randomInt()}@test.com" + try { + createProfile(name = "Test Payment Source", email = email) - client.createSource(tokenId) + val client = clientForSubject(subject = email) - return tokenId - } + val tokenId = StripePayment.createPaymentTokenId() + val cardId = StripePayment.getCardIdForTokenId(tokenId) - private fun createSourceWithStripe(client : DefaultApi) : String { - val sourceId = StripePayment.createPaymentSourceId() + // Ties source with user profile both local and with Stripe + client.createSource(tokenId) - client.createSource(sourceId) + Thread.sleep(200) - return sourceId + val newTokenId = StripePayment.createPaymentTokenId() + val newCardId = StripePayment.getCardIdForTokenId(newTokenId) + + client.createSource(newTokenId) + + // TODO: Update to fetch the Stripe customerId from 'admin' API when ready. + val customerId = StripePayment.getCustomerIdForEmail(email) + + // Verify that original 'sourceId/card' is default. + assertEquals(cardId, StripePayment.getDefaultSourceForCustomer(customerId), + "Expected $cardId to be default source for $customerId") + + // Set new default card. + client.setDefaultSource(newCardId) + + assertEquals(newCardId, StripePayment.getDefaultSourceForCustomer(customerId), + "Expected $newCardId to be default source for $customerId") + } finally { + StripePayment.deleteCustomer(email = email) + } } @Test - fun `okhttp test - PUT source set default`() { - - StripePayment.deleteAllCustomers() + fun `okhttp test - DELETE source`() { val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Payment Source", email = email) - val client = clientForSubject(subject = email) + try { + createProfile(name = "Test Payment Source", email = email) + + val client = clientForSubject(subject = email) + + Thread.sleep(200) + + val createdIds = listOf(getCardIdForTokenFromStripe(createTokenWithStripe(client)), + createSourceWithStripe(client)) + + val deletedIds = createdIds.map { it -> deleteSourceWithStripe(client, it) } + + assert(createdIds.containsAll(deletedIds.toSet())) { + "Failed to delete one or more sources: ${createdIds.toSet() - deletedIds.toSet()}" + } + } finally { + StripePayment.deleteCustomer(email = email) + } + } + + // Helpers for source handling with Stripe. + private fun getCardIdForTokenFromStripe(id: String) : String { + if (id.startsWith("tok_")) { + return StripePayment.getCardIdForTokenId(id) + } + return id + } + + private fun createTokenWithStripe(client: DefaultApi) : String { val tokenId = StripePayment.createPaymentTokenId() - val cardId = StripePayment.getCardIdForTokenId(tokenId) - // Ties source with user profile both local and with Stripe client.createSource(tokenId) - Thread.sleep(200) + return tokenId + } - val newTokenId = StripePayment.createPaymentTokenId() - val newCardId = StripePayment.getCardIdForTokenId(newTokenId) + private fun createSourceWithStripe(client: DefaultApi) : String { + val sourceId = StripePayment.createPaymentSourceId() - client.createSource(newTokenId) + client.createSource(sourceId) - // TODO: Update to fetch the Stripe customerId from 'admin' API when ready. - val customerId = StripePayment.getCustomerIdForEmail(email) + return sourceId + } - // Verify that original 'sourceId/card' is default. - assertEquals(cardId, StripePayment.getDefaultSourceForCustomer(customerId), - "Expected $cardId to be default source for $customerId") + private fun deleteSourceWithStripe(client : DefaultApi, sourceId : String) : String { - // Set new default card. - client.setDefaultSource(newCardId) + val removedSource = client.removeSource(sourceId) - assertEquals(newCardId, StripePayment.getDefaultSourceForCustomer(customerId), - "Expected $newCardId to be default source for $customerId") + return removedSource.id } } @@ -324,103 +370,109 @@ class PurchaseTest { @Test fun `okhttp test - POST products purchase`() { - StripePayment.deleteAllCustomers() - val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Purchase User", email = email) + try { + createProfile(name = "Test Purchase User", email = email) - val client = clientForSubject(subject = email) + val client = clientForSubject(subject = email) - val balanceBefore = client.bundles.first().balance + val balanceBefore = client.bundles.first().balance - val sourceId = StripePayment.createPaymentTokenId() + val sourceId = StripePayment.createPaymentTokenId() - client.purchaseProduct("1GB_249NOK", sourceId, false) + client.purchaseProduct("1GB_249NOK", sourceId, false) - Thread.sleep(200) // wait for 200 ms for balance to be updated in db + Thread.sleep(200) // wait for 200 ms for balance to be updated in db - val balanceAfter = client.bundles.first().balance + val balanceAfter = client.bundles.first().balance - assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") + assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") - val purchaseRecords = client.purchaseHistory + val purchaseRecords = client.purchaseHistory - purchaseRecords.sortBy { it.timestamp } + purchaseRecords.sortBy { it.timestamp } - assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } - assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } + assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + } finally { + StripePayment.deleteCustomer(email = email) + } } @Test fun `okhttp test - POST products purchase using default source`() { - StripePayment.deleteAllCustomers() - val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Purchase User with Default Payment Source", email = email) + try { + createProfile(name = "Test Purchase User with Default Payment Source", email = email) - val sourceId = StripePayment.createPaymentTokenId() + val sourceId = StripePayment.createPaymentTokenId() - val client = clientForSubject(subject = email) + val client = clientForSubject(subject = email) - val paymentSource: PaymentSource = client.createSource(sourceId) + val paymentSource: PaymentSource = client.createSource(sourceId) - assertNotNull(paymentSource.id, message = "Failed to create payment source") + assertNotNull(paymentSource.id, message = "Failed to create payment source") - val balanceBefore = client.bundles.first().balance + val balanceBefore = client.bundles.first().balance - val productSku = "1GB_249NOK" + val productSku = "1GB_249NOK" - client.purchaseProduct(productSku, null, null) + client.purchaseProduct(productSku, null, null) - Thread.sleep(200) // wait for 200 ms for balance to be updated in db + Thread.sleep(200) // wait for 200 ms for balance to be updated in db - val balanceAfter = client.bundles.first().balance + val balanceAfter = client.bundles.first().balance - assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") + assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") - val purchaseRecords = client.purchaseHistory + val purchaseRecords = client.purchaseHistory - purchaseRecords.sortBy { it.timestamp } + purchaseRecords.sortBy { it.timestamp } - assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } - assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } + assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + } finally { + StripePayment.deleteCustomer(email = email) + } } @Test fun `okhttp test - POST products purchase add source then pay with it`() { - StripePayment.deleteAllCustomers() - val email = "purchase-${randomInt()}@test.com" - createProfile(name = "Test Purchase User with Default Payment Source", email = email) + try { + createProfile(name = "Test Purchase User with Default Payment Source", email = email) - val sourceId = StripePayment.createPaymentTokenId() + val sourceId = StripePayment.createPaymentTokenId() - val client = clientForSubject(subject = email) + val client = clientForSubject(subject = email) - val paymentSource: PaymentSource = client.createSource(sourceId) + val paymentSource: PaymentSource = client.createSource(sourceId) - assertNotNull(paymentSource.id, message = "Failed to create payment source") + assertNotNull(paymentSource.id, message = "Failed to create payment source") - val balanceBefore = client.subscriptionStatus.remaining + val balanceBefore = client.subscriptionStatus.remaining - val productSku = "1GB_249NOK" + val productSku = "1GB_249NOK" - client.purchaseProduct(productSku, paymentSource.id, null) + client.purchaseProduct(productSku, paymentSource.id, null) - Thread.sleep(200) // wait for 200 ms for balance to be updated in db + Thread.sleep(200) // wait for 200 ms for balance to be updated in db - val balanceAfter = client.subscriptionStatus.remaining + val balanceAfter = client.subscriptionStatus.remaining - assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") + assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") - val purchaseRecords = client.purchaseHistory + val purchaseRecords = client.purchaseHistory - purchaseRecords.sortBy { it.timestamp } + purchaseRecords.sortBy { it.timestamp } - assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } - assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } + assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + } finally { + StripePayment.deleteCustomer(email = email) + } } @Test @@ -575,4 +627,4 @@ class ReferralTest { assertEquals(listOf(freeProductForReferred), secondSubscriptionStatus.purchaseRecords.map { it.product }) } -} \ No newline at end of file +} diff --git a/admin-api/build.gradle b/admin-api/build.gradle index 75e6f1408..70f917bf0 100644 --- a/admin-api/build.gradle +++ b/admin-api/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" } diff --git a/admin-api/src/main/kotlin/org/ostelco/prime/admin/api/Resources.kt b/admin-api/src/main/kotlin/org/ostelco/prime/admin/api/Resources.kt index dd2477fc8..04714845c 100644 --- a/admin-api/src/main/kotlin/org/ostelco/prime/admin/api/Resources.kt +++ b/admin-api/src/main/kotlin/org/ostelco/prime/admin/api/Resources.kt @@ -7,6 +7,7 @@ import org.ostelco.prime.model.ProductClass import org.ostelco.prime.model.Segment import org.ostelco.prime.module.getResource import org.ostelco.prime.storage.AdminDataSource +import javax.ws.rs.DELETE import javax.ws.rs.POST import javax.ws.rs.PUT import javax.ws.rs.Path @@ -69,6 +70,9 @@ class SegmentResource { // @Path("/{segment-id}") // fun getSegment(@PathParam("segment-id") segmentId: String) = adminDataSource.getSegment(segmentId) + /** + * Create new [Segment] + */ @POST fun createSegment(segment: Segment): Response { return adminDataSource.createSegment(segment) @@ -76,6 +80,9 @@ class SegmentResource { { Response.status(Response.Status.CREATED).build() }) } + /** + * Update existing [Segment]. Replace existing subscriber list with new list. + */ @PUT @Path("/{segment-id}") fun updateSegment( @@ -94,6 +101,24 @@ class SegmentResource { { Response.ok().build() }) } + /** + * Add individual subscriber to a [Segment] + */ + @POST + @Path("/{segment-id}/subscriber/{subscriber-id}") + fun addSubscriberToSegment(segment: Segment): Response { + TODO("Vihang: Needs implementation") + } + + /** + * Add individual subscriber to a [Segment] + */ + @DELETE + @Path("/{segment-id}/subscriber/{subscriber-id}") + fun removeSubscriberFromSegment(segment: Segment): Response { + TODO("Vihang: Needs implementation") + } + // private fun toStoredSegment(segment: Segment): org.ostelco.prime.model.Segment { // return org.ostelco.prime.model.Segment( // segment.id, diff --git a/analytics-grpc-api/build.gradle b/analytics-grpc-api/build.gradle index 1a37a301e..fb2ba62f8 100644 --- a/analytics-grpc-api/build.gradle +++ b/analytics-grpc-api/build.gradle @@ -9,6 +9,7 @@ dependencies { api "io.grpc:grpc-protobuf:$grpcVersion" api "io.grpc:grpc-stub:$grpcVersion" api "io.grpc:grpc-core:$grpcVersion" + implementation 'javax.annotation:javax.annotation-api:1.3.2' } protobuf { diff --git a/analytics-grpc-api/src/main/proto/analytics.proto b/analytics-grpc-api/src/main/proto/analytics.proto index ab057ce8b..220943f57 100644 --- a/analytics-grpc-api/src/main/proto/analytics.proto +++ b/analytics-grpc-api/src/main/proto/analytics.proto @@ -24,4 +24,15 @@ message AggregatedDataTrafficInfo { google.protobuf.Timestamp timestamp = 3; string apn = 4; string mccMnc = 5; +} + +message User { + string msisdn = 1; + string apn = 2; + string mccMnc = 3; +} + +message ActiveUsersInfo { + repeated User users = 1; + google.protobuf.Timestamp timestamp = 2; } \ No newline at end of file diff --git a/analytics-grpc-api/src/main/proto/prime_metrics.proto b/analytics-grpc-api/src/main/proto/prime_metrics.proto index 70dbf355a..318e4bdfb 100644 --- a/analytics-grpc-api/src/main/proto/prime_metrics.proto +++ b/analytics-grpc-api/src/main/proto/prime_metrics.proto @@ -13,6 +13,14 @@ service OcsgwAnalyticsService { message OcsgwAnalyticsReport { uint32 activeSessions = 1; + repeated User users = 2; + bool keepAlive = 3; +} + +message User { + string msisdn = 1; + string apn = 2; + string mccMnc = 3; } message OcsgwAnalyticsReply { diff --git a/analytics-module/build.gradle b/analytics-module/build.gradle index fea4ea378..b02d49302 100644 --- a/analytics-module/build.gradle +++ b/analytics-module/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" } diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt index 40de21abb..bf63afd51 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt @@ -3,6 +3,8 @@ package org.ostelco.prime.analytics import io.grpc.stub.StreamObserver import org.ostelco.prime.analytics.PrimeMetric.ACTIVE_SESSIONS import org.ostelco.prime.analytics.metrics.CustomMetricsRegistry + +import org.ostelco.prime.analytics.publishers.ActiveUsersPublisher import org.ostelco.prime.getLogger import org.ostelco.prime.metrics.api.OcsgwAnalyticsReply import org.ostelco.prime.metrics.api.OcsgwAnalyticsReport @@ -11,7 +13,7 @@ import java.util.* /** - * Serves incoming GRPC analytcs requests. + * Serves incoming GRPC analytics requests. * * It's implemented as a subclass of [OcsServiceGrpc.OcsServiceImplBase] overriding * methods that together implements the protocol described in the analytics protobuf @@ -46,7 +48,10 @@ class AnalyticsGrpcService : OcsgwAnalyticsServiceGrpc.OcsgwAnalyticsServiceImpl * @param request provides current active session as a counter with a timestamp */ override fun onNext(request: OcsgwAnalyticsReport) { - CustomMetricsRegistry.updateMetricValue(ACTIVE_SESSIONS, request.activeSessions.toLong()) + if (!request.keepAlive) { + CustomMetricsRegistry.updateMetricValue(ACTIVE_SESSIONS, request.activeSessions.toLong()) + ActiveUsersPublisher.publish(request.usersList) + } } override fun onError(t: Throwable) { diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt index 1f0c7d0b8..0b8e35dec 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt @@ -5,6 +5,7 @@ import com.fasterxml.jackson.annotation.JsonTypeName import io.dropwizard.setup.Environment import org.hibernate.validator.constraints.NotEmpty import org.ostelco.prime.analytics.metrics.CustomMetricsRegistry +import org.ostelco.prime.analytics.publishers.ActiveUsersPublisher import org.ostelco.prime.analytics.publishers.DataConsumptionInfoPublisher import org.ostelco.prime.analytics.publishers.PurchaseInfoPublisher import org.ostelco.prime.module.PrimeModule @@ -28,6 +29,7 @@ class AnalyticsModule : PrimeModule { // dropwizard starts Analytics events publisher env.lifecycle().manage(DataConsumptionInfoPublisher) env.lifecycle().manage(PurchaseInfoPublisher) + env.lifecycle().manage(ActiveUsersPublisher) } } @@ -43,6 +45,10 @@ class AnalyticsConfig { @NotEmpty @JsonProperty("purchaseInfoTopicId") lateinit var purchaseInfoTopicId: String + + @NotEmpty + @JsonProperty("activeUsersTopicId") + lateinit var activeUsersTopicId: String } object ConfigRegistry { diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt new file mode 100644 index 000000000..204af1015 --- /dev/null +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt @@ -0,0 +1,66 @@ +package org.ostelco.prime.analytics.publishers + +import com.google.api.core.ApiFutureCallback +import com.google.api.core.ApiFutures +import com.google.api.gax.rpc.ApiException +import com.google.protobuf.ByteString +import com.google.protobuf.util.JsonFormat +import com.google.protobuf.util.Timestamps +import com.google.pubsub.v1.PubsubMessage +import org.ostelco.analytics.api.ActiveUsersInfo +import org.ostelco.prime.analytics.ConfigRegistry +import org.ostelco.prime.getLogger +import org.ostelco.prime.metrics.api.User +import org.ostelco.prime.module.getResource +import org.ostelco.prime.pseudonymizer.PseudonymizerService +import java.time.Instant + +/** + * This class publishes the active users information events to the Google Cloud Pub/Sub. + */ +object ActiveUsersPublisher : + PubSubPublisher by DelegatePubSubPublisher(topicId = ConfigRegistry.config.activeUsersTopicId) { + + private val logger by getLogger() + + private val pseudonymizerService by lazy { getResource() } + private val jsonPrinter = JsonFormat.printer().includingDefaultValueFields() + + private fun convertToJson(activeUsersInfo: ActiveUsersInfo): ByteString = + ByteString.copyFromUtf8(jsonPrinter.print(activeUsersInfo)) + + fun publish(userList: List) { + val timestamp = Instant.now().toEpochMilli() + val activeUsersInfoBuilder = ActiveUsersInfo.newBuilder().setTimestamp(Timestamps.fromMillis(timestamp)) + for (user in userList) { + val userBuilder = org.ostelco.analytics.api.User.newBuilder() + val pseudonym = pseudonymizerService.getMsisdnPseudonym(user.msisdn, timestamp).pseudonym + activeUsersInfoBuilder.addUsers(userBuilder.setApn(user.apn).setMccMnc(user.mccMnc).setMsisdn(pseudonym).build()) + } + + val pubsubMessage = PubsubMessage.newBuilder() + .setData(convertToJson(activeUsersInfoBuilder.build())) + .build() + + //schedule a message to be published, messages are automatically batched + val future = publishPubSubMessage(pubsubMessage) + + // add an asynchronous callback to handle success / failure + ApiFutures.addCallback(future, object : ApiFutureCallback { + + override fun onFailure(throwable: Throwable) { + if (throwable is ApiException) { + // details on the API exception + logger.warn("Status code: {}", throwable.statusCode.code) + logger.warn("Retrying: {}", throwable.isRetryable) + } + logger.warn("Error publishing active users list") + } + + override fun onSuccess(messageId: String) { + // Once published, returns server-assigned message ids (unique within the topic) + logger.debug(messageId) + } + }, singleThreadScheduledExecutor) + } +} diff --git a/app-notifier/build.gradle b/app-notifier/build.gradle index 94b971821..576935bca 100644 --- a/app-notifier/build.gradle +++ b/app-notifier/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" } diff --git a/auth-server/Dockerfile b/auth-server/Dockerfile index 50a1964b9..99c77878a 100644 --- a/auth-server/Dockerfile +++ b/auth-server/Dockerfile @@ -1,4 +1,4 @@ -FROM azul/zulu-openjdk:8u181-8.31.0.1 +FROM openjdk:11 MAINTAINER CSI "csi@telenordigital.com" diff --git a/auth-server/build.gradle b/auth-server/build.gradle index a1c13fb54..82c788ffc 100644 --- a/auth-server/build.gradle +++ b/auth-server/build.gradle @@ -1,7 +1,7 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "application" - id "com.github.johnrengelman.shadow" version "2.0.4" + id "com.github.johnrengelman.shadow" version "4.0.0" id "idea" } @@ -9,6 +9,9 @@ dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion" implementation "io.dropwizard:dropwizard-core:$dropwizardVersion" implementation project(":firebase-extensions") + + implementation 'javax.xml.bind:jaxb-api:2.3.0' + implementation 'javax.activation:activation:1.1.1' testImplementation "io.dropwizard:dropwizard-testing:$dropwizardVersion" testImplementation "org.jetbrains.kotlin:kotlin-test-junit:$kotlinVersion" diff --git a/bq-metrics-extractor/Dockerfile b/bq-metrics-extractor/Dockerfile index ab899b783..ed8144f51 100644 --- a/bq-metrics-extractor/Dockerfile +++ b/bq-metrics-extractor/Dockerfile @@ -1,4 +1,4 @@ -FROM azul/zulu-openjdk:8u181-8.31.0.1 +FROM openjdk:11 MAINTAINER CSI "csi@telenordigital.com" diff --git a/bq-metrics-extractor/Dockerfile.test b/bq-metrics-extractor/Dockerfile.test index a9e7587fe..fda30d0b5 100644 --- a/bq-metrics-extractor/Dockerfile.test +++ b/bq-metrics-extractor/Dockerfile.test @@ -1,4 +1,4 @@ -FROM azul/zulu-openjdk:8u181-8.31.0.1 +FROM openjdk:11 MAINTAINER CSI "csi@telenordigital.com" diff --git a/bq-metrics-extractor/build.gradle b/bq-metrics-extractor/build.gradle index a5155aa67..c01398491 100644 --- a/bq-metrics-extractor/build.gradle +++ b/bq-metrics-extractor/build.gradle @@ -1,7 +1,7 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "application" - id "com.github.johnrengelman.shadow" version "2.0.4" + id "com.github.johnrengelman.shadow" version "4.0.0" id "idea" } diff --git a/bq-metrics-extractor/config/config.yaml b/bq-metrics-extractor/config/config.yaml index f92f105ce..4434c9273 100644 --- a/bq-metrics-extractor/config/config.yaml +++ b/bq-metrics-extractor/config/config.yaml @@ -92,6 +92,51 @@ bqmetrics: WHERE timestamp >= TIMESTAMP_SUB(TIMESTAMP_TRUNC(CURRENT_TIMESTAMP(), DAY), INTERVAL 1 DAY) AND timestamp < TIMESTAMP_TRUNC(CURRENT_TIMESTAMP(), DAY) ), 0) as count + - type: gauge + name: total_data_used_today_local_loltel_test + help: Total data used today local loltel-test + resultColumn: count + sql: > + SELECT COALESCE ( + (SELECT sum(bucketBytes) AS count FROM `pantel-2decb.data_consumption.raw_consumption` + WHERE timestamp >= TIMESTAMP_TRUNC(CURRENT_TIMESTAMP(), DAY) + AND apn = "loltel-test" + AND mccMnc = "24201"), 0) as count + - type: gauge + name: total_data_used_yesterday_local_lotlel_test + help: Total data used yesterday local loltel-test + resultColumn: count + sql: > + SELECT COALESCE ( + ( SELECT sum(bucketBytes) AS count FROM `pantel-2decb.data_consumption.raw_consumption` + WHERE timestamp >= TIMESTAMP_SUB(TIMESTAMP_TRUNC(CURRENT_TIMESTAMP(), DAY), INTERVAL 1 DAY) + AND timestamp < TIMESTAMP_TRUNC(CURRENT_TIMESTAMP(), DAY) + AND apn = "loltel-test" + AND mccMnc = "24201"), 0) as count + + + - type: gauge + name: total_data_used_today_roaming_loltel_test + help: Total data used today roaming loltel-test + resultColumn: count + sql: > + SELECT COALESCE ( + (SELECT sum(bucketBytes) AS count FROM `pantel-2decb.data_consumption.raw_consumption` + WHERE timestamp >= TIMESTAMP_TRUNC(CURRENT_TIMESTAMP(), DAY) + AND apn = "loltel-test" + AND mccMnc != "24201"), 0) as count + - type: gauge + name: total_data_used_yesterday_roaming_lotlel_test + help: Total data used yesterday roaming loltel-test + resultColumn: count + sql: > + SELECT COALESCE ( + ( SELECT sum(bucketBytes) AS count FROM `pantel-2decb.data_consumption.raw_consumption` + WHERE timestamp >= TIMESTAMP_SUB(TIMESTAMP_TRUNC(CURRENT_TIMESTAMP(), DAY), INTERVAL 1 DAY) + AND timestamp < TIMESTAMP_TRUNC(CURRENT_TIMESTAMP(), DAY) + AND apn = "loltel-test" + AND mccMnc != "24201"), 0) as count + - type: gauge name: revenue_today help: Revenue generated today diff --git a/build.gradle b/build.gradle index e319235f3..0211da753 100644 --- a/build.gradle +++ b/build.gradle @@ -19,24 +19,30 @@ allprojects { maven { url = "https://maven.repository.redhat.com/ga/" } maven { url = "http://clojars.org/repo/" } } + + jacoco { + toolVersion = "0.8.2" + } } subprojects { - sourceCompatibility = 1.8 - targetCompatibility = 1.8 + sourceCompatibility = 11 + targetCompatibility = 11 tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } ext { - kotlinVersion = "1.2.70" + kotlinVersion = "1.2.71" dropwizardVersion = "1.3.5" - googleCloudVersion = "1.45.0" + googleCloudVersion = "1.46.0" jacksonVersion = "2.9.7" - stripeVersion = "6.12.0" + stripeVersion = "7.0.0" guavaVersion = "26.0-jre" + junit5Version = "5.3.1" assertJVersion = "3.11.1" mockitoVersion = "2.22.0" firebaseVersion = "6.5.0" + beamVersion = "2.7.0" // Keeping it version 1.15.0 to be consistent with grpc via PubSub client lib // Keeping it version 1.15.0 to be consistent with netty via Firebase lib grpcVersion = "1.15.0" diff --git a/certs/dev.ostelco.org/.gitignore b/certs/dev.ostelco.org/.gitignore deleted file mode 100644 index 47c805dfe..000000000 --- a/certs/dev.ostelco.org/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -nginx.crt -nginx.key \ No newline at end of file diff --git a/certs/metrics.ostelco.org/.gitignore b/certs/metrics.ostelco.org/.gitignore deleted file mode 100644 index 47c805dfe..000000000 --- a/certs/metrics.ostelco.org/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -nginx.crt -nginx.key \ No newline at end of file diff --git a/certs/ocs.ostelco.org/.gitignore b/certs/ocs.ostelco.org/.gitignore deleted file mode 100644 index 47c805dfe..000000000 --- a/certs/ocs.ostelco.org/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -nginx.crt -nginx.key \ No newline at end of file diff --git a/client-api/build.gradle b/client-api/build.gradle index f0e158a68..b0519c812 100644 --- a/client-api/build.gradle +++ b/client-api/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" } @@ -14,6 +14,9 @@ dependencies { implementation "com.google.guava:guava:$guavaVersion" implementation 'io.jsonwebtoken:jjwt:0.9.1' + implementation 'javax.xml.bind:jaxb-api:2.3.0' + implementation 'javax.activation:activation:1.1.1' + testImplementation "io.dropwizard:dropwizard-client:$dropwizardVersion" testImplementation "io.dropwizard:dropwizard-testing:$dropwizardVersion" testImplementation "com.fasterxml.jackson.module:jackson-module-kotlin:$jacksonVersion" diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/metrics/Metrics.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/metrics/Metrics.kt index 9bd73a7c8..e61d240a9 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/metrics/Metrics.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/metrics/Metrics.kt @@ -3,7 +3,6 @@ package org.ostelco.prime.client.api.metrics import org.ostelco.prime.analytics.AnalyticsService import org.ostelco.prime.analytics.PrimeMetric.TOTAL_USERS import org.ostelco.prime.analytics.PrimeMetric.USERS_ACQUIRED_THROUGH_REFERRALS -import org.ostelco.prime.analytics.PrimeMetric.USERS_PAID_AT_LEAST_ONCE import org.ostelco.prime.module.getResource import org.ostelco.prime.storage.AdminDataSource @@ -13,7 +12,6 @@ val adminStore: AdminDataSource = getResource() fun reportMetricsAtStartUp() { analyticsService.reportMetric(TOTAL_USERS, adminStore.getSubscriberCount()) analyticsService.reportMetric(USERS_ACQUIRED_THROUGH_REFERRALS, adminStore.getReferredSubscriberCount()) - analyticsService.reportMetric(USERS_PAID_AT_LEAST_ONCE, adminStore.getPaidSubscriberCount()) } fun updateMetricsOnNewSubscriber() { diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/resources/PaymentResource.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/resources/PaymentResource.kt index 737bb3440..ff050e8d7 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/resources/PaymentResource.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/resources/PaymentResource.kt @@ -5,12 +5,7 @@ import org.ostelco.prime.client.api.auth.AccessTokenPrincipal import org.ostelco.prime.client.api.store.SubscriberDAO import org.ostelco.prime.getLogger import javax.validation.constraints.NotNull -import javax.ws.rs.GET -import javax.ws.rs.POST -import javax.ws.rs.PUT -import javax.ws.rs.Path -import javax.ws.rs.Produces -import javax.ws.rs.QueryParam +import javax.ws.rs.* import javax.ws.rs.core.Response /** @@ -72,4 +67,22 @@ class PaymentResource(private val dao: SubscriberDAO) { { sourceInfo -> Response.status(Response.Status.OK).entity(sourceInfo)} ).build() } + + @DELETE + @Produces("application/json") + fun removeSource(@Auth token: AccessTokenPrincipal?, + @NotNull + @QueryParam("sourceId") + sourceId: String): Response { + if (token == null) { + return Response.status(Response.Status.UNAUTHORIZED) + .build() + } + + return dao.removeSource(token.name, sourceId) + .fold( + { apiError -> Response.status(apiError.status).entity(asJson(apiError)) }, + { sourceInfo -> Response.status(Response.Status.OK).entity(sourceInfo)} + ).build() + } } diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAO.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAO.kt index 11b750282..39f377634 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAO.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAO.kt @@ -65,6 +65,8 @@ interface SubscriberDAO { fun listSources(subscriberId: String): Either> + fun removeSource(subscriberId: String, sourceId: String): Either + companion object { /** @@ -73,7 +75,8 @@ interface SubscriberDAO { fun isValidProfile(profile: Subscriber?): Boolean { return (profile != null && !profile.name.isEmpty() - && !profile.email.isEmpty()) + && !profile.email.isEmpty() + && !profile.country.isEmpty()) } /** diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt index 3280f5e23..a57c167b4 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt @@ -3,11 +3,6 @@ package org.ostelco.prime.client.api.store import arrow.core.Either import arrow.core.flatMap import org.ostelco.prime.analytics.AnalyticsService -import org.ostelco.prime.analytics.PrimeMetric.REVENUE -import org.ostelco.prime.client.api.metrics.updateMetricsOnNewSubscriber -import org.ostelco.prime.client.api.model.Consent -import org.ostelco.prime.client.api.model.Person -import org.ostelco.prime.client.api.model.SubscriptionStatus import org.ostelco.prime.apierror.ApiError import org.ostelco.prime.apierror.ApiErrorCode import org.ostelco.prime.apierror.BadGatewayError @@ -16,6 +11,10 @@ import org.ostelco.prime.apierror.InsufficientStorageError import org.ostelco.prime.apierror.NotFoundError import org.ostelco.prime.apierror.mapPaymentErrorToApiError import org.ostelco.prime.apierror.mapStorageErrorToApiError +import org.ostelco.prime.client.api.metrics.updateMetricsOnNewSubscriber +import org.ostelco.prime.client.api.model.Consent +import org.ostelco.prime.client.api.model.Person +import org.ostelco.prime.client.api.model.SubscriptionStatus import org.ostelco.prime.getLogger import org.ostelco.prime.model.ActivePseudonyms import org.ostelco.prime.model.ApplicationToken @@ -66,12 +65,12 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu override fun createProfile(subscriberId: String, profile: Subscriber, referredBy: String?): Either { if (!SubscriberDAO.isValidProfile(profile)) { logger.error("Failed to create profile. Invalid profile.") - return Either.left(BadRequestError("Incomplete profile description. Profile must contain name and email", ApiErrorCode.FAILED_TO_CREATE_PAYMENT_PROFILE)) + return Either.left(BadRequestError("Incomplete profile description. Profile must contain name and email", ApiErrorCode.FAILED_TO_CREATE_PROFILE)) } return try { storage.addSubscriber(profile, referredBy) .mapLeft { - mapStorageErrorToApiError("Failed to create profile.", ApiErrorCode.FAILED_TO_CREATE_PAYMENT_PROFILE, it) + mapStorageErrorToApiError("Failed to create profile.", ApiErrorCode.FAILED_TO_CREATE_PROFILE, it) } .flatMap { updateMetricsOnNewSubscriber() @@ -79,7 +78,7 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu } } catch (e: Exception) { logger.error("Failed to create profile for subscriberId $subscriberId", e) - Either.left(BadGatewayError("Failed to create profile", ApiErrorCode.FAILED_TO_CREATE_PAYMENT_PROFILE)) + Either.left(BadGatewayError("Failed to create profile", ApiErrorCode.FAILED_TO_CREATE_PROFILE)) } } @@ -231,13 +230,13 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu purchaseRecord = purchaseRecord, subscriberId = subscriberId, status = "success") - //TODO: Handle errors (when it becomes available) - ocsSubscriberService.topup(subscriberId, sku) - // TODO vihang: handle currency conversion - analyticsReporter.reportMetric(REVENUE, product.price.amount.toLong()) Either.right(Unit) } } + .flatMap { + ocsSubscriberService.topup(subscriberId, sku) + .mapLeft { errorReason -> BadGatewayError(description = errorReason, errorCode = ApiErrorCode.FAILED_TO_PURCHASE_PRODUCT) } + } } override fun purchaseProduct( @@ -339,16 +338,19 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu override fun listSources(subscriberId: String): Either> { return paymentProcessor.getPaymentProfile(subscriberId) - .fold( - { - paymentProcessor.createPaymentProfile(subscriberId) - .mapLeft { error -> mapPaymentErrorToApiError(error.description, ApiErrorCode.FAILED_TO_FETCH_PAYMENT_SOURCES_LIST, error) } - }, - { profileInfo -> Either.right(profileInfo) } - ) + .mapLeft { error -> mapPaymentErrorToApiError(error.description, ApiErrorCode.FAILED_TO_FETCH_PAYMENT_SOURCES_LIST, error) } .flatMap { profileInfo -> paymentProcessor.getSavedSources(profileInfo.id) .mapLeft { mapPaymentErrorToApiError("Failed to list sources", ApiErrorCode.FAILED_TO_FETCH_PAYMENT_SOURCES_LIST, it) } } } + + override fun removeSource(subscriberId: String, sourceId: String): Either { + return paymentProcessor.getPaymentProfile(subscriberId) + .mapLeft { error -> mapPaymentErrorToApiError(error.description, ApiErrorCode.FAILED_TO_REMOVE_PAYMENT_SOURCE, error) } + .flatMap { profileInfo -> + paymentProcessor.removeSource(profileInfo.id, sourceId) + .mapLeft { mapPaymentErrorToApiError("Failed to remove payment source", ApiErrorCode.FAILED_TO_REMOVE_PAYMENT_SOURCE, it) } + } + } } diff --git a/dataflow-pipelines/Dockerfile b/dataflow-pipelines/Dockerfile index 4827fd238..d0cd99b27 100644 --- a/dataflow-pipelines/Dockerfile +++ b/dataflow-pipelines/Dockerfile @@ -1,4 +1,4 @@ -FROM azul/zulu-openjdk:8u181-8.31.0.1 +FROM openjdk:11 MAINTAINER CSI "csi@telenordigital.com" diff --git a/dataflow-pipelines/build.gradle b/dataflow-pipelines/build.gradle index 5bb20659a..54b887a52 100644 --- a/dataflow-pipelines/build.gradle +++ b/dataflow-pipelines/build.gradle @@ -1,7 +1,7 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "application" - id "com.github.johnrengelman.shadow" version "2.0.4" + id "com.github.johnrengelman.shadow" version "4.0.0" id "idea" } @@ -13,13 +13,16 @@ dependencies { implementation "com.google.cloud:google-cloud-pubsub:$googleCloudVersion" - implementation 'com.google.cloud.dataflow:google-cloud-dataflow-java-sdk-all:2.5.0' - runtimeOnly 'org.apache.beam:beam-runners-google-cloud-dataflow-java:2.5.0' + implementation "org.apache.beam:beam-sdks-java-core:$beamVersion" + implementation "org.apache.beam:beam-runners-google-cloud-dataflow-java:$beamVersion" implementation 'ch.qos.logback:logback-classic:1.2.3' - testImplementation "org.jetbrains.kotlin:kotlin-test-junit:$kotlinVersion" testRuntimeOnly 'org.hamcrest:hamcrest-all:1.3' + testRuntimeOnly "org.apache.beam:beam-runners-direct-java:$beamVersion" + + testImplementation "org.junit.jupiter:junit-jupiter-api:$junit5Version" + testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:$junit5Version" } shadowJar { @@ -29,4 +32,15 @@ shadowJar { version = null } +test { + // native support to Junit5 in Gradle 4.6+ + useJUnitPlatform { + includeEngines 'junit-jupiter' + } + testLogging { + exceptionFormat = 'full' + events "PASSED", "FAILED", "SKIPPED" + } +} + apply from: '../jacoco.gradle' \ No newline at end of file diff --git a/dataflow-pipelines/src/main/resources/table_schema.ddl b/dataflow-pipelines/src/main/resources/table_schema.ddl index 2e6d229b3..5c13066d3 100644 --- a/dataflow-pipelines/src/main/resources/table_schema.ddl +++ b/dataflow-pipelines/src/main/resources/table_schema.ddl @@ -3,7 +3,9 @@ CREATE TABLE IF NOT EXISTS ( msisdn STRING NOT NULL, bytes INT64 NOT NULL, - timestamp TIMESTAMP NOT NULL + timestamp TIMESTAMP NOT NULL, + apn STRING NOT NULL, + mccMnc STRING NOT NULL ) PARTITION BY DATE(timestamp); @@ -14,6 +16,8 @@ CREATE TABLE IF NOT EXISTS msisdn STRING NOT NULL, bucketBytes INT64 NOT NULL, bundleBytes INT64 NOT NULL, - timestamp TIMESTAMP NOT NULL + timestamp TIMESTAMP NOT NULL, + apn STRING NOT NULL, + mccMnc STRING NOT NULL ) PARTITION BY DATE(timestamp); \ No newline at end of file diff --git a/dataflow-pipelines/src/test/kotlin/org/ostelco/dataflow/pipelines/ConsumptionPerMsisdnTest.kt b/dataflow-pipelines/src/test/kotlin/org/ostelco/dataflow/pipelines/ConsumptionPerMsisdnTest.kt index 0a785c668..0ac637e44 100644 --- a/dataflow-pipelines/src/test/kotlin/org/ostelco/dataflow/pipelines/ConsumptionPerMsisdnTest.kt +++ b/dataflow-pipelines/src/test/kotlin/org/ostelco/dataflow/pipelines/ConsumptionPerMsisdnTest.kt @@ -12,6 +12,8 @@ import org.joda.time.Instant import org.junit.Rule import org.junit.Test import org.junit.experimental.categories.Category +import org.junit.jupiter.api.condition.EnabledOnJre +import org.junit.jupiter.api.condition.JRE.JAVA_8 import org.ostelco.analytics.api.AggregatedDataTrafficInfo import org.ostelco.analytics.api.DataTrafficInfo import org.ostelco.dataflow.pipelines.definitions.consumptionPerMsisdn @@ -19,11 +21,11 @@ import org.ostelco.dataflow.pipelines.definitions.consumptionPerMsisdn class ConsumptionPerMsisdnTest { @Rule - @Transient @JvmField val pipeline: TestPipeline? = TestPipeline.create() @Test + @EnabledOnJre(JAVA_8) @Category(NeedsRunner::class) fun testPipeline() { diff --git a/diameter-stack/build.gradle b/diameter-stack/build.gradle index 9060d9b76..4b2846403 100644 --- a/diameter-stack/build.gradle +++ b/diameter-stack/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" id "signing" id "maven" diff --git a/diameter-stack/src/main/kotlin/org/ostelco/diameter/model/Model.kt b/diameter-stack/src/main/kotlin/org/ostelco/diameter/model/Model.kt index 5469f6185..bd54c5a0b 100644 --- a/diameter-stack/src/main/kotlin/org/ostelco/diameter/model/Model.kt +++ b/diameter-stack/src/main/kotlin/org/ostelco/diameter/model/Model.kt @@ -210,4 +210,6 @@ enum class UserEquipmentInfoType { data class SessionContext( val sessionId: String, val originHost: String, - val originRealm: String) \ No newline at end of file + val originRealm: String, + val apn: String, + val mccMnc: String) \ No newline at end of file diff --git a/diameter-test/build.gradle b/diameter-test/build.gradle index 1e33e6c78..c6d970380 100644 --- a/diameter-test/build.gradle +++ b/diameter-test/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" id "signing" id "maven" diff --git a/docker-compose.override.yaml b/docker-compose.override.yaml index 16b2785f0..f920554c9 100644 --- a/docker-compose.override.yaml +++ b/docker-compose.override.yaml @@ -114,7 +114,7 @@ services: datastore-emulator: container_name: datastore-emulator - image: google/cloud-sdk:206.0.0 + image: google/cloud-sdk:218.0.0 expose: - "8081" environment: diff --git a/exporter/Dockerfile b/exporter/Dockerfile index 9e003a63c..471f1ecb7 100644 --- a/exporter/Dockerfile +++ b/exporter/Dockerfile @@ -18,9 +18,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY script/idle.sh /idle.sh COPY script/export_data.sh /export_data.sh COPY script/delete_export_data.sh /delete_export_data.sh +COPY script/map_subscribers.sh /map_subscribers.sh +COPY script/subscriber-schema.json /subscriber-schema.json RUN chmod +x /idle.sh RUN chmod +x /export_data.sh RUN chmod +x /delete_export_data.sh +RUN chmod +x /map_subscribers.sh CMD ["/idle.sh"] \ No newline at end of file diff --git a/exporter/README.md b/exporter/README.md index cd8e306bc..bd30f12b5 100644 --- a/exporter/README.md +++ b/exporter/README.md @@ -1,4 +1,4 @@ -# Exporter +## Exporter This contains a set of scripts to generate the data for analayis. The export script `export_data.sh` creates a new big query table with a new uuid which maps the pseudonyms to @@ -31,8 +31,23 @@ kubectl exec -it -- /bin/bash # Run exporter from the above shell /export_data.sh +# This results in 3 csv files in GCS +1) Data consumption Records: gs://pantel-2decb-dataconsumption-export/.csv +2) Purchase Records: gs://pantel-2decb-dataconsumption-export/-purchases.csv +3) Subscriber to MSISDN mappings: gs://pantel-2decb-dataconsumption-export/-sub2msisdn.csv + +# Run subsciber reverse lookup from the above shell +/map_subscribers.sh + +# Delete all tables and files for an export +/delete_export_data.sh # Delete deployment kubectl delete deployment exporter -``` \ No newline at end of file +``` + + +## How to get data from the exporter + + .... tbd \ No newline at end of file diff --git a/exporter/script/delete_export_data.sh b/exporter/script/delete_export_data.sh index 2d59e7e2b..740c99fa9 100644 --- a/exporter/script/delete_export_data.sh +++ b/exporter/script/delete_export_data.sh @@ -19,6 +19,9 @@ csvfile=$projectId-dataconsumption-export/$exportId.csv purchasesCsvfile=$projectId-dataconsumption-export/$exportId-purchases.csv sub2msisdnCsvfile=$projectId-dataconsumption-export/$exportId-sub2msisdn.csv +inputSubscriberTable=exported_pseudonyms.${exportId}_pseudo_subscriber +outputSubscriberTable=exported_pseudonyms.${exportId}_clear_subscriber + echo "Cleaning all data for export $exportId" echo "Deleting Table $msisdnPseudonymsTable" bq rm -f -t $msisdnPseudonymsTable @@ -35,6 +38,12 @@ bq rm -f -t $dataConsumptionTable echo "Deleting Table $purchaseRecordsTable" bq rm -f -t $purchaseRecordsTable +echo "Deleting Table $inputSubscriberTable" +bq rm -f -t $inputSubscriberTable + +echo "Deleting Table $outputSubscriberTable" +bq rm -f -t $outputSubscriberTable + echo "Deleting csv gs://$csvfile" gsutil rm gs://$csvfile diff --git a/exporter/script/map_subscribers.sh b/exporter/script/map_subscribers.sh new file mode 100644 index 000000000..98400bf9f --- /dev/null +++ b/exporter/script/map_subscribers.sh @@ -0,0 +1,50 @@ +#!/bin/bash +#set -x + +exportId=$1 +if [ -z "$1" ]; then + echo "To convert subscribers, specify the id of the export operation" + exit +fi +exportId=${exportId//-} +exportId=${exportId,,} +projectId=pantel-2decb + +csvfile=$projectId-dataconsumption-export/${exportId}-resultsegment-pseudoanonymized.csv +outputCsvfile=$projectId-dataconsumption-export/${exportId}-resultsegment-cleartext.csv +inputSubscriberTable=exported_pseudonyms.${exportId}_pseudo_subscriber +subscriberPseudonymsTable=exported_pseudonyms.${exportId}_subscriber +outputSubscriberTable=exported_pseudonyms.${exportId}_clear_subscriber + + +echo "Importing data from csv $csvfile" +bq --location=EU load --replace --source_format=CSV $projectId:$inputSubscriberTable gs://$csvfile /subscriber-schema.json +echo "Exported data to $inputSubscriberTable" + +echo "Creating table $outputSubscriberTable" +# SQL for joining pseudonym & hourly consumption tables. +read -r -d '' sqlForJoin << EOM +CREATE TEMP FUNCTION URLDECODE(url STRING) AS (( + SELECT SAFE_CONVERT_BYTES_TO_STRING( + ARRAY_TO_STRING(ARRAY_AGG( + IF(STARTS_WITH(y, '%'), FROM_HEX(SUBSTR(y, 2)), CAST(y AS BYTES)) ORDER BY i + ), b'')) + FROM UNNEST(REGEXP_EXTRACT_ALL(url, r"%[0-9a-fA-F]{2}|[^%]+")) AS y WITH OFFSET AS i +)); + +SELECT + DISTINCT(sub.subscriberId) as pseudoId, URLDECODE(ps.subscriberId) as subscriberId +FROM + \`$inputSubscriberTable\` as sub +JOIN + \`$subscriberPseudonymsTable\` as ps +ON ps.pseudoid = sub.subscriberId +EOM + +# Run the query using bq & dump results to the new table +bq --location=EU --format=none query --destination_table $outputSubscriberTable --replace --use_legacy_sql=false $sqlForJoin +echo "Created table $outputSubscriberTable" + +echo "Exporting data to csv $outputCsvfile" +bq --location=EU extract --destination_format=CSV $outputSubscriberTable gs://$outputCsvfile +echo "Exported data to gs://$outputCsvfile" diff --git a/exporter/script/subscriber-schema.json b/exporter/script/subscriber-schema.json new file mode 100644 index 000000000..e5ab3cbce --- /dev/null +++ b/exporter/script/subscriber-schema.json @@ -0,0 +1,7 @@ +[ + { + "mode": "REQUIRED", + "name": "subscriberId", + "type": "STRING" + } +] diff --git a/ext-auth-provider/Dockerfile b/ext-auth-provider/Dockerfile index 78e784b6d..a69b2b1a3 100644 --- a/ext-auth-provider/Dockerfile +++ b/ext-auth-provider/Dockerfile @@ -1,4 +1,4 @@ -FROM azul/zulu-openjdk:8u181-8.31.0.1 +FROM openjdk:11 MAINTAINER CSI "csi@telenordigital.com" diff --git a/ext-auth-provider/build.gradle b/ext-auth-provider/build.gradle index 225fb0949..2917c7371 100644 --- a/ext-auth-provider/build.gradle +++ b/ext-auth-provider/build.gradle @@ -1,7 +1,7 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "application" - id "com.github.johnrengelman.shadow" version "2.0.4" + id "com.github.johnrengelman.shadow" version "4.0.0" } dependencies { @@ -10,6 +10,9 @@ dependencies { implementation "io.dropwizard:dropwizard-core:$dropwizardVersion" implementation 'io.jsonwebtoken:jjwt:0.9.1' + implementation 'javax.xml.bind:jaxb-api:2.3.0' + implementation 'javax.activation:activation:1.1.1' + testImplementation "io.dropwizard:dropwizard-testing:$dropwizardVersion" testImplementation "org.jetbrains.kotlin:kotlin-test:$kotlinVersion" testImplementation "org.jetbrains.kotlin:kotlin-test-junit:$kotlinVersion" diff --git a/firebase-extensions/build.gradle b/firebase-extensions/build.gradle index d741c6479..c39c5cc27 100644 --- a/firebase-extensions/build.gradle +++ b/firebase-extensions/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" } diff --git a/firebase-store/build.gradle b/firebase-store/build.gradle index 14a377823..3c30ffebe 100644 --- a/firebase-store/build.gradle +++ b/firebase-store/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" } diff --git a/model/build.gradle b/model/build.gradle index 3c2a0a6cd..73d0f5aef 100644 --- a/model/build.gradle +++ b/model/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" } diff --git a/neo4j-store/build.gradle b/neo4j-store/build.gradle index 70ed8509f..a9161fc15 100644 --- a/neo4j-store/build.gradle +++ b/neo4j-store/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" } diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt index d9c3ea8ce..18ad1be2f 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jModule.kt @@ -51,16 +51,22 @@ fun initDatabase() { price = Price(0, "NOK"), properties = mapOf("noOfBytes" to "1_000_000_000"))) - val segment = Segment(id = "all") - Neo4jStoreSingleton.createSegment(segment) + val segments = listOf( + Segment(id = getSegmentNameFromCountryCode("NO")), + Segment(id = getSegmentNameFromCountryCode("SG")) + ) + segments.map { Neo4jStoreSingleton.createSegment(it) } val offer = Offer( id = "default_offer", - segments = listOf("all"), + segments = listOf(getSegmentNameFromCountryCode("NO")), products = listOf("1GB_249NOK", "2GB_299NOK", "3GB_349NOK", "5GB_399NOK")) Neo4jStoreSingleton.createOffer(offer) } +// Helper for naming of default segments based on country code. +fun getSegmentNameFromCountryCode(countryCode: String) : String = "country-$countryCode".toLowerCase() + class Config { lateinit var host: String lateinit var protocol: String diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index 733f9b63c..1dab69abd 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -1,12 +1,10 @@ package org.ostelco.prime.storage.graph import arrow.core.Either -import arrow.core.Tuple4 +import arrow.core.Tuple3 import arrow.core.flatMap import org.neo4j.driver.v1.Transaction import org.ostelco.prime.analytics.AnalyticsService -import org.ostelco.prime.analytics.PrimeMetric.REVENUE -import org.ostelco.prime.analytics.PrimeMetric.USERS_PAID_AT_LEAST_ONCE import org.ostelco.prime.getLogger import org.ostelco.prime.model.Bundle import org.ostelco.prime.model.Offer @@ -152,6 +150,23 @@ object Neo4jStoreSingleton : GraphStore { val bundleId = subscriber.id val either = subscriberStore.create(subscriber, transaction) + .flatMap { + subscriberToSegmentStore + .create(subscriber.id, + getSegmentNameFromCountryCode(subscriber.country), + transaction) + .mapLeft { storeError -> + if (storeError is NotFoundError && storeError.type == segmentEntity.name) { + ValidationError( + type = subscriberEntity.name, + id = subscriber.id, + message = "Unsupported country: ${subscriber.country}") + } else { + storeError + } + } + } + if (referredBy != null) { // Give 1 GB if subscriber is referred either @@ -190,7 +205,6 @@ object Neo4jStoreSingleton : GraphStore { Either.right(Unit) } }.flatMap { subscriberToBundleStore.create(subscriber.id, bundleId, transaction) } - .flatMap { subscriberToSegmentStore.create(subscriber.id, "all", transaction) } .ifFailedThenRollback(transaction) } // << END @@ -341,7 +355,7 @@ object Neo4jStoreSingleton : GraphStore { sourceId: String?, saveCard: Boolean): Either = writeTransaction { - val result = getProduct(subscriberId, sku, transaction) + getProduct(subscriberId, sku, transaction) // If we can't find the product, return not-found .mapLeft { org.ostelco.prime.paymentprocessor.core.NotFoundError("Product unavailable") } .flatMap { product: Product -> @@ -351,40 +365,52 @@ object Neo4jStoreSingleton : GraphStore { { paymentProcessor.createPaymentProfile(subscriberId) }, { profileInfo -> Either.right(profileInfo) } ) - .map { profileInfo -> Pair(product, profileInfo) } + .map { profileInfo -> Pair(product, profileInfo.id) } } - .flatMap { (product, profileInfo) -> + .flatMap { (product, paymentCustomerId) -> // Add payment source if (sourceId != null) { - paymentProcessor.getSavedSources(profileInfo.id) - .fold( - { - Either.left(org.ostelco.prime.paymentprocessor.core.BadGatewayError("Failed to fetch sources for user", it.description)) - }, - { - var linkedSource = sourceId - if (!it.any{ sourceDetailsInfo -> sourceDetailsInfo.id == sourceId }) { - paymentProcessor.addSource(profileInfo.id, sourceId).map { sourceInfo -> linkedSource = sourceInfo.id } - } - Either.right(Triple(product,profileInfo, linkedSource)) - } - ) + // First fetch all existing saved sources + paymentProcessor.getSavedSources(paymentCustomerId) + .fold( + { + Either.left(org.ostelco.prime.paymentprocessor.core.BadGatewayError("Failed to fetch sources for user", it.description)) + }, + { + // If the sourceId is not found in existing list of saved sources, + // then save the source + if (!it.any { sourceDetailsInfo -> sourceDetailsInfo.id == sourceId }) { + paymentProcessor.addSource(paymentCustomerId, sourceId) + // TODO payment: Should we remove the sourceId for saveCard == false even when captureCharge has failed? + // For success case, saved source is removed after "capture charge" is saveCard == false. + // Making sure same happens even for failure case by linking reversal action to transaction + .finallyDo(transaction) { _ -> removePaymentSource(saveCard, paymentCustomerId, sourceId) } + .map { sourceInfo -> Triple(product, paymentCustomerId, sourceInfo.id) } + } else { + Either.right(Triple(product, paymentCustomerId, sourceId)) + } + } + ) } else { - Either.right(Triple(product, profileInfo, null)) + Either.right(Triple(product, paymentCustomerId, null)) } } - .flatMap { (product, profileInfo, savedSourceId) -> + .flatMap { (product, paymentCustomerId, sourceId) -> // Authorize stripe charge for this purchase val price = product.price //TODO: If later steps fail, then refund the authorized charge - paymentProcessor.authorizeCharge(profileInfo.id, savedSourceId, price.amount, price.currency) + paymentProcessor.authorizeCharge(paymentCustomerId, sourceId, price.amount, price.currency) .mapLeft { apiError -> - logger.error("failed to authorize purchase for customerId ${profileInfo.id}, sourceId $savedSourceId, sku $sku") + logger.error("failed to authorize purchase for paymentCustomerId $paymentCustomerId, sourceId $sourceId, sku $sku") apiError } - .map { chargeId -> Tuple4(profileInfo, savedSourceId, chargeId, product) } + .linkReversalActionToTransaction(transaction) { chargeId -> + paymentProcessor.refundCharge(chargeId) + logger.error("failed to refund charge for paymentCustomerId $paymentCustomerId, chargeId $chargeId. Fix this in Stripe dashboard") + } + .map { chargeId -> Tuple3(product, paymentCustomerId, chargeId) } } - .flatMap { (profileInfo, savedSourceId, chargeId, product) -> + .flatMap { (product, paymentCustomerId, chargeId) -> val purchaseRecord = PurchaseRecord( id = chargeId, product = product, @@ -393,51 +419,54 @@ object Neo4jStoreSingleton : GraphStore { // Create purchase record createPurchaseRecordRelation(subscriberId, purchaseRecord, transaction) .mapLeft { storeError -> - paymentProcessor.refundCharge(chargeId) - logger.error("failed to save purchase record, for customerId ${profileInfo.id}, chargeId $chargeId, payment will be unclaimed in Stripe") + logger.error("failed to save purchase record, for paymentCustomerId $paymentCustomerId, chargeId $chargeId, payment will be unclaimed in Stripe") BadGatewayError(storeError.message) } - // Notify OCS .flatMap { //TODO: While aborting transactions, send a record with "reverted" status analyticsReporter.reportPurchaseInfo( purchaseRecord = purchaseRecord, subscriberId = subscriberId, status = "success") - //TODO vihang: Handle errors (when it becomes available) - ocs.topup(subscriberId, sku) - // TODO vihang: handle currency conversion - analyticsReporter.reportMetric(REVENUE, product.price.amount.toLong()) - analyticsReporter.reportMetric(USERS_PAID_AT_LEAST_ONCE, getPaidSubscriberCount(transaction)) - Either.right(Tuple4(profileInfo, savedSourceId, chargeId, product)) + + Either.right(Tuple3(product, paymentCustomerId, chargeId)) } } - .mapLeft { error -> - transaction.failure() - error + .flatMap { (product, paymentCustomerId, chargeId) -> + // Notify OCS + ocs.topup(subscriberId, sku) + .bimap({ BadGatewayError(description = "Failed to perform topup", externalErrorMessage = it) }, + { Tuple3(product, paymentCustomerId, chargeId) }) } + .map { (product, paymentCustomerId, chargeId) -> + + // Even if the "capture charge operation" failed, we do not want to rollback. + // In that case, we just want to log it at error level. + // These transactions can then me manually changed before they are auto rollback'ed in 'X' days. + paymentProcessor.captureCharge(chargeId, paymentCustomerId) + .mapLeft { + // TODO payment: retry capture charge + logger.error("Capture failed for paymentCustomerId $paymentCustomerId, chargeId $chargeId, Fix this in Stripe Dashboard") + } - result.map { (profileInfo, _, chargeId, _) -> - // Capture the charge, our database have been updated. - paymentProcessor.captureCharge(chargeId, profileInfo.id) - .mapLeft { - // TODO payment: retry capture charge - logger.error("Capture failed for customerId ${profileInfo.id}, chargeId $chargeId, Fix this in Stripe Dashboard") + // Ignore failure to capture charge and always send Either.right() + ProductInfo(product.sku) + } + .ifFailedThenRollback(transaction) + } + // << END + + private fun removePaymentSource(saveCard: Boolean, paymentCustomerId: String, sourceId: String) { + // In case we fail to remove saved source, we log it at error level. + // These saved sources can then me manually removed. + if (!saveCard) { + paymentProcessor.removeSource(paymentCustomerId, sourceId) + .mapLeft { paymentError -> + logger.error("Failed to remove card, for customerId $paymentCustomerId, sourceId $sourceId") + paymentError } } - result.map { (profileInfo, savedSourceId, _, _) -> - // Remove the payment source - if (!saveCard && savedSourceId != null) { - paymentProcessor.removeSource(profileInfo.id, savedSourceId) - .mapLeft { paymentError -> - logger.error("Failed to remove card, for customerId ${profileInfo.id}, sourceId $savedSourceId") - paymentError - } - } - } - result.map { (_, _, _, product) -> ProductInfo(product.sku) } } - // << END override fun getPurchaseRecords(subscriberId: String): Either> { return readTransaction { @@ -710,11 +739,4 @@ object Neo4jStoreSingleton : GraphStore { // override fun getSegment(id: String): Segment? = segmentStore.get(id)?.let { Segment().apply { this.id = it.id } } // override fun getProductClass(id: String): ProductClass? = productClassStore.get(id) -} - -fun Either.ifFailedThenRollback(transaction: Transaction): Either { - if (this.isLeft()) { - transaction.failure() - } - return this } \ No newline at end of file diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/PrimeTransaction.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/PrimeTransaction.kt new file mode 100644 index 000000000..dd6dfecc5 --- /dev/null +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/PrimeTransaction.kt @@ -0,0 +1,73 @@ +package org.ostelco.prime.storage.graph + +import arrow.core.Either +import org.neo4j.driver.v1.Transaction +import org.ostelco.prime.storage.graph.ActionType.FINAL +import org.ostelco.prime.storage.graph.ActionType.REVERSAL + +class PrimeTransaction(private val transaction: Transaction) : Transaction by transaction { + + private val reversalActions = mutableListOf<() -> Unit>() + private val finalActions = mutableListOf<() -> Unit>() + + private fun toActionList(actionType: ActionType) = when (actionType) { + REVERSAL -> reversalActions + FINAL -> finalActions + } + + private fun doActions(actionType: ActionType) { + val actions = toActionList(actionType) + while (actions.isNotEmpty()) { + actions[0]() + actions.removeAt(0) + } + } + + fun addAction(actionType: ActionType, action: () -> Unit) { + toActionList(actionType).add(action) + } + + override fun failure() { + transaction.failure() + doActions(REVERSAL) + } + + override fun close() { + transaction.close() + finalActions.reverse() + doActions(FINAL) + } +} + +enum class ActionType { + REVERSAL, + FINAL, +} + +typealias Action

= (P) -> Unit + +private fun Either.addAction( + primeTransaction: PrimeTransaction, + action: Action, + actionType: ActionType): Either { + + this.map { param -> + primeTransaction.addAction(actionType) { + action(param) + } + } + return this +} + +fun Either.linkReversalActionToTransaction( + primeTransaction: PrimeTransaction, + reversalAction: Action): Either = addAction(primeTransaction, reversalAction, REVERSAL) + +fun Either.finallyDo( + primeTransaction: PrimeTransaction, + finalAction: Action): Either = addAction(primeTransaction, finalAction, FINAL) + +fun Either.ifFailedThenRollback(primeTransaction: PrimeTransaction): Either = mapLeft { error -> + primeTransaction.failure() + error +} \ No newline at end of file diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Schema.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Schema.kt index 4500633c0..396b28350 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Schema.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Schema.kt @@ -301,7 +301,7 @@ fun readTransaction(action: ReadTransaction.() -> R): R = Neo4jClient.driver.session(READ) .use { session -> session.readTransaction { - action(ReadTransaction(it)) + action(ReadTransaction(PrimeTransaction(it))) } } @@ -309,12 +309,12 @@ fun writeTransaction(action: WriteTransaction.() -> R): R = Neo4jClient.driver.session(WRITE) .use { session -> session.writeTransaction { - action(WriteTransaction(it)) + action(WriteTransaction(PrimeTransaction(it))) } } -data class ReadTransaction(val transaction: Transaction) -data class WriteTransaction(val transaction: Transaction) +data class ReadTransaction(val transaction: PrimeTransaction) +data class WriteTransaction(val transaction: PrimeTransaction) // // Object mapping functions diff --git a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt index 3d46a5fc1..f0ca74eb5 100644 --- a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt +++ b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/GraphStoreTest.kt @@ -40,27 +40,27 @@ class GraphStoreTest { Neo4jStoreSingleton.createProduct( Product(sku = "100MB_FREE_ON_JOINING", - price = Price(0, "NOK"), + price = Price(0, CURRENCY), properties = mapOf("noOfBytes" to "100_000_000"))) Neo4jStoreSingleton.createProduct( Product(sku = "1GB_FREE_ON_REFERRED", - price = Price(0, "NOK"), + price = Price(0, CURRENCY), properties = mapOf("noOfBytes" to "1_000_000_000"))) - val allSegment = Segment(id = "all") + val allSegment = Segment(id = getSegmentNameFromCountryCode(COUNTRY)) Neo4jStoreSingleton.createSegment(allSegment) } @Test fun `add subscriber`() { - Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME), referredBy = null) + Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME, country = COUNTRY), referredBy = null) .mapLeft { fail(it.message) } Neo4jStoreSingleton.getSubscriber(EMAIL).bimap( { fail(it.message) }, - { assertEquals(Subscriber(email = EMAIL, name = NAME, referralId = EMAIL), it) }) + { assertEquals(Subscriber(email = EMAIL, name = NAME, referralId = EMAIL, country = COUNTRY), it) }) // TODO vihang: fix argument captor for neo4j-store tests // val bundleArgCaptor: ArgumentCaptor = ArgumentCaptor.forClass(Bundle::class.java) @@ -71,7 +71,7 @@ class GraphStoreTest { @Test fun `fail to add subscriber with invalid referred by`() { - Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME), referredBy = "blah") + Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME, country = COUNTRY), referredBy = "blah") .fold({ assertEquals( expected = "Failed to create REFERRED - blah -> foo@bar.com", @@ -83,7 +83,7 @@ class GraphStoreTest { @Test fun `add subscription`() { - Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME), referredBy = null) + Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME, country = COUNTRY), referredBy = null) .mapLeft { fail(it.message) } Neo4jStoreSingleton.addSubscription(EMAIL, MSISDN) @@ -107,7 +107,7 @@ class GraphStoreTest { @Test fun `set and get Purchase record`() { - assert(Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME), referredBy = null).isRight()) + assert(Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME, country = COUNTRY), referredBy = null).isRight()) val product = createProduct("1GB_249NOK", 24900) val now = Instant.now().toEpochMilli() @@ -129,7 +129,7 @@ class GraphStoreTest { @Test fun `create products, offer, segment and then get products for a subscriber`() { - assert(Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME), referredBy = null).isRight()) + assert(Neo4jStoreSingleton.addSubscriber(Subscriber(email = EMAIL, name = NAME, country = COUNTRY), referredBy = null).isRight()) Neo4jStoreSingleton.createProduct(createProduct("1GB_249NOK", 24900)) Neo4jStoreSingleton.createProduct(createProduct("2GB_299NOK", 29900)) @@ -215,6 +215,8 @@ class GraphStoreTest { companion object { const val EMAIL = "foo@bar.com" const val NAME = "Test User" + const val CURRENCY = "NOK" + const val COUNTRY = "NO" const val MSISDN = "4712345678" @ClassRule @@ -226,7 +228,7 @@ class GraphStoreTest { HealthChecks.toRespond2xxOverHttp(7474) { port -> port.inFormat("http://\$HOST:\$EXTERNAL_PORT/browser") }, - Duration.standardSeconds(20L)) + Duration.standardSeconds(40L)) .build() @BeforeClass diff --git a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/SchemaTest.kt b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/SchemaTest.kt index dda1dc303..ddc64ff2f 100644 --- a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/SchemaTest.kt +++ b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/SchemaTest.kt @@ -201,7 +201,7 @@ class SchemaTest { HealthChecks.toRespond2xxOverHttp(7474) { port -> port.inFormat("http://\$HOST:\$EXTERNAL_PORT/browser") }, - Duration.standardSeconds(10L)) + Duration.standardSeconds(40L)) .build() @BeforeClass diff --git a/ocs-grpc-api/build.gradle b/ocs-grpc-api/build.gradle index 1a37a301e..fb2ba62f8 100644 --- a/ocs-grpc-api/build.gradle +++ b/ocs-grpc-api/build.gradle @@ -9,6 +9,7 @@ dependencies { api "io.grpc:grpc-protobuf:$grpcVersion" api "io.grpc:grpc-stub:$grpcVersion" api "io.grpc:grpc-core:$grpcVersion" + implementation 'javax.annotation:javax.annotation-api:1.3.2' } protobuf { diff --git a/ocs/build.gradle b/ocs/build.gradle index aef0024c8..9444e7e74 100644 --- a/ocs/build.gradle +++ b/ocs/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" } @@ -13,7 +13,7 @@ dependencies { implementation project(':prime-modules') implementation 'com.lmax:disruptor:3.4.2' - implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:0.26.1" + // implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:0.30.0" testImplementation "org.jetbrains.kotlin:kotlin-test:$kotlinVersion" testImplementation "org.jetbrains.kotlin:kotlin-test-junit:$kotlinVersion" diff --git a/ocs/design.puml b/ocs/design.puml index 4ee79d418..501724acb 100644 --- a/ocs/design.puml +++ b/ocs/design.puml @@ -6,7 +6,7 @@ [ocsgw] -interface OcsGrpcService +[OcsGrpcService] [OcsGrpcServer] diff --git a/ocs/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsReporter.kt b/ocs/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsReporter.kt index e564c291c..0a0b19e8f 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsReporter.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsReporter.kt @@ -1,7 +1,6 @@ package org.ostelco.prime.analytics import com.lmax.disruptor.EventHandler -import org.ostelco.prime.analytics.PrimeMetric.MEGABYTES_CONSUMED import org.ostelco.prime.disruptor.EventMessageType.CREDIT_CONTROL_REQUEST import org.ostelco.prime.disruptor.OcsEvent import org.ostelco.prime.getLogger @@ -34,10 +33,6 @@ object AnalyticsReporter : EventHandler { bundleBytes = event.bundleBytes, apn = event.request?.serviceInformation?.psInformation?.calledStationId, mccMnc = event.request?.serviceInformation?.psInformation?.sgsnMccMnc) - analyticsReporter.reportMetric( - primeMetric = MEGABYTES_CONSUMED, - value = (event.request?.msccList?.firstOrNull()?.used?.totalOctets ?: 0L) / 1_000_000) - } } } diff --git a/ocs/src/main/kotlin/org/ostelco/prime/consumption/OcsService.kt b/ocs/src/main/kotlin/org/ostelco/prime/consumption/OcsService.kt index a529586a6..41a4588a8 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/consumption/OcsService.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/consumption/OcsService.kt @@ -72,7 +72,6 @@ class OcsService(private val producer: EventProducer) : OcsAsyncRequestConsumer, } override fun activateOnNextResponse(response: ActivateResponse) { - // TODO martin: send activate MSISDN to selective ocsgw instead of all this.activateMsisdnClientMap.forEach { _ , responseStream -> responseStream.onNext(response) } diff --git a/ocs/src/main/kotlin/org/ostelco/prime/handler/PurchaseRequestHandler.kt b/ocs/src/main/kotlin/org/ostelco/prime/handler/PurchaseRequestHandler.kt index f4044b339..2ed28397e 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/handler/PurchaseRequestHandler.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/handler/PurchaseRequestHandler.kt @@ -80,7 +80,7 @@ class PurchaseRequestHandler( val future = CompletableFuture() requestMap[requestId] = future producer.topupDataBundleBalanceEvent(requestId = requestId, bundleId = bundleId, bytes = noOfBytes) - val error = future.get(5, MILLISECONDS) + val error = future.get(100, MILLISECONDS) if (error.isNotBlank()) { return Either.left(error) } diff --git a/ocsgw/Dockerfile b/ocsgw/Dockerfile index 2e2ffd740..b6b5264ee 100644 --- a/ocsgw/Dockerfile +++ b/ocsgw/Dockerfile @@ -1,4 +1,4 @@ -FROM azul/zulu-openjdk:8u181-8.31.0.1 +FROM openjdk:11 MAINTAINER CSI "csi@telenordigital.com" diff --git a/ocsgw/build.gradle b/ocsgw/build.gradle index 75bd6fd43..0c92891f4 100644 --- a/ocsgw/build.gradle +++ b/ocsgw/build.gradle @@ -1,10 +1,9 @@ plugins { id "application" + // FIXME: unable to update to 4.0.0 id "com.github.johnrengelman.shadow" version "2.0.4" } -ext.junit5Version = "5.3.1" - dependencies { implementation project(':ocs-grpc-api') implementation project(':analytics-grpc-api') @@ -12,10 +11,13 @@ dependencies { implementation project(':diameter-stack') implementation "com.google.cloud:google-cloud-core-grpc:$googleCloudVersion" + implementation 'javax.xml.bind:jaxb-api:2.3.0' + implementation 'javax.activation:activation:1.1.1' + implementation 'ch.qos.logback:logback-classic:1.2.3' // log to gcp stack-driver - implementation 'com.google.cloud:google-cloud-logging-logback:0.63.0-alpha' + implementation 'com.google.cloud:google-cloud-logging-logback:0.64.0-alpha' testImplementation project(':diameter-test') testImplementation "org.junit.jupiter:junit-jupiter-api:$junit5Version" @@ -25,6 +27,13 @@ dependencies { testRuntimeOnly "org.junit.vintage:junit-vintage-engine:$junit5Version" } +shadowJar { + mainClassName = 'org.ostelco.ocsgw.OcsApplication' + mergeServiceFiles() + classifier = "uber" + version = null +} + test { // native support to Junit5 in Gradle 4.6+ useJUnitPlatform { @@ -36,13 +45,6 @@ test { } } -shadowJar { - mainClassName = 'org.ostelco.ocsgw.OcsApplication' - mergeServiceFiles() - classifier = "uber" - version = null -} - task pack(dependsOn: ['packDev', 'packProd']) task packProd(type: Zip, dependsOn: 'shadowJar') { @@ -93,20 +95,6 @@ task packDev(type: Zip, dependsOn: 'shadowJar') { fileName.replace('dev.', '') } } - // TODO vihang: figure out why wild-card certs fail to verify - from ('../certs/dev.ostelco.org/nginx.crt') { - into (project.name + '/config/') - rename { String fileName -> - fileName.replace('nginx', 'ocs') - } - } - from ('../certs/dev.ostelco.org/nginx.crt') { - into (project.name + '/config/') - rename { String fileName -> - fileName.replace('nginx', 'metrics') - } - } - // END of certs from ('config/pantel-prod.json') { into (project.name + '/config/') } diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/OcsServer.java b/ocsgw/src/main/java/org/ostelco/ocsgw/OcsServer.java index e3b4c684a..6a757aa29 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/OcsServer.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/OcsServer.java @@ -27,7 +27,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.naming.ConfigurationException; import java.io.IOException; diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java index 2b7362b99..a320da85c 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java @@ -25,6 +25,8 @@ import org.ostelco.ocs.api.*; import org.ostelco.ocsgw.OcsServer; import org.ostelco.ocsgw.data.DataSource; +import org.ostelco.prime.metrics.api.OcsgwAnalyticsReport; +import org.ostelco.prime.metrics.api.User; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,13 +35,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -256,18 +252,17 @@ private void handleGrpcCcrAnswer(CreditControlAnswerInfo answer) { } private void addToSessionMap(CreditControlContext creditControlContext) { - switch (getRequestType(creditControlContext)) { - case INITIAL_REQUEST: - case UPDATE_REQUEST: - case TERMINATION_REQUEST: - sessionIdMap.put(creditControlContext.getCreditControlRequest().getMsisdn(), new SessionContext(creditControlContext.getSessionId(), creditControlContext.getCreditControlRequest().getOriginHost(), creditControlContext.getCreditControlRequest().getOriginRealm())); + try { + SessionContext sessionContext = new SessionContext(creditControlContext.getSessionId(), + creditControlContext.getCreditControlRequest().getOriginHost(), + creditControlContext.getCreditControlRequest().getOriginRealm(), + creditControlContext.getCreditControlRequest().getServiceInformation().get(0).getPsInformation().get(0).getCalledStationId(), + creditControlContext.getCreditControlRequest().getServiceInformation().get(0).getPsInformation().get(0).getSgsnMccMnc()); + if (sessionIdMap.put(creditControlContext.getCreditControlRequest().getMsisdn(), sessionContext) == null) { updateAnalytics(); - break; - case EVENT_REQUEST: - break; - default: - LOG.warn("Unknown request type"); - break; + } + } catch (Exception e) { + LOG.error("Failed to update session map", e); } } @@ -279,9 +274,14 @@ private void removeFromSessionMap(CreditControlContext creditControlContext) { } private void updateAnalytics() { - LOG.info("Number of active sesssions is {}", sessionIdMap.size()); + LOG.info("Number of active sessions is {}", sessionIdMap.size()); - ocsgwAnalytics.sendAnalytics(sessionIdMap.size()); + OcsgwAnalyticsReport.Builder builder = OcsgwAnalyticsReport.newBuilder().setActiveSessions(sessionIdMap.size()); + builder.setKeepAlive(false); + sessionIdMap.forEach((msisdn, sessionContext) -> { + builder.addUsers(User.newBuilder().setApn(sessionContext.getApn()).setMccMnc(sessionContext.getMccMnc()).setMsisdn(msisdn).build()); + }); + ocsgwAnalytics.sendAnalytics(builder.build()); } private void initActivate() { diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java index 3b863e8dd..4e8524cff 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java @@ -42,7 +42,7 @@ class OcsgwMetrics { private ScheduledFuture keepAliveFuture = null; - private int lastActiveSessions = 0; + private OcsgwAnalyticsReport lastActiveSessions = OcsgwAnalyticsReport.newBuilder().setKeepAlive(true).build(); OcsgwMetrics(String metricsServerHostname, ServiceAccountJwtAccessCredentials credentials) { @@ -126,15 +126,15 @@ public void onNext(OcsgwAnalyticsReply value) { private void initKeepAlive() { // this is used to keep connection alive keepAliveFuture = executorService.scheduleWithFixedDelay(() -> { - sendAnalytics(lastActiveSessions); + sendAnalytics(OcsgwAnalyticsReport.newBuilder().setKeepAlive(true).build()); }, 15, 50, TimeUnit.SECONDS); } - void sendAnalytics(int size) { - ocsgwAnalyticsReport.onNext(OcsgwAnalyticsReport.newBuilder().setActiveSessions(size).build()); - lastActiveSessions = size; + void sendAnalytics(OcsgwAnalyticsReport report) { + ocsgwAnalyticsReport.onNext(report); + lastActiveSessions = report; } } \ No newline at end of file diff --git a/ocsgw/src/test/java/org/ostelco/ocsgw/OcsApplicationTest.java b/ocsgw/src/test/java/org/ostelco/ocsgw/OcsApplicationTest.java index 43362f09d..e4918dd8f 100644 --- a/ocsgw/src/test/java/org/ostelco/ocsgw/OcsApplicationTest.java +++ b/ocsgw/src/test/java/org/ostelco/ocsgw/OcsApplicationTest.java @@ -41,6 +41,8 @@ public class OcsApplicationTest { private static final String OCS_HOST = "ocs"; private static final String PGW_HOST = "testclient"; private static final String PGW_REALM = "loltel"; + private static final String APN = "loltel-test"; + private static final String MCC_MNC = "24201"; private static final String MSISDN = "4790300123"; @@ -162,7 +164,7 @@ public void testReAuthRequest() { client.initRequestTest(); - OcsServer.getInstance().sendReAuthRequest(new SessionContext(session.getSessionId(), PGW_HOST, PGW_REALM)); + OcsServer.getInstance().sendReAuthRequest(new SessionContext(session.getSessionId(), PGW_HOST, PGW_REALM, APN, MCC_MNC)); waitForRequest(); try { AvpSet resultAvps = client.getResultAvps(); diff --git a/payment-processor/build.gradle b/payment-processor/build.gradle index e47629b26..9c2069846 100644 --- a/payment-processor/build.gradle +++ b/payment-processor/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" id "idea" } diff --git a/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt b/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt index 6b0d8aecc..4f1987749 100644 --- a/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt +++ b/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt @@ -1,6 +1,10 @@ package org.ostelco.prime.paymentprocessor +import arrow.core.getOrElse +import arrow.core.right +import arrow.core.some import com.stripe.Stripe +import com.stripe.model.Source import com.stripe.model.Token import org.junit.After import org.junit.Before @@ -17,19 +21,40 @@ class StripePaymentProcessorTest { private var stripeCustomerId = "" - fun createPaymentSourceId(): String { + private fun createPaymentTokenId() : String { val cardMap = mapOf( "number" to "4242424242424242", "exp_month" to 8, "exp_year" to 2019, "cvc" to "314") - val tokenMap = mapOf("card" to cardMap) + val token = Token.create(tokenMap) return token.id } + private fun createPaymentSourceId() : String { + + val sourceMap = mapOf( + "type" to "card", + "card" to mapOf( + "number" to "4242424242424242", + "exp_month" to 8, + "exp_year" to 2019, + "cvc" to "314"), + "owner" to mapOf( + "address" to mapOf( + "city" to "Oslo", + "country" to "Norway" + ), + "email" to "me@somewhere.com") + ) + + val source = Source.create(sourceMap) + return source.id + } + private fun addCustomer() { val resultAdd = paymentProcessor.createPaymentProfile(testCustomer) assertEquals(true, resultAdd.isRight()) @@ -68,10 +93,52 @@ class StripePaymentProcessorTest { assertEquals(false, result.isRight()) } + @Test + fun ensureSourcesSorted() { + + run { + paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) + // Ensure that not all sources falls within the same second. + Thread.sleep(1_001) + paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + } + + // Should be in descending sorted order by the "created" timestamp. + val sources = paymentProcessor.getSavedSources(stripeCustomerId) + + val createdTimestamps = sources.getOrElse { + fail("The 'created' field is missing from the list of sources: ${sources}") + }.map { it.details["created"] as Long } + + val createdTimestampsSorted = createdTimestamps.sortedByDescending { it } + + assertEquals(createdTimestamps, createdTimestampsSorted, + "The list of sources is not in descending sorted order by 'created' timestamp: ${sources}") + } + + @Test + fun addAndRemoveMultipleSources() { + + val sources= listOf( + paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()), + paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + ) + + val sourcesRemoved = sources.map { + paymentProcessor.removeSource(stripeCustomerId, it.getOrElse { + fail("Failed to remove source ${it}") + }.id) + } + + sourcesRemoved.forEach { it -> + assertEquals(true, it.isRight(), "Unexpected failure when removing source ${it}") + } + } + @Test fun addSourceToCustomerAndRemove() { - val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) val resultStoredSources = paymentProcessor.getSavedSources(stripeCustomerId) assertEquals(1, resultStoredSources.fold({ 0 }, { it.size })) @@ -87,8 +154,8 @@ class StripePaymentProcessorTest { } @Test - fun addSourceToCustomerTwise() { - val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + fun addSourceToCustomerTwice() { + val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) val resultStoredSources = paymentProcessor.getSavedSources(stripeCustomerId) assertEquals(1, resultStoredSources.fold({ 0 }, { it.size })) @@ -109,7 +176,7 @@ class StripePaymentProcessorTest { @Test fun addDefaultSourceAndRemove() { - val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) assertEquals(true, resultAddSource.isRight()) val resultAddDefault = paymentProcessor.setDefaultSource(stripeCustomerId, resultAddSource.fold({ "" }, { it.id })) @@ -125,7 +192,7 @@ class StripePaymentProcessorTest { @Test fun createAuthorizeChargeAndRefund() { - val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) assertEquals(true, resultAddSource.isRight()) val resultAuthorizeCharge = paymentProcessor.authorizeCharge(stripeCustomerId, resultAddSource.fold({ "" }, { it.id }), 1000, "nok") @@ -152,7 +219,7 @@ class StripePaymentProcessorTest { @Test fun subscribeAndUnsubscribePlan() { - val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) assertEquals(true, resultAddSource.isRight()) val resultCreateProduct = paymentProcessor.createProduct("TestSku") diff --git a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt index 6b6f980ca..af56dbe54 100644 --- a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt +++ b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt @@ -2,11 +2,33 @@ package org.ostelco.prime.paymentprocessor import arrow.core.Either import arrow.core.flatMap -import com.stripe.exception.* -import org.ostelco.prime.getLogger -import com.stripe.model.* -import org.ostelco.prime.paymentprocessor.core.* +import com.stripe.exception.ApiConnectionException +import com.stripe.exception.AuthenticationException +import com.stripe.exception.CardException +import com.stripe.exception.InvalidRequestException +import com.stripe.exception.RateLimitException +import com.stripe.exception.StripeException +import com.stripe.model.Card +import com.stripe.model.Charge import com.stripe.model.Customer +import com.stripe.model.ExternalAccount +import com.stripe.model.Plan +import com.stripe.model.Product +import com.stripe.model.Refund +import com.stripe.model.Source +import com.stripe.model.Subscription +import org.ostelco.prime.getLogger +import org.ostelco.prime.paymentprocessor.core.BadGatewayError +import org.ostelco.prime.paymentprocessor.core.ForbiddenError +import org.ostelco.prime.paymentprocessor.core.NotFoundError +import org.ostelco.prime.paymentprocessor.core.PaymentError +import org.ostelco.prime.paymentprocessor.core.PlanInfo +import org.ostelco.prime.paymentprocessor.core.ProductInfo +import org.ostelco.prime.paymentprocessor.core.ProfileInfo +import org.ostelco.prime.paymentprocessor.core.SourceDetailsInfo +import org.ostelco.prime.paymentprocessor.core.SourceInfo +import org.ostelco.prime.paymentprocessor.core.SubscriptionInfo + class StripePaymentProcessor : PaymentProcessor { @@ -14,21 +36,20 @@ class StripePaymentProcessor : PaymentProcessor { override fun getSavedSources(customerId: String): Either> = either("Failed to retrieve sources for customer $customerId") { - val sources = mutableListOf() val customer = Customer.retrieve(customerId) - customer.sources.data.forEach { + val sources: List = customer.sources.data.map { val details = getAccountDetails(it) - sources.add(SourceDetailsInfo(it.id, getAccountType(details), details)) + SourceDetailsInfo(it.id, getAccountType(details), details) } - sources + sources.sortedByDescending { it.details["created"] as Long } } private fun getAccountType(details: Map) : String { - return details.get("type").toString() + return details["type"].toString() } /* Returns detailed 'account details' for the given Stripe source/account. - Note that including the fields 'id' and 'type' are manadatory. */ + Note that including the fields 'id', 'type' and 'created' are mandatory. */ private fun getAccountDetails(accountInfo: ExternalAccount) : Map { when (accountInfo) { is Card -> { @@ -43,6 +64,8 @@ class StripePaymentProcessor : PaymentProcessor { "country" to accountInfo.country, "currency" to accountInfo.currency, "cvcCheck" to accountInfo.cvcCheck, + "created" to getCreatedTimestampFromMetadata(accountInfo.id, + accountInfo.metadata), "expMonth" to accountInfo.expMonth, "expYear" to accountInfo.expYear, "fingerprint" to accountInfo.fingerprint, @@ -54,6 +77,7 @@ class StripePaymentProcessor : PaymentProcessor { is Source -> { return mapOf("id" to accountInfo.id, "type" to "source", + "created" to accountInfo.created, "typeData" to accountInfo.typeData, "owner" to accountInfo.owner) } @@ -61,11 +85,30 @@ class StripePaymentProcessor : PaymentProcessor { logger.error("Received unsupported Stripe source/account type: {}", accountInfo) return mapOf("id" to accountInfo.id, - "type" to "unsupported") + "type" to "unsupported", + "created" to getSecondsSinceEpoch()) } } } + /* Handle type conversion when reading the 'created' field from the + metadata returned from Stripe. (It might seem like that Stripe + returns stored metadata values as strings, even if they where stored + using an another type. Needs to be verified.) */ + private fun getCreatedTimestampFromMetadata(id: String, metadata: Map) : Long { + val created: String? = metadata["created"] as? String + return created?.toLongOrNull() ?: run { + logger.warn("No 'created' timestamp found in metadata for Stripe account {}", + id) + getSecondsSinceEpoch() + } + } + + /* Seconds since Epoch in UTC zone. */ + private fun getSecondsSinceEpoch() : Long { + return System.currentTimeMillis() / 1000L + } + override fun createPaymentProfile(userEmail: String): Either = either("Failed to create profile for user $userEmail") { val customerParams = mapOf("email" to userEmail) @@ -77,13 +120,11 @@ class StripePaymentProcessor : PaymentProcessor { "limit" to "1", "email" to userEmail) val customerList = Customer.list(customerParams) - if (customerList.data.isEmpty()) { - return Either.left(NotFoundError("Could not find a payment profile for user $userEmail")) - } else if (customerList.data.size > 1){ - return Either.left(NotFoundError("Multiple profiles for user $userEmail found")) - } else { - return Either.right(ProfileInfo(customerList.data.first().id)) - } + return when { + customerList.data.isEmpty() -> Either.left(NotFoundError("Could not find a payment profile for user $userEmail")) + customerList.data.size > 1 -> Either.left(NotFoundError("Multiple profiles for user $userEmail found")) + else -> Either.right(ProfileInfo(customerList.data.first().id)) + } } override fun createPlan(productId: String, amount: Int, currency: String, interval: PaymentProcessor.Interval): Either = @@ -119,7 +160,8 @@ class StripePaymentProcessor : PaymentProcessor { override fun addSource(customerId: String, sourceId: String): Either = either("Failed to add source $sourceId to customer $customerId") { val customer = Customer.retrieve(customerId) - val params = mapOf("source" to sourceId) + val params = mapOf("source" to sourceId, + "metadata" to mapOf("created" to getSecondsSinceEpoch())) SourceInfo(customer.sources.create(params).id) } @@ -210,9 +252,16 @@ class StripePaymentProcessor : PaymentProcessor { Refund.create(refundParams).charge } - override fun removeSource(customerId: String, sourceId: String): Either = + override fun removeSource(customerId: String, sourceId: String): Either = either("Failed to remove source $sourceId from customer $customerId") { - Customer.retrieve(customerId).sources.retrieve(sourceId).delete().id + val accountInfo = Customer.retrieve(customerId).sources.retrieve(sourceId) + when (accountInfo) { + is Card -> accountInfo.delete() + is Source -> accountInfo.detach() + else -> + Either.left(BadGatewayError("Attempt to remove unsupported account-type $accountInfo")) + } + SourceInfo(sourceId) } private fun either(errorDescription: String, action: () -> RETURN): Either { @@ -220,28 +269,28 @@ class StripePaymentProcessor : PaymentProcessor { Either.right(action()) } catch (e: CardException) { // If something is decline with a card purchase, CardException will be caught - logger.warn("Payment error : $errorDescription , Stripe Error Code: ${e.getCode()}", e) + logger.warn("Payment error : $errorDescription , Stripe Error Code: ${e.code}", e) Either.left(ForbiddenError(errorDescription, e.message)) } catch (e: RateLimitException) { // Too many requests made to the API too quickly - logger.warn("Payment error : $errorDescription , Stripe Error Code: ${e.getCode()}", e) + logger.warn("Payment error : $errorDescription , Stripe Error Code: ${e.code}", e) Either.left(BadGatewayError(errorDescription, e.message)) } catch (e: InvalidRequestException) { // Invalid parameters were supplied to Stripe's API - logger.warn("Payment error : $errorDescription , Stripe Error Code: ${e.getCode()}", e) + logger.warn("Payment error : $errorDescription , Stripe Error Code: ${e.code}", e) Either.left(NotFoundError(errorDescription, e.message)) } catch (e: AuthenticationException) { // Authentication with Stripe's API failed // (maybe you changed API keys recently) - logger.warn("Payment error : $errorDescription , Stripe Error Code: ${e.getCode()}", e) + logger.warn("Payment error : $errorDescription , Stripe Error Code: ${e.code}", e) Either.left(BadGatewayError(errorDescription)) } catch (e: ApiConnectionException) { // Network communication with Stripe failed - logger.warn("Payment error : $errorDescription , Stripe Error Code: ${e.getCode()}", e) + logger.warn("Payment error : $errorDescription , Stripe Error Code: ${e.code}", e) Either.left(BadGatewayError(errorDescription)) } catch (e: StripeException) { // Unknown Stripe error - logger.error("Payment error : $errorDescription , Stripe Error Code: ${e.getCode()}", e) + logger.error("Payment error : $errorDescription , Stripe Error Code: ${e.code}", e) Either.left(BadGatewayError(errorDescription)) } catch (e: Exception) { // Something else happened, could be completely unrelated to Stripe diff --git a/prime-client-api/build.gradle b/prime-client-api/build.gradle index ba58c49f1..09a55d0d2 100644 --- a/prime-client-api/build.gradle +++ b/prime-client-api/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id 'java-library' id 'org.hidetake.swagger.generator' version '2.13.0' id "idea" @@ -41,6 +41,8 @@ dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion" swaggerCodegen 'io.swagger:swagger-codegen-cli:2.3.1' + implementation 'javax.annotation:javax.annotation-api:1.3.2' + // taken from build/swagger-code-java-client/build.gradle implementation 'io.swagger:swagger-annotations:1.5.21' implementation 'com.google.code.gson:gson:2.8.5' diff --git a/prime-modules/build.gradle b/prime-modules/build.gradle index a302cb169..43c9bbe4e 100644 --- a/prime-modules/build.gradle +++ b/prime-modules/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" } diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt index f91b0afd6..ee2ff6a1d 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt @@ -1,6 +1,5 @@ package org.ostelco.prime.analytics -import org.ostelco.prime.analytics.MetricType.COUNTER import org.ostelco.prime.analytics.MetricType.GAUGE import org.ostelco.prime.model.PurchaseRecord @@ -14,11 +13,8 @@ enum class PrimeMetric(val metricType: MetricType) { // sorted alphabetically ACTIVE_SESSIONS(GAUGE), - MEGABYTES_CONSUMED(COUNTER), - REVENUE(COUNTER), TOTAL_USERS(GAUGE), - USERS_ACQUIRED_THROUGH_REFERRALS(GAUGE), - USERS_PAID_AT_LEAST_ONCE(GAUGE); + USERS_ACQUIRED_THROUGH_REFERRALS(GAUGE); val metricName: String get() = name.toLowerCase() diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiErrorCodes.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiErrorCodes.kt index 4b0950a03..95e438d7c 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiErrorCodes.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiErrorCodes.kt @@ -18,6 +18,8 @@ enum class ApiErrorCode { FAILED_TO_STORE_PAYMENT_SOURCE, FAILED_TO_SET_DEFAULT_PAYMENT_SOURCE, FAILED_TO_FETCH_PAYMENT_SOURCES_LIST, + FAILED_TO_REMOVE_PAYMENT_SOURCE, + FAILED_TO_CREATE_PROFILE, FAILED_TO_UPDATE_PROFILE, FAILED_TO_FETCH_CONSENT, FAILED_TO_IMPORT_OFFER, diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/PaymentProcessor.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/PaymentProcessor.kt index 098abdb17..eda338db4 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/PaymentProcessor.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/PaymentProcessor.kt @@ -124,6 +124,5 @@ interface PaymentProcessor { * @param sourceId id of the payment source * @return id if removed */ - fun removeSource(customerId: String, sourceId: String): Either - + fun removeSource(customerId: String, sourceId: String): Either } \ No newline at end of file diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/core/Model.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/core/Model.kt index bf608316c..21c0b9301 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/core/Model.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/core/Model.kt @@ -1,13 +1,13 @@ package org.ostelco.prime.paymentprocessor.core -class PlanInfo(val id: String) +data class PlanInfo(val id: String) -class ProductInfo(val id: String) +data class ProductInfo(val id: String) -class ProfileInfo(val id: String) +data class ProfileInfo(val id: String) -class SourceInfo(val id: String) +data class SourceInfo(val id: String) -class SourceDetailsInfo(val id: String, val type: String, val details: Map) +data class SourceDetailsInfo(val id: String, val type: String, val details: Map) -class SubscriptionInfo(val id: String) +data class SubscriptionInfo(val id: String) diff --git a/prime/Dockerfile b/prime/Dockerfile index a7903c099..c89b95728 100644 --- a/prime/Dockerfile +++ b/prime/Dockerfile @@ -1,4 +1,4 @@ -FROM azul/zulu-openjdk:8u181-8.31.0.1 +FROM openjdk:11 MAINTAINER CSI "csi@telenordigital.com" diff --git a/prime/Dockerfile.test b/prime/Dockerfile.test index 0c542a2b4..7e088682f 100644 --- a/prime/Dockerfile.test +++ b/prime/Dockerfile.test @@ -1,6 +1,6 @@ # This Dockerfile is used when running locally using docker-compose for Acceptance Testing. -FROM azul/zulu-openjdk:8u181-8.31.0.1 +FROM openjdk:11 MAINTAINER CSI "csi@telenordigital.com" diff --git a/prime/build.gradle b/prime/build.gradle index 42fc01c4e..6e73050c9 100644 --- a/prime/build.gradle +++ b/prime/build.gradle @@ -1,7 +1,7 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "application" - id "com.github.johnrengelman.shadow" version "2.0.4" + id "com.github.johnrengelman.shadow" version "4.0.0" id "idea" } @@ -18,7 +18,7 @@ sourceSets { } } -version = "1.15.0" +version = "1.16.0" repositories { maven { diff --git a/prime/cloudbuild.dev.yaml b/prime/cloudbuild.dev.yaml index 88c3dceca..3394a66cf 100644 --- a/prime/cloudbuild.dev.yaml +++ b/prime/cloudbuild.dev.yaml @@ -50,7 +50,7 @@ steps: path: /root/out_zip # Build docker images - name: gcr.io/cloud-builders/docker - args: ['build', '--tag=eu.gcr.io/$PROJECT_ID/prime:$SHORT_SHA', '--cache-from', 'azul/zulu-openjdk:8u181-8.31.0.1', 'prime'] + args: ['build', '--tag=eu.gcr.io/$PROJECT_ID/prime:$SHORT_SHA', '--cache-from', 'openjdk:11', 'prime'] timeout: 120s # Deploy new docker image to Google Kubernetes Engine (GKE) - name: ubuntu diff --git a/prime/cloudbuild.yaml b/prime/cloudbuild.yaml index d4cba67b0..ff9d63966 100644 --- a/prime/cloudbuild.yaml +++ b/prime/cloudbuild.yaml @@ -50,7 +50,7 @@ steps: path: /root/out_zip # Build docker images - name: gcr.io/cloud-builders/docker - args: ['build', '--tag=eu.gcr.io/$PROJECT_ID/prime:$TAG_NAME', '--cache-from', 'azul/zulu-openjdk:8u181-8.31.0.1', 'prime'] + args: ['build', '--tag=eu.gcr.io/$PROJECT_ID/prime:$TAG_NAME', '--cache-from', 'openjdk:11', 'prime'] timeout: 120s # Deploy new docker image to Google Kubernetes Engine (GKE) - name: ubuntu diff --git a/prime/config/config.yaml b/prime/config/config.yaml index 008759df7..f96a3fcc4 100644 --- a/prime/config/config.yaml +++ b/prime/config/config.yaml @@ -12,6 +12,7 @@ modules: projectId: pantel-2decb dataTrafficTopicId: ${DATA_TRAFFIC_TOPIC} purchaseInfoTopicId: ${PURCHASE_INFO_TOPIC} + activeUsersTopicId: ${ACTIVE_USERS_TOPIC} - type: ocs config: lowBalanceThreshold: 100000000 diff --git a/prime/config/test.yaml b/prime/config/test.yaml index a178eb66f..57ffe6db3 100644 --- a/prime/config/test.yaml +++ b/prime/config/test.yaml @@ -14,6 +14,7 @@ modules: projectId: pantel-2decb dataTrafficTopicId: data-traffic purchaseInfoTopicId: purchase-info + activeUsersTopicId: active-users - type: ocs config: lowBalanceThreshold: 0 diff --git a/prime/infra/README.md b/prime/infra/README.md index 9162e4760..e1d3d696d 100644 --- a/prime/infra/README.md +++ b/prime/infra/README.md @@ -83,6 +83,8 @@ Reference: Generate self-contained protobuf descriptor file - `ocs_descriptor.pb` & `metrics_descriptor.pb` ```bash +pyenv versions +pyenv local 3.5.2 pip install grpcio grpcio-tools python -m grpc_tools.protoc \ @@ -342,3 +344,25 @@ logName="projects/pantel-2decb/logs/prime" ## Connect using Neo4j Browser Check [docs/NEO4J.md](../docs/NEO4J.md) + +## Deploy dataflow pipeline for raw_activeusers + +```bash +# For dev cluster +gcloud dataflow jobs run active-users-dev \ + --gcs-location gs://dataflow-templates/latest/PubSub_to_BigQuery \ + --region europe-west1 \ + --parameters \ +inputTopic=projects/pantel-2decb/topics/active-users-dev,\ +outputTableSpec=pantel-2decb:ocs_gateway_dev.raw_activeusers + + +# For production cluster +gcloud dataflow jobs run active-users \ + --gcs-location gs://dataflow-templates/latest/PubSub_to_BigQuery \ + --region europe-west1 \ + --parameters \ +inputTopic=projects/pantel-2decb/topics/active-users,\ +outputTableSpec=pantel-2decb:ocs_gateway.raw_activeusers + +``` \ No newline at end of file diff --git a/prime/infra/dev/metrics-api.yaml b/prime/infra/dev/metrics-api.yaml index 750adc869..fafd2ff79 100644 --- a/prime/infra/dev/metrics-api.yaml +++ b/prime/infra/dev/metrics-api.yaml @@ -27,4 +27,4 @@ authentication: rules: - selector: "*" requirements: - - provider_id: google_service_account + - provider_id: google_service_account \ No newline at end of file diff --git a/prime/infra/dev/prime-client-api.yaml b/prime/infra/dev/prime-client-api.yaml index 4f258dfa6..4038789bc 100644 --- a/prime/infra/dev/prime-client-api.yaml +++ b/prime/infra/dev/prime-client-api.yaml @@ -151,6 +151,28 @@ paths: description: "User not found." security: - auth0_jwt: [] + delete: + description: "Remove a payment source for user" + produces: + - application/json + operationId: "removeSource" + parameters: + - name: sourceId + in: query + description: "The stripe-id of the source to be removed" + required: true + type: string + responses: + 200: + description: "Successfully removed the source" + schema: + $ref: '#/definitions/PaymentSource' + 400: + description: "The source could not be removed" + 404: + description: "No such source for user" + security: + - auth0_jwt: [] "/products": get: description: "Get all products for the user." diff --git a/prime/infra/dev/prime.yaml b/prime/infra/dev/prime.yaml index ec00224e2..05a80140b 100644 --- a/prime/infra/dev/prime.yaml +++ b/prime/infra/dev/prime.yaml @@ -155,6 +155,8 @@ spec: value: data-traffic-dev - name: PURCHASE_INFO_TOPIC value: purchase-info-dev + - name: ACTIVE_USERS_TOPIC + value: active-users-dev - name: STRIPE_API_KEY valueFrom: secretKeyRef: diff --git a/prime/infra/new-dev/prime-client-api.yaml b/prime/infra/new-dev/prime-client-api.yaml index d192203a3..7af2407e1 100644 --- a/prime/infra/new-dev/prime-client-api.yaml +++ b/prime/infra/new-dev/prime-client-api.yaml @@ -151,6 +151,28 @@ paths: description: "User not found." security: - auth0_jwt: [] + delete: + description: "Remove a payment source for user" + produces: + - application/json + operationId: "removeSource" + parameters: + - name: sourceId + in: query + description: "The stripe-id of the source to be removed" + required: true + type: string + responses: + 200: + description: "Successfully removed the source" + schema: + $ref: '#/definitions/PaymentSource' + 400: + description: "The source could not be removed" + 404: + description: "No such source for user" + security: + - auth0_jwt: [] "/products": get: description: "Get all products for the user." @@ -573,4 +595,4 @@ securityDefinitions: type: "oauth2" x-google-issuer: "https://ostelco.eu.auth0.com/" x-google-jwks_uri: "https://ostelco.eu.auth0.com/.well-known/jwks.json" - x-google-audiences: "http://google_api" \ No newline at end of file + x-google-audiences: "http://google_api" diff --git a/prime/infra/new-prod/prime-client-api.yaml b/prime/infra/new-prod/prime-client-api.yaml index 7c705f9ae..09d1d2448 100644 --- a/prime/infra/new-prod/prime-client-api.yaml +++ b/prime/infra/new-prod/prime-client-api.yaml @@ -151,6 +151,28 @@ paths: description: "User not found." security: - auth0_jwt: [] + delete: + description: "Remove a payment source for user" + produces: + - application/json + operationId: "removeSource" + parameters: + - name: sourceId + in: query + description: "The stripe-id of the source to be removed" + required: true + type: string + responses: + 200: + description: "Successfully removed the source" + schema: + $ref: '#/definitions/PaymentSource' + 400: + description: "The source could not be removed" + 404: + description: "No such source for user" + security: + - auth0_jwt: [] "/products": get: description: "Get all products for the user." @@ -573,4 +595,4 @@ securityDefinitions: type: "oauth2" x-google-issuer: "https://ostelco.eu.auth0.com/" x-google-jwks_uri: "https://ostelco.eu.auth0.com/.well-known/jwks.json" - x-google-audiences: "http://google_api" \ No newline at end of file + x-google-audiences: "http://google_api" diff --git a/prime/infra/prod/metrics-api.yaml b/prime/infra/prod/metrics-api.yaml new file mode 100644 index 000000000..7badcd6a5 --- /dev/null +++ b/prime/infra/prod/metrics-api.yaml @@ -0,0 +1,30 @@ +type: google.api.Service + +config_version: 3 + +name: metrics.ostelco.org + +title: Prime Metrics Reporter Service gRPC API + +apis: + - name: org.ostelco.prime.metrics.api.OcsgwAnalyticsService + +usage: + rules: + # All methods can be called without an API Key. + - selector: "*" + allow_unregistered_calls: true + +authentication: + providers: + - id: google_service_account + issuer: prime-service-account@pantel-2decb.iam.gserviceaccount.com + jwks_uri: https://www.googleapis.com/robot/v1/metadata/x509/prime-service-account@pantel-2decb.iam.gserviceaccount.com + audiences: > + https://metrics.ostelco.org/org.ostelco.prime.metrics.api.OcsgwAnalyticsService, + metrics.ostelco.org/org.ostelco.prime.metrics.api.OcsgwAnalyticsService, + metrics.ostelco.org + rules: + - selector: "*" + requirements: + - provider_id: google_service_account \ No newline at end of file diff --git a/prime/infra/prod/prime-client-api.yaml b/prime/infra/prod/prime-client-api.yaml index 2312044bf..023650751 100644 --- a/prime/infra/prod/prime-client-api.yaml +++ b/prime/infra/prod/prime-client-api.yaml @@ -151,6 +151,28 @@ paths: description: "User not found." security: - auth0_jwt: [] + delete: + description: "Remove a payment source for user" + produces: + - application/json + operationId: "removeSource" + parameters: + - name: sourceId + in: query + description: "The stripe-id of the source to be removed" + required: true + type: string + responses: + 200: + description: "Successfully removed the source" + schema: + $ref: '#/definitions/PaymentSource' + 400: + description: "The source could not be removed" + 404: + description: "No such source for user" + security: + - auth0_jwt: [] "/products": get: description: "Get all products for the user." diff --git a/prime/infra/prod/prime.yaml b/prime/infra/prod/prime.yaml index 56189a91c..54691ae8a 100644 --- a/prime/infra/prod/prime.yaml +++ b/prime/infra/prod/prime.yaml @@ -37,6 +37,25 @@ spec: --- apiVersion: v1 kind: Service +metadata: + name: prime-metrics + labels: + app: prime + tier: backend +spec: + type: LoadBalancer + loadBalancerIP: 35.240.23.167 + ports: + - name: grpc + port: 443 + targetPort: 9443 + protocol: TCP + selector: + app: prime + tier: backend +--- +apiVersion: v1 +kind: Service metadata: name: pseudonym-server-service labels: @@ -71,7 +90,7 @@ spec: prometheus.io/port: '8081' spec: containers: - - name: esp + - name: ocs-esp image: gcr.io/endpoints-release/endpoints-runtime:1 args: [ "--http2_port=9000", @@ -105,6 +124,23 @@ spec: - mountPath: /etc/nginx/ssl name: api-ostelco-ssl readOnly: true + - name: metrics-esp + image: gcr.io/endpoints-release/endpoints-runtime:1 + args: [ + "--http2_port=9004", + "--ssl_port=9443", + "--status_port=8094", + "--service=metrics.ostelco.org", + "--rollout_strategy=managed", + "--backend=grpc://127.0.0.1:8083" + ] + ports: + - containerPort: 9004 + - containerPort: 9443 + volumeMounts: + - mountPath: /etc/nginx/ssl + name: metrics-ostelco-ssl + readOnly: true - name: prime image: eu.gcr.io/pantel-2decb/prime:PRIME_VERSION imagePullPolicy: Always @@ -117,6 +153,8 @@ spec: value: data-traffic - name: PURCHASE_INFO_TOPIC value: purchase-info + - name: ACTIVE_USERS_TOPIC + value: active-users - name: STRIPE_API_KEY valueFrom: secretKeyRef: @@ -130,6 +168,7 @@ spec: - containerPort: 8080 - containerPort: 8081 - containerPort: 8082 + - containerPort: 8083 volumes: - name: secret-config secret: @@ -140,3 +179,6 @@ spec: - name: ocs-ostelco-ssl secret: secretName: ocs-ostelco-ssl + - name: metrics-ostelco-ssl + secret: + secretName: metrics-ostelco-ssl diff --git a/prime/infra/raw_activeusers.ddl b/prime/infra/raw_activeusers.ddl new file mode 100644 index 000000000..7279d883a --- /dev/null +++ b/prime/infra/raw_activeusers.ddl @@ -0,0 +1,23 @@ +# Table for dev cluster +CREATE TABLE ocs_gateway_dev.raw_activeusers + ( + timestamp TIMESTAMP NOT NULL, + users ARRAY< STRUCT< + msisdn STRING NOT NULL, + apn STRING NOT NULL, + mccMnc STRING NOT NULL + > > +) +PARTITION BY DATE(_PARTITIONTIME) + +# Table for production cluster +CREATE TABLE ocs_gateway.raw_activeusers + ( + timestamp TIMESTAMP NOT NULL, + users ARRAY< STRUCT< + msisdn STRING NOT NULL, + apn STRING NOT NULL, + mccMnc STRING NOT NULL + > > +) +PARTITION BY DATE(_PARTITIONTIME) \ No newline at end of file diff --git a/prime/src/integration-tests/kotlin/org/ostelco/prime/TestPrimeConfig.kt b/prime/src/integration-tests/kotlin/org/ostelco/prime/TestPrimeConfig.kt index ef1885e27..f90e11ae6 100644 --- a/prime/src/integration-tests/kotlin/org/ostelco/prime/TestPrimeConfig.kt +++ b/prime/src/integration-tests/kotlin/org/ostelco/prime/TestPrimeConfig.kt @@ -39,7 +39,7 @@ class TestPrimeConfig { HealthChecks.toRespond2xxOverHttp(7474) { port -> port.inFormat("http://\$HOST:\$EXTERNAL_PORT/browser") }, - Duration.standardSeconds(20L)) + Duration.standardSeconds(40L)) .build() @JvmStatic diff --git a/prime/src/integration-tests/kotlin/org/ostelco/prime/ocs/OcsTest.kt b/prime/src/integration-tests/kotlin/org/ostelco/prime/ocs/OcsTest.kt index f7c47c525..ab6e08749 100644 --- a/prime/src/integration-tests/kotlin/org/ostelco/prime/ocs/OcsTest.kt +++ b/prime/src/integration-tests/kotlin/org/ostelco/prime/ocs/OcsTest.kt @@ -217,7 +217,7 @@ class OcsTest { HealthChecks.toRespond2xxOverHttp(7474) { port -> port.inFormat("http://\$HOST:\$EXTERNAL_PORT/browser") }, - Duration.standardSeconds(20L)) + Duration.standardSeconds(40L)) .build() @BeforeClass diff --git a/prime/src/integration-tests/kotlin/org/ostelco/prime/storage/graph/Neo4jStorageTest.kt b/prime/src/integration-tests/kotlin/org/ostelco/prime/storage/graph/Neo4jStorageTest.kt index bc960b646..2dd16ae3d 100644 --- a/prime/src/integration-tests/kotlin/org/ostelco/prime/storage/graph/Neo4jStorageTest.kt +++ b/prime/src/integration-tests/kotlin/org/ostelco/prime/storage/graph/Neo4jStorageTest.kt @@ -36,7 +36,7 @@ class Neo4jStorageTest { sleep(MILLIS_TO_WAIT_WHEN_STARTING_UP.toLong()) storage.removeSubscriber(EPHERMERAL_EMAIL) - storage.addSubscriber(Subscriber(EPHERMERAL_EMAIL), referredBy = null) + storage.addSubscriber(Subscriber(EPHERMERAL_EMAIL, country = COUNTRY), referredBy = null) .mapLeft { fail(it.message) } storage.addSubscription(EPHERMERAL_EMAIL, MSISDN) .mapLeft { fail(it.message) } @@ -90,6 +90,7 @@ class Neo4jStorageTest { private const val EPHERMERAL_EMAIL = "attherate@dotcom.com" private const val MSISDN = "4747116996" + private const val COUNTRY = "NO" private const val MILLIS_TO_WAIT_WHEN_STARTING_UP = 3000 @@ -104,7 +105,7 @@ class Neo4jStorageTest { HealthChecks.toRespond2xxOverHttp(7474) { port -> port.inFormat("http://\$HOST:\$EXTERNAL_PORT/browser") }, - Duration.standardSeconds(30L)) + Duration.standardSeconds(40L)) .build() @JvmStatic diff --git a/prime/src/integration-tests/resources/config.yaml b/prime/src/integration-tests/resources/config.yaml index 123175946..dce9318e8 100644 --- a/prime/src/integration-tests/resources/config.yaml +++ b/prime/src/integration-tests/resources/config.yaml @@ -12,6 +12,7 @@ modules: projectId: pantel-2decb dataTrafficTopicId: data-traffic purchaseInfoTopicId: purchase-info + activeUsersTopicId: active-users - type: ocs config: lowBalanceThreshold: 0 diff --git a/pseudonym-server/build.gradle b/pseudonym-server/build.gradle index eb1a06383..a6ceecd35 100644 --- a/pseudonym-server/build.gradle +++ b/pseudonym-server/build.gradle @@ -1,5 +1,5 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "java-library" } diff --git a/sample-agent/README.md b/sample-agent/README.md new file mode 100644 index 000000000..a3582e70c --- /dev/null +++ b/sample-agent/README.md @@ -0,0 +1,19 @@ + +This is a sample of how an analytic agent could be made +=== + +The objective of an anlytic agent is to take input from the exporter +module, do something with it, and then import into the admin-API's +"importer" interface with the result that new offers are made +available to whomever the agent decides it should be made available +to. + +The code in this directory is intended as a proof of concept that +it is indeed possible to export customer data, use it for something, +and use it to produce segments and offers. + +The functions being tested are the pseudo-anonymization happening in +the exporter and importers, and also the general "ergonomics" of the +agent. If it's not something we believe can be used by a competent +person of an external organization after a two-minute setup time, +then it's not good enugh. diff --git a/sample-agent/check_dependencies_get_environment_coordinates.sh b/sample-agent/check_dependencies_get_environment_coordinates.sh new file mode 100644 index 000000000..7e4ff7a18 --- /dev/null +++ b/sample-agent/check_dependencies_get_environment_coordinates.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +## Intended to be sourced by other programs + + +# +# Check for dependencies +# + +if [[ -z "$DEPENDENCIES ]] ; then + + DEPENDENCIES="gcloud kubectl gsutil" + + for dep in $DEPENDENCIES ; do + if [[ -z $(which $dep) ]] ; then + echo "ERROR: Could not find dependency $dep" + fi + done +fi + +# +# Figure out relevant parts of the environment and check their +# sanity. +# + +if [[ -z "$PROJECT_ID" ]] ; then + PROJECT_ID=$(gcloud config get-value project) + + if [[ -z "$PROJECT_ID" ]] ; then + echo "ERROR: Unknown google project ID" + exit 1 + fi +fi + +if [[ -z "$EXPORTER_PODNAME" ]] ; then + EXPORTER_PODNAME=$(kubectl get pods | grep exporter- | awk '{print $1}') + if [[ -z "$EXPORTER_PODNAME" ]] ; then + echo "ERROR: Unknown exporter podname" + exit 1 + fi +fi diff --git a/sample-agent/run-export.sh b/sample-agent/run-export.sh new file mode 100755 index 000000000..b7fb26826 --- /dev/null +++ b/sample-agent/run-export.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +## +## Run an export, return the identifier for the export, put the +## files from the export in a directory denoted as the single +## command line parameter. +## + + + +# Absolute path to this script, e.g. /home/user/bin/foo.sh +SCRIPT=$(readlink -f "$0") +# Absolute path this script is in, thus /home/user/bin +SCRIPTPATH=$(dirname "$SCRIPT") +echo $SCRIPTPATH + + +# +# Get command line parameter, which should be an existing +# directory in which to store the results +# + +TARGET_DIR=$1 +if [[ -z "$TARGET_DIR" ]] ; then + echo "$0 Missing parameter" + echo "usage $0 target-dir" + exit 1 +fi + +if [[ ! -d "$TARGET_DIR" ]] ; then + echo "$0 parameter does not designate an existing directory" + echo "usage $0 target-dir" + exit 1 +fi + +$SCRIPTPATH/check_dependencies_get_environment_coordinates.sh + +# +# Run an export inside the kubernetes cluster, then parse +# the output of the script thar ran the export +# +#TEMPFILE="$(mktemp /tmp/abc-script.XXXXXX)" +TEMPFILE="tmpfile.txt" + +kubectl exec -it "${EXPORTER_PODNAME}" -- /bin/bash -c /export_data.sh > "$TEMPFILE" + +# Fail if the exec failed +retVal=$? +if [ $retVal -ne 0 ]; then + echo "ERROR: Failed to export data:" + cat $TMPFILE + rm $TMPFILE + exit 1 +fi + +# +# Parse the output of the tmpfile, getting the export ID, and +# the google filestore URLs for the output files. +# + + +EXPORT_ID=$(grep "Starting export job for" $TEMPFILE | awk '{print $5}' | sed 's/\r$//' ) + +PURCHASES_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID-purchases.csv" +SUB_2_MSISSDN_MAPPING_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID-sub2msisdn.csv" +CONSUMPTION_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID.csv" + +# +# Then copy the CSV files to local storage (current directory) +# + +gsutil cp $PURCHASES_GS $TARGET_DIR +gsutil cp $SUB_2_MSISSDN_MAPPING_GS $TARGET_DIR +gsutil cp $CONSUMPTION_GS $TARGET_DIR + +# +# Clean up the tempfile +# + +rm "$TEMPFILE" + +# +# Finally output the ID of the export, since that's +# what will be used by users of this script to access +# the output +# + +echo $EXPORT_ID +exit 0 diff --git a/sample-agent/sample-agent.sh b/sample-agent/sample-agent.sh new file mode 100644 index 000000000..c2f683400 --- /dev/null +++ b/sample-agent/sample-agent.sh @@ -0,0 +1,322 @@ +#!/bin/bash + +set -e + +### +### VALIDATING AND PARSING COMMAND LINE PARAMETERS +### + +# +# Get command line parameter, which should be an existing +# directory in which to store the results +# + +TARGET_DIR=$1 +if [[ -z "$TARGET_DIR" ]] ; then + echo "$0 Missing parameter" + echo "usage $0 target-dir" + exit 1 +fi + +if [[ ! -d "$TARGET_DIR" ]] ; then + echo "$0 $TARGET_DIR is not a directory" + echo "usage $0 target-dir" + exit 1 +fi + +### +### PRELIMINARIES +### + +# Be able to die from inside procedures + +trap "exit 1" TERM +export TOP_PID=$$ + +function die() { + kill -s TERM $TOP_PID +} + +# +# Check for dependencies being satisfied +# + +DEPENDENCIES="gcloud kubectl gsutil" + +for dep in $DEPENDENCIES ; do + if [[ -z $(which $dep) ]] ; then + echo "ERROR: Could not find dependency $dep" + fi +done + +# +# Figure out relevant parts of the environment and check their +# sanity. +# + +PROJECT_ID=$(gcloud config get-value project) + +if [[ -z "$PROJECT_ID" ]] ; then + echo "ERROR: Unknown google project ID" + exit 1 +fi + +EXPORTER_PODNAME=$(kubectl get pods | grep exporter- | awk '{print $1}') +if [[ -z "$EXPORTER_PODNAME" ]] ; then + echo "ERROR: Unknown exporter podname" + exit 1 +fi + +PRIME_PODNAME=$(kubectl get pods | grep prime- | awk '{print $1}') +if [[ -z "$PRIME_PODNAME" ]] ; then + echo "ERROR: Unknown prime podname" + exit 1 +fi + +echo "$0: Assuming that prime is running at $PRIME_PODNAME" +echo "$0: and that you have done" +echo "$0: kubectl port-forward $PRIME_PODNAME 8080:8080" + + +### +### COMMUNICATION WITH EXPORTER SCRIPTS RUNNING IN A KUBERNETES PODS +### + +# +# Run named script on the inside of the kubernetes exporter pod, +# put the output from running that script into a temporary file, return the +# name of that temporary file as the result of running the function. +# The second argument is the intent of the invocation, and is used +# when producing error messages: +# +# runScriptOnExporterPod /export_data.sh "export data" +# +function runScriptOnExporterPod { + if [[ $# -ne 2 ]] ; then + echo "$0 ERROR: runScriptOnExporterPod requires exactly two parameters" + fi + local scriptname=$1 + local intentDescription=$2 + + # TEMPFILE="$(mktemp /tmp/abc-script.XXXXXX)" + # XXX The tmpfile is the same thing all the time, bad practice, but + # until I figure out how to make tempfiles dependent on the top + # level process's lifetime, I'll do it this way. + TEMPFILE="tmpfile.txt" + [[ -f "$TMPFILE" ]] && rm "$TMPFILE" + + kubectl exec -it "${EXPORTER_PODNAME}" -- /bin/bash -c "$scriptname" > "$TEMPFILE" + + # Fail if the exec failed + retVal=$? + if [[ $retVal -ne 0 ]]; then + echo "ERROR: Failed to $intentDescription" + cat $TEMPFILE + rm $TEMPFILE + die + fi + + # Return result by setting resutlvar to be the temporary filename + echo $TEMPFILE +} + + +# +# Create a data export batch, return a string identifying that +# batch. Typical usage: +# EXPORT_ID=$(exportDataFromExporterPod) +# +function exportDataFromExporterPod { + local tmpfilename="$(runScriptOnExporterPod /export_data.sh "export data")" + if [[ -z "$tmpfilename" ]] ; then + echo "$0 ERROR: Running the runScriptOnExporterPod failed to return the name of a resultfile." + die + fi + + local exportId="$(grep "Starting export job for" $tmpfilename | awk '{print $5}' | sed 's/\r$//' )" + + if [[ -z "$exportId" ]] ; then + echo "$0 Could not get export batch from exporter pod" + fi + rm $tmpfilename + echo $exportId +} + +function mapPseudosToUserids { + local tmpfile="$(runScriptOnExporterPod /map_subscribers.sh "mapping pseudoids to subscriber ids")" + [[ -f "$tmpfile" ]] && rm "$tmpfile" +} + +# +# Generate the Google filesystem names of components associated with +# a particular export ID: Typical usage +# +# PURCHASES_GS="$(gsExportCsvFilename "ab234245cvsr" "purchases")" + +function gsExportCsvFilename { + if [[ $# -ne 2 ]] ; then + echo "$0 ERROR: gsExportCsvFilename requires exactly two parameters, got '$@'" + die + fi + + local exportId=$1 + local componentName=$2 + if [[ -z "$exportId" ]] ; then + echo "$0 ERROR: gsExportCsvFilename got a null exportId" + die + fi + if [[ -n "$componentName" ]] ; then + componentName="-$componentName" + fi + + echo "gs://${PROJECT_ID}-dataconsumption-export/${exportId}${componentName}.csv" +} + + +# +# Generate a filename +# +function importedCsvFilename { + if [[ $# -ne 3 ]] ; then + echo "$0 ERROR: importedCsvFilename requires exactly three parameters, got $@" + die + fi + + local exportId=$1 + local importDirectory=$2 + local componentName=$3 + + if [[ -z "$exportId" ]] ; then + echo "$0 ERROR: importedCsvFilename got a null exportId" + die + fi + + if [[ -z "$importDirectory" ]] ; then + echo "$0 ERROR: importDirectory got a null exportId" + die + fi + + if [[ -n "$componentName" ]] ; then + componentName="-$componentName" + fi + + echo "${importDirectory}/${exportId}${componentName}.csv" +} + + +### +### MAIN SCRIPT +### + + + +EXPORT_ID="$(exportDataFromExporterPod)" +echo "EXPORT_ID = $EXPORT_ID" +if [[ -z "$EXPORT_ID" ]] ; then + echo "$0 ERROR: Could not determine export id" +fi + +# +# Copy all the export artifacts from gs:/ to local filesystem storage +# + +for component in "purchases" "sub2msisdn" "" ; do + + source="$(gsExportCsvFilename "$EXPORT_ID" "$component")" + if [[ -z "$source" ]] ; then + echo "$0 ERROR: Could not determine source file for export component '$component'" + fi + + destination="$(importedCsvFilename "$EXPORT_ID" "$TARGET_DIR" "$component")" + if [[ -z "$destination" ]] ; then + echo "$0 ERROR: Could not determine destination file for export component '$component'" + fi + + gsutil cp "$source" "$destination" +done + + +## +## Generate a sample segment by just ripping out +## all the subscriber IDs in the sub2msisdn file. +## +## This is clearly not a realistic scenario, much to simple +## but it is formally correct so it will serve as a placeholder +## until we get something more realistic. +## + +SEGMENT_TMPFILE_PSEUDO="$(importedCsvFilename "$EXPORT_ID" "$TARGET_DIR" "tmpsegment-pseudo")" +awk -F, '!/^subscriberId/{print $1'} $(importedCsvFilename "$EXPORT_ID" "$TARGET_DIR" "sub2msisdn") > $SEGMENT_TMPFILE_PSEUDO + + +## +## Convert from pseudos to actual IDs +## + + +RESULTSEG_PSEUDO_BASENAME="resultsegment-pseudoanonymized" +RESULTSEG_CLEARTEXT_BASENAME="resultsegment-cleartext" +RESULT_SEGMENT_PSEUDO_GS="$(gsExportCsvFilename "$EXPORT_ID" "$RESULTSEG_PSEUDO_BASENAME")" +RESULT_SEGMENT_CLEAR_GS="$(gsExportCsvFilename "$EXPORT_ID" "$RESULTSEG_CLEARTEXT_BASENAME")" +RESULT_SEGMENT_CLEAR="$(importedCsvFilename "$EXPORT_ID" "$TARGET_DIR" "$RESULTSEG_CLEARTEXT_BASENAME")" +RESULT_SEGMENT_SINGLE_COLUMN="$(importedCsvFilename "$EXPORT_ID" "$TARGET_DIR" "$RESULTSEG_CLEARTEXT_BASENAME")" + +# Copy the segment pseudo file to gs + +gsutil cp $SEGMENT_TMPFILE_PSEUDO $RESULT_SEGMENT_PSEUDO_GS + +# Then run the script that will convert it into a none-anonymized +# file and fetch the results from gs:/ +mapPseudosToUserids + +gsutil cp "$RESULT_SEGMENT_CLEAR_GS" "$RESULT_SEGMENT_CLEAR" + +echo "Just placed the results in the file $RESULT_SEGMENT_CLEAR" +# Then extract only the column we need (the real userids) + +awk -F, '!/^pseudoId/{print $2'} $RESULT_SEGMENT_CLEAR > $RESULT_SEGMENT_SINGLE_COLUMN + + +## +## Generate the yaml output +## + + +IMPORTFILE_YML=tmpfile.yml + +cat > $IMPORTFILE_YML <> $IMPORTFILE_YML + +## +## Send it to the importer +## (assuming the kubectl port forwarding is enabled) + +IMPORTER_URL=http://127.0.0.1:8080/importer +curl --data-binary @$IMPORTFILE_YML $IMPORTER_URL + + +## +## Remove tempfiles +## + +# .... eventually + + diff --git a/sample-agent/set-gs-names.sh b/sample-agent/set-gs-names.sh new file mode 100644 index 000000000..34075a239 --- /dev/null +++ b/sample-agent/set-gs-names.sh @@ -0,0 +1,20 @@ +#!/bin/bash + + +if [[ -z "$PROJECT_ID" ]] ; then + echo "$0 PROJECT_ID variable not set, cannot determine google filestore coordinates" + exit 1 +fi + + +if [[ -z "$EXPORT_ID" ]] ; then + echo "$0 EXPORT_ID variable not set, cannot determine google filestore coordinates" + exit 1 +fi + + +PURCHASES_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID-purchases.csv" +SUB_2_MSISSDN_MAPPING_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID-sub2msisdn.csv" +CONSUMPTION_GS="gs://${PROJECT_ID}-dataconsumption-export/$EXPORT_ID.csv" +RESULT_SEGMENT_PSEUDO_GS="gs://${PROJECT_ID}-dataconsumption-export/${EXPORT_ID}-resultsegment-pseudoanonymized.csv" +RESULT_SEGMENT_CLEAR_GS="gs://${PROJECT_ID}-dataconsumption-export/${EXPORT_ID}-resultsegment-cleartext.csv" diff --git a/scripts/deploy-ocsgw.sh b/scripts/deploy-ocsgw.sh index ad84ea7c5..31f65f514 100755 --- a/scripts/deploy-ocsgw.sh +++ b/scripts/deploy-ocsgw.sh @@ -14,6 +14,15 @@ if [ "$1" = prod ] ; then variant=prod fi +echo "Starting update.." +echo "Creating zip files" +gradle pack + +if [ ! -f build/deploy/ostelco-core-${variant}.zip ]; then + echo "build/deploy/ostelco-core-${variant}.zip not found!" + exit 1 +fi + echo "Starting to deploy OCSGW to $variant" echo "The last thing this script will do is to look at logs from the ocsgw" echo "It will continue to do so until terminated by ^C" diff --git a/tools/neo4j-admin-tools/build.gradle b/tools/neo4j-admin-tools/build.gradle index 017a5f34c..c22bf1e04 100644 --- a/tools/neo4j-admin-tools/build.gradle +++ b/tools/neo4j-admin-tools/build.gradle @@ -1,7 +1,7 @@ plugins { - id "org.jetbrains.kotlin.jvm" version "1.2.70" + id "org.jetbrains.kotlin.jvm" version "1.2.71" id "application" - id "com.github.johnrengelman.shadow" version "2.0.4" + id "com.github.johnrengelman.shadow" version "4.0.0" id "idea" } diff --git a/tools/neo4j-admin-tools/src/main/kotlin/org/ostelco/tools/migration/MainKt.kt b/tools/neo4j-admin-tools/src/main/kotlin/org/ostelco/tools/migration/MainKt.kt index bb57ed0bd..d1a32f104 100644 --- a/tools/neo4j-admin-tools/src/main/kotlin/org/ostelco/tools/migration/MainKt.kt +++ b/tools/neo4j-admin-tools/src/main/kotlin/org/ostelco/tools/migration/MainKt.kt @@ -5,8 +5,8 @@ import java.nio.file.Files import java.nio.file.Paths fun main(args: Array) { - neo4jExporterToCypherFile() - // cypherFileToNeo4jImporter() + // neo4jExporterToCypherFile() + cypherFileToNeo4jImporter() } fun neo4jExporterToCypherFile() { @@ -40,7 +40,7 @@ fun cypherFileToNeo4jImporter() { println("Import from file to Neo4j") - importFromCypherFile("src/main/resources/backup.prod.cypher") { query -> + importFromCypherFile("src/main/resources/init.cypher") { query -> txn.run(query) } diff --git a/tools/neo4j-admin-tools/src/main/resources/init.cypher b/tools/neo4j-admin-tools/src/main/resources/init.cypher index d6bbbf15d..d66abcab9 100644 --- a/tools/neo4j-admin-tools/src/main/resources/init.cypher +++ b/tools/neo4j-admin-tools/src/main/resources/init.cypher @@ -1,4 +1,4 @@ -// Create product +// For country:NO CREATE (:Product {`id`: '1GB_249NOK', `presentation/isDefault`: 'true', `presentation/offerLabel`: 'Default Offer', @@ -36,47 +36,74 @@ CREATE (:Product {`id`: '5GB_399NOK', `properties/noOfBytes`: '5_000_000_000', `sku`: '5GB_399NOK'}); -CREATE (:Product {`id`: '100MB_FREE_ON_JOINING', - `presentation/priceLabel`: 'Free', - `presentation/productLabel`: '100MB Welcome Pack', - `price/amount`: '0', - `price/currency`: 'NOK', - `properties/noOfBytes`: '100_000_000', - `sku`: '100MB_FREE_ON_JOINING'}); - -CREATE (:Product {`id`: '1GB_FREE_ON_REFERRED', - `presentation/priceLabel`: 'Free', - `presentation/productLabel`: '1GB Referral Pack', - `price/amount`: '0', - `price/currency`: 'NOK', - `properties/noOfBytes`: '1_000_000_000', - `sku`: '1GB_FREE_ON_REFERRED'}); - -CREATE (:Segment {`id`: 'all'}); +CREATE (:Segment {`id`: 'country-no'}); -CREATE (:Offer {`id`: 'default_offer'}); +CREATE (:Offer {`id`: 'default_offer-no'}); -MATCH (n:Offer {id: 'default_offer'}) +MATCH (n:Offer {id: 'default_offer-no'}) WITH n MATCH (m:Product {id: '1GB_249NOK'}) CREATE (n)-[:OFFER_HAS_PRODUCT]->(m); -MATCH (n:Offer {id: 'default_offer'}) +MATCH (n:Offer {id: 'default_offer-no'}) WITH n MATCH (m:Product {id: '2GB_299NOK'}) CREATE (n)-[:OFFER_HAS_PRODUCT]->(m); -MATCH (n:Offer {id: 'default_offer'}) +MATCH (n:Offer {id: 'default_offer-no'}) WITH n MATCH (m:Product {id: '3GB_349NOK'}) CREATE (n)-[:OFFER_HAS_PRODUCT]->(m); -MATCH (n:Offer {id: 'default_offer'}) +MATCH (n:Offer {id: 'default_offer-no'}) WITH n MATCH (m:Product {id: '5GB_399NOK'}) CREATE (n)-[:OFFER_HAS_PRODUCT]->(m); -MATCH (n:Offer {id: 'default_offer'}) +MATCH (n:Offer {id: 'default_offer-no'}) WITH n -MATCH (m:Segment {id: 'all'}) -CREATE (n)-[:OFFERED_TO_SEGMENT]->(m); \ No newline at end of file +MATCH (m:Segment {id: 'country-no'}) +CREATE (n)-[:OFFERED_TO_SEGMENT]->(m); + +// For country:SG +CREATE (:Product {`id`: '1GB_1SGD', + `presentation/isDefault`: 'true', + `presentation/offerLabel`: 'Default Offer', + `presentation/priceLabel`: '1 SGD', + `presentation/productLabel`: '+1GB', + `price/amount`: '100', + `price/currency`: 'SGD', + `properties/noOfBytes`: '1_000_000_000', + `sku`: '1GB_1SGD'}); + +CREATE (:Segment {`id`: 'country-sg'}); + +CREATE (:Offer {`id`: 'default_offer-sg'}); + +MATCH (n:Offer {id: 'default_offer-sg'}) +WITH n +MATCH (m:Product {id: '1GB_1SGD'}) +CREATE (n)-[:OFFER_HAS_PRODUCT]->(m); + +MATCH (n:Offer {id: 'default_offer-sg'}) +WITH n +MATCH (m:Segment {id: 'country-sg'}) +CREATE (n)-[:OFFERED_TO_SEGMENT]->(m); + +// Generic +CREATE (:Product {`id`: '100MB_FREE_ON_JOINING', + `presentation/priceLabel`: 'Free', + `presentation/productLabel`: '100MB Welcome Pack', + `price/amount`: '0', + `price/currency`: '', + `properties/noOfBytes`: '100_000_000', + `sku`: '100MB_FREE_ON_JOINING'}); + +CREATE (:Product {`id`: '1GB_FREE_ON_REFERRED', + `presentation/priceLabel`: 'Free', + `presentation/productLabel`: '1GB Referral Pack', + `price/amount`: '0', + `price/currency`: '', + `properties/noOfBytes`: '1_000_000_000', + `sku`: '1GB_FREE_ON_REFERRED'}); +