diff --git a/README.md b/README.md index 804ce931b..ae502309c 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,6 @@ Mono Repository for core protocols and services around a OCS/BSS for packet data * [diameter-stack](./diameter-stack/README.md) * [diameter-test](./diameter-test/README.md) * [exporter](./exporter/README.md) - * [ext-pgw](./ext-pgw/README.md) * [ocs-api](./ocs-api/README.md) * [ocsgw](./ocsgw/README.md) * [ostelco-lib](./ostelco-lib/README.md) diff --git a/acceptance-tests/Dockerfile b/acceptance-tests/Dockerfile index 2217a0487..f897cb74a 100644 --- a/acceptance-tests/Dockerfile +++ b/acceptance-tests/Dockerfile @@ -6,6 +6,7 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends netcat \ && rm -rf /var/lib/apt/lists/* +COPY config/ /secret/ COPY src/main/resources/ / COPY script/wait.sh /wait.sh COPY build/libs/acceptance-tests-uber.jar /acceptance-tests.jar diff --git a/acceptance-tests/build.gradle b/acceptance-tests/build.gradle index 3b6a16248..312e6a21b 100644 --- a/acceptance-tests/build.gradle +++ b/acceptance-tests/build.gradle @@ -14,7 +14,9 @@ dependencies { implementation project(":prime-client-api") implementation project(':diameter-test') - implementation "com.stripe:stripe-java:6.8.0" + implementation 'com.google.firebase:firebase-admin:6.4.0' + + implementation "com.stripe:stripe-java:$stripeVersion" implementation 'io.jsonwebtoken:jjwt:0.9.1' // tests fail when updated to 2.27 implementation "org.glassfish.jersey.media:jersey-media-json-jackson:2.25.1" diff --git a/acceptance-tests/config/.gitignore b/acceptance-tests/config/.gitignore new file mode 100644 index 000000000..bf045303f --- /dev/null +++ b/acceptance-tests/config/.gitignore @@ -0,0 +1 @@ +pantel-prod.json \ No newline at end of file diff --git a/acceptance-tests/script/wait.sh b/acceptance-tests/script/wait.sh old mode 100755 new mode 100644 index 6a86cecfb..1d4dcbae6 --- a/acceptance-tests/script/wait.sh +++ b/acceptance-tests/script/wait.sh @@ -22,12 +22,14 @@ java -cp '/acceptance-tests.jar' org.junit.runner.JUnitCore \ org.ostelco.at.okhttp.GetPseudonymsTest \ org.ostelco.at.okhttp.GetProductsTest \ org.ostelco.at.okhttp.GetSubscriptionStatusTest \ + org.ostelco.at.okhttp.SourceTest \ org.ostelco.at.okhttp.PurchaseTest \ org.ostelco.at.okhttp.ConsentTest \ org.ostelco.at.okhttp.ProfileTest \ org.ostelco.at.jersey.GetPseudonymsTest \ org.ostelco.at.jersey.GetProductsTest \ org.ostelco.at.jersey.GetSubscriptionStatusTest \ + org.ostelco.at.jersey.SourceTest \ org.ostelco.at.jersey.PurchaseTest \ org.ostelco.at.jersey.AnalyticsTest \ org.ostelco.at.jersey.ConsentTest \ diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/common/Firebase.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/common/Firebase.kt new file mode 100644 index 000000000..35052c378 --- /dev/null +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/common/Firebase.kt @@ -0,0 +1,41 @@ +package org.ostelco.at.common + +import com.google.auth.oauth2.GoogleCredentials +import com.google.firebase.FirebaseApp +import com.google.firebase.FirebaseOptions +import com.google.firebase.database.FirebaseDatabase +import java.io.FileInputStream +import java.nio.file.Files +import java.nio.file.Paths + +object Firebase { + + private fun setupFirebaseInstance(): FirebaseDatabase { + + try { + FirebaseApp.getInstance() + } catch (e: Exception) { + val databaseName = "pantel-2decb" + val configFile = System.getenv("GOOGLE_APPLICATION_CREDENTIALS") ?: "config/pantel-prod.json" + + val credentials: GoogleCredentials = if (Files.exists(Paths.get(configFile))) { + FileInputStream(configFile).use { serviceAccount -> GoogleCredentials.fromStream(serviceAccount) } + } else { + throw Exception() + } + + val options = FirebaseOptions.Builder() + .setCredentials(credentials) + .setDatabaseUrl("https://$databaseName.firebaseio.com/") + .build() + + FirebaseApp.initializeApp(options) + } + + return FirebaseDatabase.getInstance() + } + + fun deleteAllPaymentCustomers() { + setupFirebaseInstance().getReference("test/paymentId").removeValueAsync().get() + } +} \ No newline at end of file diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt index 42602b971..55351aa4d 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt @@ -2,10 +2,12 @@ package org.ostelco.at.common import com.stripe.Stripe import com.stripe.model.Customer +import com.stripe.model.Source import com.stripe.model.Token object StripePayment { - fun createPaymentSourceId(): String { + + fun createPaymentTokenId(): String { // https://stripe.com/docs/api/java#create_card_token Stripe.apiKey = System.getenv("STRIPE_API_KEY") @@ -21,6 +23,53 @@ object StripePayment { return token.id } + fun createPaymentSourceId(): String { + + // https://stripe.com/docs/api/java#create_source + Stripe.apiKey = System.getenv("STRIPE_API_KEY") + + // TODO martin: set valid map values + val sourceMap = mapOf() + val source = Source.create(sourceMap) + return source.id + } + + fun getCardIdForTokenId(tokenId: String) : String { + + // https://stripe.com/docs/api/java#create_source + Stripe.apiKey = System.getenv("STRIPE_API_KEY") + + val token = Token.retrieve(tokenId) + return token.card.id + } + + /** + * Obtains 'default source' directly from Stripe. Use in tests to + * verify that the correspondng 'setDefaultSource' API works as + * intended. + */ + fun getDefaultSourceForCustomer(customerId: String) : String { + + // https://stripe.com/docs/api/java#create_source + Stripe.apiKey = System.getenv("STRIPE_API_KEY") + + val customer = Customer.retrieve(customerId) + return customer.defaultSource + } + + /** + * Obtains the Stripe 'customerId' directly from Stripe. + */ + fun getCustomerIdForEmail(email: String) : String { + + // https://stripe.com/docs/api/java#create_card_token + Stripe.apiKey = System.getenv("STRIPE_API_KEY") + + val customers = Customer.list(emptyMap()).data + + return customers.filter { it.email.equals(email) }.first().id + } + fun deleteAllCustomers() { // https://stripe.com/docs/api/java#create_card_token Stripe.apiKey = System.getenv("STRIPE_API_KEY") diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt index 91186ed16..12b6a676c 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt @@ -1,6 +1,7 @@ package org.ostelco.at.jersey import org.junit.Test +import org.ostelco.at.common.Firebase import org.ostelco.at.common.StripePayment import org.ostelco.at.common.createProfile import org.ostelco.at.common.createSubscription @@ -10,6 +11,8 @@ import org.ostelco.at.common.randomInt import org.ostelco.prime.client.model.ActivePseudonyms import org.ostelco.prime.client.model.ApplicationToken import org.ostelco.prime.client.model.Consent +import org.ostelco.prime.client.model.PaymentSource +import org.ostelco.prime.client.model.PaymentSourceList import org.ostelco.prime.client.model.Person import org.ostelco.prime.client.model.Price import org.ostelco.prime.client.model.Product @@ -223,12 +226,140 @@ class GetProductsTest { } } +class SourceTest { + + @Test + fun `jersey test - POST source create`() { + + StripePayment.deleteAllCustomers() + Firebase.deleteAllPaymentCustomers() + + val email = "purchase-${randomInt()}@test.com" + createProfile(name = "Test Payment Source", email = email) + + val tokenId = StripePayment.createPaymentTokenId() + + // Ties source with user profile both local and with Stripe + post { + path = "/paymentSources" + subscriberId = email + queryParams = mapOf("sourceId" to tokenId) + } + + Thread.sleep(200) + + val sources: PaymentSourceList = get { + path = "/paymentSources" + subscriberId = email + } + assert(sources.isNotEmpty()) { "Expected at least one payment source for profile $email" } + + val cardId = StripePayment.getCardIdForTokenId(tokenId) + assertNotNull(sources.first { it.id == cardId }, "Expected card $cardId in list of payment sources for profile $email") + } + + @Test + fun `okhttp test - GET list sources`() { + + StripePayment.deleteAllCustomers() + Firebase.deleteAllPaymentCustomers() + + val email = "purchase-${randomInt()}@test.com" + createProfile(name = "Test Payment Source", email = email) + + val tokenId = StripePayment.createPaymentTokenId() + val cardId = StripePayment.getCardIdForTokenId(tokenId) + + // Ties source with user profile both local and with Stripe + post { + path = "/paymentSources" + subscriberId = email + queryParams = mapOf("sourceId" to tokenId) + } + + Thread.sleep(200) + + val newTokenId = StripePayment.createPaymentTokenId() + val newCardId = StripePayment.getCardIdForTokenId(newTokenId) + + post { + path = "/paymentSources" + subscriberId = email + queryParams = mapOf("sourceId" to newTokenId) + } + + val sources : PaymentSourceList = get { + path = "/paymentSources" + subscriberId = email + } + + assert(sources.isNotEmpty()) { "Expected at least one payment source for profile $email" } + assert(sources.map{ it.id }.containsAll(listOf(cardId, newCardId))) + { "Expected to find both $cardId and $newCardId in list of sources for profile $email" } + + sources.forEach { + assert(it.details.id.isNotEmpty()) { "Expected 'id' to be set in source account details for profile $email" } + assertEquals("card", it.details.accountType, + "Unexpected source account type ${it.details.accountType} for profile $email") + } + } + + @Test + fun `jersey test - PUT source set default`() { + + StripePayment.deleteAllCustomers() + Firebase.deleteAllPaymentCustomers() + + val email = "purchase-${randomInt()}@test.com" + createProfile(name = "Test Payment Source", email = email) + + val tokenId = StripePayment.createPaymentTokenId() + val cardId = StripePayment.getCardIdForTokenId(tokenId) + + // Ties source with user profile both local and with Stripe + post { + path = "/paymentSources" + subscriberId = email + queryParams = mapOf("sourceId" to tokenId) + } + + Thread.sleep(200) + + val newTokenId = StripePayment.createPaymentTokenId() + val newCardId = StripePayment.getCardIdForTokenId(newTokenId) + + post { + path = "/paymentSources" + subscriberId = email + queryParams = mapOf("sourceId" to newTokenId) + } + + // TODO: Update to fetch the Stripe customerId from 'admin' API when ready. + val customerId = StripePayment.getCustomerIdForEmail(email) + + // Verify that original 'sourceId/card' is default. + assertEquals(cardId, StripePayment.getDefaultSourceForCustomer(customerId), + "Expected $cardId to be default source for $customerId") + + // Set new default card. + put { + path = "/paymentSources" + subscriberId = email + queryParams = mapOf("sourceId" to newCardId) + } + + assertEquals(newCardId, StripePayment.getDefaultSourceForCustomer(customerId), + "Expected $newCardId to be default source for $customerId") + } +} + class PurchaseTest { @Test fun `jersey test - POST products purchase`() { StripePayment.deleteAllCustomers() + Firebase.deleteAllPaymentCustomers() val email = "purchase-${randomInt()}@test.com" createProfile(name = "Test Purchase User", email = email) @@ -240,12 +371,65 @@ class PurchaseTest { val balanceBefore = subscriptionStatusBefore.remaining val productSku = "1GB_249NOK" - val sourceId = StripePayment.createPaymentSourceId() + val sourceId = StripePayment.createPaymentTokenId() + + post { + path = "/products/$productSku/purchase" + subscriberId = email + queryParams = mapOf("sourceId" to sourceId) + } + + Thread.sleep(100) // wait for 100 ms for balance to be updated in db + + val subscriptionStatusAfter: SubscriptionStatus = get { + path = "/subscription/status" + subscriberId = email + } + val balanceAfter = subscriptionStatusAfter.remaining + + assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") + + val purchaseRecords: PurchaseRecordList = get { + path = "/purchases" + subscriberId = email + } + + purchaseRecords.sortBy { it.timestamp } + + assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } + assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + } + + @Test + fun `jersey test - POST products purchase using default source`() { + + StripePayment.deleteAllCustomers() + Firebase.deleteAllPaymentCustomers() + + val email = "purchase-${randomInt()}@test.com" + createProfile(name = "Test Purchase User with Default Payment Source", email = email) + + val sourceId = StripePayment.createPaymentTokenId() + + val paymentSource: PaymentSource = post { + path = "/paymentSources" + subscriberId = email + queryParams = mapOf("sourceId" to sourceId) + } + + assertNotNull(paymentSource.id, message = "Failed to create payment source") + + val subscriptionStatusBefore: SubscriptionStatus = get { + path = "/subscription/status" + subscriberId = email + } + val balanceBefore = subscriptionStatusBefore.remaining + + val productSku = "1GB_249NOK" post { path = "/products/$productSku/purchase" subscriberId = email - queryParams = mapOf( "sourceId" to sourceId) } Thread.sleep(100) // wait for 100 ms for balance to be updated in db diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt index 673645bad..7a9e99ed3 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt @@ -1,6 +1,7 @@ package org.ostelco.at.okhttp import org.junit.Test +import org.ostelco.at.common.Firebase import org.ostelco.at.common.StripePayment import org.ostelco.at.common.createProfile import org.ostelco.at.common.createSubscription @@ -8,13 +9,21 @@ import org.ostelco.at.common.expectedProducts import org.ostelco.at.common.logger import org.ostelco.at.common.randomInt import org.ostelco.at.okhttp.ClientFactory.clientForSubject +import org.ostelco.prime.client.model.ApplicationToken import org.ostelco.prime.client.model.Consent +import org.ostelco.prime.client.model.PaymentSource +import org.ostelco.prime.client.model.Person +import org.ostelco.prime.client.model.PersonList import org.ostelco.prime.client.model.Price import org.ostelco.prime.client.model.Product import org.ostelco.prime.client.model.Profile +import org.ostelco.prime.client.model.SubscriptionStatus import java.time.Instant +import java.util.* import kotlin.test.assertEquals +import kotlin.test.assertFails import kotlin.test.assertNotNull +import kotlin.test.assertNull class ProfileTest { @@ -34,7 +43,7 @@ class ProfileTest { .postCode("") .referralId("") - client.createProfile(createProfile) + client.createProfile(createProfile, null) val profile: Profile = client.profile @@ -72,6 +81,32 @@ class ProfileTest { assertEquals("", clearedProfile.city, "Incorrect 'city' in response after clearing profile") assertEquals("", clearedProfile.country, "Incorrect 'country' in response after clearing profile") } + + @Test + fun `okhttp test - GET application token`() { + + val email = "token-${randomInt()}@test.com" + createProfile("Test Token User", email) + + createSubscription(email) + + val token = UUID.randomUUID().toString() + val applicationId = "testApplicationId" + val tokenType = "FCM" + + val testToken = ApplicationToken() + .token(token) + .applicationID(applicationId) + .tokenType(tokenType) + + val client = clientForSubject(subject = email) + + val reply = client.storeApplicationToken(testToken) + + assertEquals(token, reply.token, "Incorrect token in reply after posting new token") + assertEquals(applicationId, reply.applicationID, "Incorrect applicationId in reply after posting new token") + assertEquals(tokenType, reply.tokenType, "Incorrect tokenType in reply after posting new token") + } } class GetSubscriptions { @@ -163,12 +198,117 @@ class GetProductsTest { } } +class SourceTest { + + @Test + fun `okhttp test - POST source create`() { + + StripePayment.deleteAllCustomers() + Firebase.deleteAllPaymentCustomers() + + val email = "purchase-${randomInt()}@test.com" + createProfile(name = "Test Payment Source", email = email) + + val client = clientForSubject(subject = email) + + val tokenId = StripePayment.createPaymentTokenId() + val cardId = StripePayment.getCardIdForTokenId(tokenId) + + // Ties source with user profile both local and with Stripe + client.createSource(tokenId) + + Thread.sleep(200) + + val sources = client.listSources() + + assert(sources.isNotEmpty()) { "Expected at least one payment source for profile $email" } + assertNotNull(sources.first { it.id == cardId }, + "Expected card $cardId in list of payment sources for profile $email") + } + + @Test + fun `okhttp test - GET list sources`() { + + StripePayment.deleteAllCustomers() + Firebase.deleteAllPaymentCustomers() + + val email = "purchase-${randomInt()}@test.com" + createProfile(name = "Test Payment Source", email = email) + + val client = clientForSubject(subject = email) + + val tokenId = StripePayment.createPaymentTokenId() + val cardId = StripePayment.getCardIdForTokenId(tokenId) + + // Ties source with user profile both local and with Stripe + client.createSource(tokenId) + + Thread.sleep(200) + + val newTokenId = StripePayment.createPaymentTokenId() + val newCardId = StripePayment.getCardIdForTokenId(newTokenId) + + client.createSource(newTokenId) + + val sources = client.listSources() + + assert(sources.isNotEmpty()) { "Expected at least one payment source for profile $email" } + assert(sources.map{ it.id }.containsAll(listOf(cardId, newCardId))) + { "Expected to find both $cardId and $newCardId in list of sources for profile $email" } + + sources.forEach { + assert(it.details.id.isNotEmpty()) { "Expected 'id' to be set in source account details for profile $email" } + assertEquals("card", it.details.accountType, + "Unexpected source account type ${it.details.accountType} for profile $email") + } + } + + @Test + fun `okhttp test - PUT source set default`() { + + StripePayment.deleteAllCustomers() + Firebase.deleteAllPaymentCustomers() + + val email = "purchase-${randomInt()}@test.com" + createProfile(name = "Test Payment Source", email = email) + + val client = clientForSubject(subject = email) + + val tokenId = StripePayment.createPaymentTokenId() + val cardId = StripePayment.getCardIdForTokenId(tokenId) + + // Ties source with user profile both local and with Stripe + client.createSource(tokenId) + + Thread.sleep(200) + + val newTokenId = StripePayment.createPaymentTokenId() + val newCardId = StripePayment.getCardIdForTokenId(newTokenId) + + client.createSource(newTokenId) + + // TODO: Update to fetch the Stripe customerId from 'admin' API when ready. + val customerId = StripePayment.getCustomerIdForEmail(email) + + // Verify that original 'sourceId/card' is default. + assertEquals(cardId, StripePayment.getDefaultSourceForCustomer(customerId), + "Expected $cardId to be default source for $customerId") + + // Set new default card. + client.setDefaultSource(newCardId) + + assertEquals(newCardId, StripePayment.getDefaultSourceForCustomer(customerId), + "Expected $newCardId to be default source for $customerId") + } +} + class PurchaseTest { @Test fun `okhttp test - POST products purchase`() { StripePayment.deleteAllCustomers() + Firebase.deleteAllPaymentCustomers() val email = "purchase-${randomInt()}@test.com" createProfile(name = "Test Purchase User", email = email) @@ -177,7 +317,7 @@ class PurchaseTest { val balanceBefore = client.subscriptionStatus.remaining - val sourceId = StripePayment.createPaymentSourceId() + val sourceId = StripePayment.createPaymentTokenId() client.purchaseProduct("1GB_249NOK", sourceId, false) @@ -195,6 +335,43 @@ class PurchaseTest { assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") } + @Test + fun `okhttp test - POST products purchase using default source`() { + + StripePayment.deleteAllCustomers() + Firebase.deleteAllPaymentCustomers() + + val email = "purchase-${randomInt()}@test.com" + createProfile(name = "Test Purchase User with Default Payment Source", email = email) + + val sourceId = StripePayment.createPaymentTokenId() + + val client = clientForSubject(subject = email) + + val paymentSource: PaymentSource = client.createSource(sourceId) + + assertNotNull(paymentSource.id, message = "Failed to create payment source") + + val balanceBefore = client.subscriptionStatus.remaining + + val productSku = "1GB_249NOK" + + client.purchaseProduct(productSku, null, null) + + Thread.sleep(200) // wait for 200 ms for balance to be updated in db + + val balanceAfter = client.subscriptionStatus.remaining + + assertEquals(1_000_000_000, balanceAfter - balanceBefore, "Balance did not increased by 1GB after Purchase") + + val purchaseRecords = client.purchaseHistory + + purchaseRecords.sortBy { it.timestamp } + + assert(Instant.now().toEpochMilli() - purchaseRecords.last().timestamp < 10_000) { "Missing Purchase Record" } + assertEquals(expectedProducts().first(), purchaseRecords.last().product, "Incorrect 'Product' in purchase record") + } + @Test fun `okhttp test - POST products purchase without payment`() { @@ -240,7 +417,7 @@ class ConsentTest { assertEquals(consentId, defaultConsent[0].consentId, "Incorrect 'consent id' in fetched consent") // TODO vihang: Update consent operation is missing response entity - // val acceptedConsent: Consent = +// val acceptedConsent: Consent = client.updateConsent(consentId, true) // assertEquals(consentId, acceptedConsent.consentId, "Incorrect 'consent id' in response after accepting consent") @@ -252,4 +429,99 @@ class ConsentTest { // assertEquals(consentId, rejectedConsent.consentId, "Incorrect 'consent id' in response after rejecting consent") // assertFalse(rejectedConsent.isAccepted ?: true, "Accepted consent not reflected in response after rejecting consent") } +} + +class ReferralTest { + + @Test + fun `okhttp test - POST profile with invalid referred by`() { + + val email = "referred_by_invalid-${randomInt()}@test.com" + + val client = clientForSubject(subject = email) + + val invalid = "invalid_referrer@test.com" + + val profile = Profile() + .email(email) + .name("Test Referral Second User") + .address("") + .city("") + .country("") + .postCode("") + .referralId("") + + val failedToCreate = assertFails { + client.createProfile(profile, invalid) + } + + assertEquals(""" +{"description":"Incomplete profile description. Subscriber - $invalid not found."} expected:<201> but was:<403> + """.trimIndent(), failedToCreate.message) + + val failedToGet = assertFails { + client.profile + } + + assertEquals(""" +{"description":"Incomplete profile description. Subscriber - $email not found."} expected:<200> but was:<404> + """.trimIndent(), failedToGet.message) + } + + @Test + fun `okhttp test - POST profile`() { + + val firstEmail = "referral_first-${randomInt()}@test.com" + createProfile(name = "Test Referral First User", email = firstEmail) + + val secondEmail = "referral_second-${randomInt()}@test.com" + + val profile = Profile() + .email(secondEmail) + .name("Test Referral Second User") + .address("") + .city("") + .country("") + .postCode("") + .referralId("") + + val firstEmailClient = clientForSubject(subject = firstEmail) + val secondEmailClient = clientForSubject(subject = secondEmail) + + secondEmailClient.createProfile(profile, firstEmail) + + // for first + val referralsForFirst: PersonList = firstEmailClient.referred + + assertEquals(listOf("Test Referral Second User"), referralsForFirst.map { it.name }) + + val referredByForFirst: Person = firstEmailClient.referredBy + assertNull(referredByForFirst.name) + + // No need to test SubscriptionStatus for first, since it is already tested in GetSubscriptionStatusTest. + + // for referred_by_foo + val referralsForSecond: List = secondEmailClient.referred + + assertEquals(emptyList(), referralsForSecond.map { it.name }) + + val referredByForSecond: Person = secondEmailClient.referredBy + + assertEquals("Test Referral First User", referredByForSecond.name) + + val secondSubscriptionStatus: SubscriptionStatus = secondEmailClient.subscriptionStatus + + assertEquals(1_000_000_000, secondSubscriptionStatus.remaining) + + val freeProductForReferred = Product() + .sku("1GB_FREE_ON_REFERRED") + .price(Price().apply { + this.amount = 0 + this.currency = "NOK" + }) + .properties(mapOf("noOfBytes" to "1_000_000_000")) + .presentation(emptyMap()) + + assertEquals(listOf(freeProductForReferred), secondSubscriptionStatus.purchaseRecords.map { it.product }) + } } \ No newline at end of file diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/pgw/OcsTest.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/pgw/OcsTest.kt index 29297d83f..5e21982ac 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/pgw/OcsTest.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/pgw/OcsTest.kt @@ -150,7 +150,7 @@ class OcsTest { @Test - fun creditControlRequestInitNoCredit() { + fun creditControlRequestInitTerminateNoCredit() { val client = testClient ?: fail("Test client is null") @@ -181,8 +181,6 @@ class OcsTest { } // There is 2 step in graceful shutdown. First OCS send terminate, then P-GW report used units in a final update - assertEquals(INITIAL_BALANCE, getBalance(), message = "Incorrect balance after init using wrong msisdn") - val updateRequest = client.createRequest( DEST_REALM, DEST_HOST, @@ -208,7 +206,25 @@ class OcsTest { assertEquals(86400L, validTime.unsigned32) } - assertEquals(INITIAL_BALANCE, getBalance(), message = "Incorrect balance after update using wrong msisdn") + // Last step is user disconnecting connection forcing a terminate + val terminateRequest = client.createRequest( + DEST_REALM, + DEST_HOST, + session + ) ?: fail("Failed to create request") + TestHelper.createTerminateRequest(terminateRequest.avps, "4333333333") + + client.sendNextRequest(terminateRequest, session) + + waitForAnswer() + + run { + assertEquals(2001L, client.resultCodeAvp?.integer32?.toLong()) + val resultAvps = client.resultAvps ?: fail("Missing AVPs") + assertEquals(DEST_HOST, resultAvps.getAvp(Avp.ORIGIN_HOST).utF8String) + assertEquals(DEST_REALM, resultAvps.getAvp(Avp.ORIGIN_REALM).utF8String) + assertEquals(RequestType.TERMINATION_REQUEST.toLong(), resultAvps.getAvp(Avp.CC_REQUEST_TYPE).integer32.toLong()) + } } diff --git a/analytics-grpc-api/src/main/proto/analytics.proto b/analytics-grpc-api/src/main/proto/analytics.proto index 6ba8085b9..f2d68aa83 100644 --- a/analytics-grpc-api/src/main/proto/analytics.proto +++ b/analytics-grpc-api/src/main/proto/analytics.proto @@ -19,5 +19,5 @@ message DataTrafficInfo { message AggregatedDataTrafficInfo { string msisdn = 1; uint64 dataBytes = 2; - string dateTime = 3; + google.protobuf.Timestamp timestamp = 3; } \ No newline at end of file diff --git a/analytics-module/build.gradle b/analytics-module/build.gradle index adcf08f5a..74bea1607 100644 --- a/analytics-module/build.gradle +++ b/analytics-module/build.gradle @@ -9,10 +9,13 @@ dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion" implementation "io.dropwizard:dropwizard-core:$dropwizardVersion" implementation "com.google.cloud:google-cloud-pubsub:$googleCloudVersion" + implementation 'com.google.code.gson:gson:2.8.5' + //compile group: 'com.google.api', name: 'gax-grpc', version: '0.14.0' + testCompile group: 'com.google.api', name: 'gax-grpc', version: '1.30.0' testImplementation "io.dropwizard:dropwizard-testing:$dropwizardVersion" - testImplementation 'org.mockito:mockito-core:2.18.3' - testImplementation 'org.assertj:assertj-core:3.10.0' + testImplementation "org.mockito:mockito-core:$mockitoVersion" + testImplementation "org.assertj:assertj-core:$assertJVersion" } apply from: '../jacoco.gradle' diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt index 98b6a8ca8..1f0c7d0b8 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt @@ -6,6 +6,7 @@ import io.dropwizard.setup.Environment import org.hibernate.validator.constraints.NotEmpty import org.ostelco.prime.analytics.metrics.CustomMetricsRegistry import org.ostelco.prime.analytics.publishers.DataConsumptionInfoPublisher +import org.ostelco.prime.analytics.publishers.PurchaseInfoPublisher import org.ostelco.prime.module.PrimeModule @JsonTypeName("analytics") @@ -26,6 +27,7 @@ class AnalyticsModule : PrimeModule { // dropwizard starts Analytics events publisher env.lifecycle().manage(DataConsumptionInfoPublisher) + env.lifecycle().manage(PurchaseInfoPublisher) } } @@ -35,8 +37,12 @@ class AnalyticsConfig { lateinit var projectId: String @NotEmpty - @JsonProperty("topicId") - lateinit var topicId: String + @JsonProperty("dataTrafficTopicId") + lateinit var dataTrafficTopicId: String + + @NotEmpty + @JsonProperty("purchaseInfoTopicId") + lateinit var purchaseInfoTopicId: String } object ConfigRegistry { diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsServiceImpl.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsServiceImpl.kt index 99b9c34c5..7dafff765 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsServiceImpl.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsServiceImpl.kt @@ -2,7 +2,9 @@ package org.ostelco.prime.analytics import org.ostelco.prime.analytics.metrics.CustomMetricsRegistry import org.ostelco.prime.analytics.publishers.DataConsumptionInfoPublisher +import org.ostelco.prime.analytics.publishers.PurchaseInfoPublisher import org.ostelco.prime.logger +import org.ostelco.prime.model.PurchaseRecord class AnalyticsServiceImpl : AnalyticsService { @@ -16,4 +18,8 @@ class AnalyticsServiceImpl : AnalyticsService { override fun reportMetric(primeMetric: PrimeMetric, value: Long) { CustomMetricsRegistry.updateMetricValue(primeMetric, value) } + + override fun reportPurchaseInfo(purchaseRecord: PurchaseRecord, subscriberId: String, status: String) { + PurchaseInfoPublisher.publish(purchaseRecord, subscriberId, status) + } } \ No newline at end of file diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DataConsumptionInfoPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DataConsumptionInfoPublisher.kt index 674750198..0ea3850af 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DataConsumptionInfoPublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DataConsumptionInfoPublisher.kt @@ -3,53 +3,33 @@ package org.ostelco.prime.analytics.publishers import com.google.api.core.ApiFutureCallback import com.google.api.core.ApiFutures import com.google.api.gax.rpc.ApiException -import com.google.cloud.pubsub.v1.Publisher import com.google.protobuf.util.Timestamps -import com.google.pubsub.v1.ProjectTopicName import com.google.pubsub.v1.PubsubMessage -import io.dropwizard.lifecycle.Managed import org.ostelco.analytics.api.DataTrafficInfo -import org.ostelco.prime.analytics.ConfigRegistry.config +import org.ostelco.prime.analytics.ConfigRegistry import org.ostelco.prime.logger import org.ostelco.prime.module.getResource import org.ostelco.prime.pseudonymizer.PseudonymizerService -import java.io.IOException import java.time.Instant /** * This class publishes the data consumption information events to the Google Cloud Pub/Sub. */ -object DataConsumptionInfoPublisher : Managed { +object DataConsumptionInfoPublisher : + PubSubPublisher by DelegatePubSubPublisher(topicId = ConfigRegistry.config.dataTrafficTopicId) { private val logger by logger() private val pseudonymizerService by lazy { getResource() } - private lateinit var publisher: Publisher - - @Throws(IOException::class) - override fun start() { - - val topicName = ProjectTopicName.of(config.projectId, config.topicId) - - // Create a publisher instance with default settings bound to the topic - publisher = Publisher.newBuilder(topicName).build() - } - - @Throws(Exception::class) - override fun stop() { - // When finished with the publisher, shutdown to free up resources. - publisher.shutdown() - } - fun publish(msisdn: String, usedBucketBytes: Long, bundleBytes: Long) { if (usedBucketBytes == 0L) { return } - + val now = Instant.now().toEpochMilli() - val pseudonym = pseudonymizerService.getPseudonymEntityFor(msisdn, now).pseudonym + val pseudonym = pseudonymizerService.getMsisdnPseudonym(msisdn, now).pseudonym val data = DataTrafficInfo.newBuilder() .setMsisdn(pseudonym) @@ -64,7 +44,7 @@ object DataConsumptionInfoPublisher : Managed { .build() //schedule a message to be published, messages are automatically batched - val future = publisher.publish(pubsubMessage) + val future = publishPubSubMessage(pubsubMessage) // add an asynchronous callback to handle success / failure ApiFutures.addCallback(future, object : ApiFutureCallback { @@ -80,8 +60,8 @@ object DataConsumptionInfoPublisher : Managed { override fun onSuccess(messageId: String) { // Once published, returns server-assigned message ids (unique within the topic) - logger.debug(messageId) + logger.debug("Published message $messageId") } - }) + }, singleThreadScheduledExecutor) } } diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DelegatePubSubPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DelegatePubSubPublisher.kt new file mode 100644 index 000000000..7db538c90 --- /dev/null +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DelegatePubSubPublisher.kt @@ -0,0 +1,50 @@ +package org.ostelco.prime.analytics.publishers + +import com.google.api.core.ApiFuture +import com.google.api.gax.core.NoCredentialsProvider +import com.google.api.gax.grpc.GrpcTransportChannel +import com.google.api.gax.rpc.FixedTransportChannelProvider +import com.google.cloud.pubsub.v1.Publisher +import com.google.pubsub.v1.ProjectTopicName +import com.google.pubsub.v1.PubsubMessage +import io.grpc.ManagedChannelBuilder +import org.ostelco.prime.analytics.ConfigRegistry +import java.util.concurrent.Executors +import java.util.concurrent.ScheduledExecutorService + +class DelegatePubSubPublisher( + private val topicId: String, + private val projectId: String = ConfigRegistry.config.projectId) : PubSubPublisher { + + private lateinit var publisher: Publisher + + override lateinit var singleThreadScheduledExecutor: ScheduledExecutorService + + override fun start() { + + singleThreadScheduledExecutor = Executors.newSingleThreadScheduledExecutor() + + val topicName = ProjectTopicName.of(projectId, topicId) + val strSocketAddress = System.getenv("PUBSUB_EMULATOR_HOST") + publisher = if (!strSocketAddress.isNullOrEmpty()) { + val channel = ManagedChannelBuilder.forTarget(strSocketAddress).usePlaintext().build() + // Create a publisher instance with default settings bound to the topic + val channelProvider = FixedTransportChannelProvider.create(GrpcTransportChannel.create(channel)) + val credentialsProvider = NoCredentialsProvider() + Publisher.newBuilder(topicName) + .setChannelProvider(channelProvider) + .setCredentialsProvider(credentialsProvider) + .build(); + } else { + Publisher.newBuilder(topicName).build() + } + } + + override fun stop() { + // When finished with the publisher, shutdown to free up resources. + publisher.shutdown() + singleThreadScheduledExecutor.shutdown() + } + + override fun publishPubSubMessage(pubsubMessage: PubsubMessage): ApiFuture = publisher.publish(pubsubMessage) +} \ No newline at end of file diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PubSubPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PubSubPublisher.kt new file mode 100644 index 000000000..16c7d648d --- /dev/null +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PubSubPublisher.kt @@ -0,0 +1,11 @@ +package org.ostelco.prime.analytics.publishers + +import com.google.api.core.ApiFuture +import com.google.pubsub.v1.PubsubMessage +import io.dropwizard.lifecycle.Managed +import java.util.concurrent.ScheduledExecutorService + +interface PubSubPublisher : Managed { + var singleThreadScheduledExecutor: ScheduledExecutorService + fun publishPubSubMessage(pubsubMessage: PubsubMessage): ApiFuture +} \ No newline at end of file diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PurchaseInfoPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PurchaseInfoPublisher.kt new file mode 100644 index 000000000..ac8abe10a --- /dev/null +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PurchaseInfoPublisher.kt @@ -0,0 +1,88 @@ +package org.ostelco.prime.analytics.publishers + +import com.google.api.core.ApiFutureCallback +import com.google.api.core.ApiFutures +import com.google.api.gax.rpc.ApiException +import com.google.gson.Gson +import com.google.gson.GsonBuilder +import com.google.gson.JsonArray +import com.google.gson.JsonObject +import com.google.gson.JsonSerializer +import com.google.gson.reflect.TypeToken +import com.google.protobuf.ByteString +import com.google.pubsub.v1.PubsubMessage +import org.ostelco.prime.analytics.ConfigRegistry +import org.ostelco.prime.logger +import org.ostelco.prime.model.PurchaseRecord +import org.ostelco.prime.model.PurchaseRecordInfo +import org.ostelco.prime.module.getResource +import org.ostelco.prime.pseudonymizer.PseudonymizerService +import java.net.URLEncoder + + +/** + * This class publishes the purchase information events to the Google Cloud Pub/Sub. + */ +object PurchaseInfoPublisher : + PubSubPublisher by DelegatePubSubPublisher(topicId = ConfigRegistry.config.purchaseInfoTopicId) { + + private val logger by logger() + + private val pseudonymizerService by lazy { getResource() } + + private var gson: Gson = createGson() + + private fun createGson(): Gson { + val builder = GsonBuilder() + // Type for this conversion is explicitly set to java.util.Map + // This is needed because of kotlin's own Map interface + val mapType = object : TypeToken>() {}.type + val serializer = JsonSerializer> { src, _, _ -> + val array = JsonArray() + src.forEach { k, v -> + val property = JsonObject() + property.addProperty("key", k) + property.addProperty("value", v) + array.add(property) + } + array + } + builder.registerTypeAdapter(mapType, serializer) + return builder.create() + } + + private fun convertToJson(purchaseRecordInfo: PurchaseRecordInfo): ByteString = + ByteString.copyFromUtf8(gson.toJson(purchaseRecordInfo)) + + + fun publish(purchaseRecord: PurchaseRecord, subscriberId: String, status: String) { + + val encodedSubscriberId = URLEncoder.encode(subscriberId, "UTF-8") + val pseudonym = pseudonymizerService.getSubscriberIdPseudonym(encodedSubscriberId, purchaseRecord.timestamp).pseudonym + + val pubsubMessage = PubsubMessage.newBuilder() + .setData(convertToJson(PurchaseRecordInfo(purchaseRecord, pseudonym, status))) + .build() + + //schedule a message to be published, messages are automatically batched + val future = publishPubSubMessage(pubsubMessage) + + // add an asynchronous callback to handle success / failure + ApiFutures.addCallback(future, object : ApiFutureCallback { + + override fun onFailure(throwable: Throwable) { + if (throwable is ApiException) { + // details on the API exception + logger.warn("Status code: {}", throwable.statusCode.code) + logger.warn("Retrying: {}", throwable.isRetryable) + } + logger.warn("Error publishing purchase record for msisdn: {}", purchaseRecord.msisdn) + } + + override fun onSuccess(messageId: String) { + // Once published, returns server-assigned message ids (unique within the topic) + logger.debug(messageId) + } + }, singleThreadScheduledExecutor) + } +} diff --git a/app-notifier/src/main/kotlin/org/ostelco/prime/appnotifier/FirebaseModule.kt b/app-notifier/src/main/kotlin/org/ostelco/prime/appnotifier/FirebaseModule.kt index da4a2d12c..4ddb234c5 100644 --- a/app-notifier/src/main/kotlin/org/ostelco/prime/appnotifier/FirebaseModule.kt +++ b/app-notifier/src/main/kotlin/org/ostelco/prime/appnotifier/FirebaseModule.kt @@ -17,7 +17,6 @@ class FirebaseModule : PrimeModule { @JsonProperty("config") fun setConfig(config: FirebaseConfig) { - println("Config set for AppNotifier") setupFirebaseApp(config.databaseName, config.configFile) } @@ -26,7 +25,6 @@ class FirebaseModule : PrimeModule { configFile: String) { try { - println("Setting up Firebase for FirebaseAppNotifier. databaseName : $databaseName , configFile : $configFile ") val credentials: GoogleCredentials = if (Files.exists(Paths.get(configFile))) { FileInputStream(configFile).use { serviceAccount -> GoogleCredentials.fromStream(serviceAccount) } } else { diff --git a/bq-metrics-extractor/.gitignore b/bq-metrics-extractor/.gitignore new file mode 100644 index 000000000..57be5a48e --- /dev/null +++ b/bq-metrics-extractor/.gitignore @@ -0,0 +1,35 @@ +### Eclipse ### +.checkstyle +.classpath +.metadata +.loadpath +.project +.settings/ + +### Gradle ### +/.gradle/ +/build/ + +### Intellij ### +.idea/ +*.iml +*.ipr +*.iws +out/ + +### Mac OSX + Windows ### +.DS_Store +Thumbs.db + +### Node ### +/node_modules/ +npm-debug.log + +### SublimeText + TextMate ### +*.sublime-workspace +*.sublime-project +*.tmproj +*.tmproject + +### Vim ### +*.sw[op] diff --git a/bq-metrics-extractor/Dockerfile b/bq-metrics-extractor/Dockerfile new file mode 100644 index 000000000..3eefa70c0 --- /dev/null +++ b/bq-metrics-extractor/Dockerfile @@ -0,0 +1,28 @@ +FROM alpine:3.7 + +MAINTAINER CSI "csi@telenordigital.com" + +# +# Copy the files we need +# + +COPY script/start.sh /start.sh +COPY config/config.yaml /config/config.yaml +COPY build/libs/bq-metrics-extractor-uber.jar /bq-metrics-extractor.jar + +# +# Load, then dump the standard java classes into the +# image being built, to speed up java load time +# using Class Data Sharing. The "quit" command will +# simply quit the program after it's dumped the list of +# classes that should be cached. +# + +CMD ["java", "-Dfile.encoding=UTF-8", "-Xshare:on", "-Xshare:dump", "-jar", "/bq-metrics-extractor.jar", "quit", "config/config.yaml"] + + +# +# Finally the actual entry point +# + +ENTRYPOINT ["/start.sh"] diff --git a/bq-metrics-extractor/Dockerfile.test b/bq-metrics-extractor/Dockerfile.test new file mode 100644 index 000000000..312aae869 --- /dev/null +++ b/bq-metrics-extractor/Dockerfile.test @@ -0,0 +1,29 @@ +FROM alpine:3.7 + +MAINTAINER CSI "csi@telenordigital.com" + +# +# Copy the files we need +# + +COPY script/start.sh /start.sh +COPY config/pantel-prod.json /secret/pantel-prod.json +COPY config/config.yaml /config/config.yaml +COPY build/libs/bq-metrics-extractor-uber.jar /bq-metrics-extractor.jar + +# +# Load, then dump the standard java classes into the +# image being built, to speed up java load time +# using Class Data Sharing. The "quit" command will +# simply quit the program after it's dumped the list of +# classes that should be cached. +# + +CMD ["java", "-Dfile.encoding=UTF-8", "-Xshare:on", "-Xshare:dump", "-jar", "/bq-metrics-extractor.jar", "quit", "config/config.yaml"] + + +# +# Finally the actual entry point +# + +ENTRYPOINT ["/start.sh"] diff --git a/bq-metrics-extractor/README.md b/bq-metrics-extractor/README.md new file mode 100644 index 000000000..ef069b2b1 --- /dev/null +++ b/bq-metrics-extractor/README.md @@ -0,0 +1,107 @@ +BigQuery metrics extractor +======= + + +This module is a standalone, command-line launched dropwizard application +that will: + +* Talk to google [BigQuery](https://cloud.google.com/bigquery/) and + extract metrics using [Googles BigQuery java library](https://cloud.google.com/bigquery/docs/reference/libraries) +* Talk to [Prometheus](https://prometheus.io) + [Pushgateway](https://github.com/prometheus/pushgateway) and push + those metrics there. Prometheus servers can then scrape those + metrics at their leisure. We will use the + [Prometheus java client](https://github.com/prometheus/client_java) + to talk to the pushgateway. + +The component will be built as a docker component, and will then be periodically +run as a command line application, as a +[Kubernetes cron job](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/). + + +The component is packaged as an individual docker artefact (details below), +and deployed as a cronjob (also described below). + +To run the program from the command line, which is useful when debugging and +necessary to know when constructing a Docker file, do this: + + java -jar /bq-metrics-extractor.jar query --pushgateway pushgateway:8080 config/config.yaml + +the pushgateway:8080 is the hostname (dns resolvable) and portnumber of the Prometheus Push Gateway. + +The config.yaml file contains specifications of queries and how they map to metrics: + + bqmetrics: + - type: summary + name: active_users + help: Number of active users + resultColumn: count + sql: > + SELECT count(distinct user_pseudo_id) AS count FROM `pantel-2decb.analytics_160712959.events_*` + WHERE event_name = "first_open" + LIMIT 1000 + +The idea being that to add queries of a type that is already know by the extractor program, +only an addition to the bqmetrics list. +Use [standardSQL syntax (not legacy)](https://cloud.google.com/bigquery/sql-reference/) for queries. + +If not running in a google kubernetes cluster (e.g. in docker compose, or from the command line), +it's necessary to set the environment variable GOOGLE_APPLICATION_CREDENTIALS to point to +a credentials file that will provide access for the BigQuery library. + + + +How to build and deploy the cronjob manually +=== + +##First get credentials (upgrade gcloud for good measure): + + gcloud components update + gcloud container clusters get-credentials dev-cluster --zone europe-west1-b --project pantel-2decb + +##Build the artefact: + + gradle build + docker build . + +##Authorize tag and push to docker registry in google cloud: + + gcloud auth configure-docker + docker tag foobarbaz eu.gcr.io/pantel-2decb/bq-metrics-extractor + docker push eu.gcr.io/pantel-2decb/bq-metrics-extractor + +... where foobarbaz is the id of the container built by docker build. + +## Then start the cronjob in kubernetes + kubectl apply -f cronjob/extractor.yaml + kubectl describe cronjob bq-metrics-extractor + +## To talk to the prometheus in the monitoring namespace & watch the users metrics evolve + kubectl port-forward --namespace=monitoring $(kubectl get pods --namespace=monitoring | grep prometheus-core | awk '{print $1}') 9090 + watch 'curl -s localhost:9090/metrics | grep users' + + +TODO +=== + +* Rewrite the SQL so that it will pick up only today/yesterday's data, + use a template language, either premade or ad-hoc. + As of now, the sql in config is static. + We need to make it as a template. + Table name to be changed from events_* to events_${yyyyMMdd} + + Here, the suffix is yesterday’s date in yyyyMMdd format. (edited) + There are some libraries which we can use, which enable f + reemarker expressions in dropwizard config file + (this comment is @vihangpatil 's I just nicked it from + slack where he made the comment) + + +* Add more metrics. +* Make an acceptance tests that runs a roundtrip test ub + in docker compose, based on something like this: curl http://localhost:9091/metrics | grep -i active +* Push the first metric to production, use Kubernetes crontab + to ensure periodic execution. +* Make it testable to send send metrics to pushgateway. +* Extend to more metrics. +* remove the TODO list and declare victory :-) \ No newline at end of file diff --git a/bq-metrics-extractor/build.gradle b/bq-metrics-extractor/build.gradle new file mode 100644 index 000000000..e8256569e --- /dev/null +++ b/bq-metrics-extractor/build.gradle @@ -0,0 +1,42 @@ +plugins { + id "org.jetbrains.kotlin.jvm" version "1.2.61" + id "application" + id "com.github.johnrengelman.shadow" version "2.0.4" + id "idea" +} + + +dependencies { + + implementation "io.dropwizard:dropwizard-core:$dropwizardVersion" + implementation "io.dropwizard:dropwizard-client:$dropwizardVersion" + + testImplementation "io.dropwizard:dropwizard-client:$dropwizardVersion" + testImplementation "io.dropwizard:dropwizard-testing:$dropwizardVersion" + testImplementation "com.fasterxml.jackson.module:jackson-module-kotlin:$jacksonVersion" + testImplementation "org.mockito:mockito-core:2.18.3" + testImplementation 'org.assertj:assertj-core:3.10.0' + + implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion" + // Bigquery dependency + compile 'com.google.cloud:google-cloud-bigquery:1.40.0' + +runtimeOnly "io.dropwizard:dropwizard-json-logging:$dropwizardVersion" + + // Prometheus pushgateway dependencies (we might not need all of these) + // compile 'io.prometheus:simpleclient:0.5.0' + // compile 'io.prometheus:simpleclient_hotspot:0.5.0' + // compile 'io.prometheus:simpleclient_httpserver:0.5.0' + compile 'io.prometheus:simpleclient_pushgateway:0.5.0' + compile 'com.google.apis:google-api-services-pubsub:v1-rev399-1.25.0' + +} + +shadowJar { + mainClassName = 'org.ostelco.bqmetrics.BqMetricsExtractorApplicationKt' + mergeServiceFiles() + classifier = "uber" + version = null +} + +apply from: '../jacoco.gradle' \ No newline at end of file diff --git a/bq-metrics-extractor/config/.gitignore b/bq-metrics-extractor/config/.gitignore new file mode 100644 index 000000000..bf045303f --- /dev/null +++ b/bq-metrics-extractor/config/.gitignore @@ -0,0 +1 @@ +pantel-prod.json \ No newline at end of file diff --git a/bq-metrics-extractor/config/config.yaml b/bq-metrics-extractor/config/config.yaml new file mode 100644 index 000000000..69a3b60d1 --- /dev/null +++ b/bq-metrics-extractor/config/config.yaml @@ -0,0 +1,47 @@ +logging: + level: INFO + loggers: + org.ostelco: DEBUG + appenders: + - type: console + layout: + type: json + customFieldNames: + level: severity + +bqmetrics: + - type: summary + name: active_users + help: Number of active users + resultColumn: count + sql: > + SELECT count(distinct user_pseudo_id) AS count FROM `pantel-2decb.analytics_160712959.events_*` + WHERE event_name = "first_open" + - type: gauge + name: sims_who_have_used_data + help: Number of SIMs that has used data last 24 hours + resultColumn: count + sql: > + SELECT count(DISTINCT msisdn) AS count FROM `pantel-2decb.data_consumption.raw_consumption` + WHERE timestamp >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 DAY) + - type: gauge + name: total_data_used + help: Total data used last 24 hours + resultColumn: count + sql: > + SELECT sum(bucketBytes) AS count FROM `pantel-2decb.data_consumption.raw_consumption` + WHERE timestamp >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 DAY) + - type: gauge + name: revenue_last24hours + help: Revenue for last 24 hours + resultColumn: revenue + sql: > + SELECT SUM(product.price.amount) as revenue FROM `pantel-2decb.purchases.raw_purchases` + WHERE TIMESTAMP_MILLIS(timestamp) > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 24 HOUR) + - type: gauge + name: total_paid_users + help: Number of users who have purchased in last 24 hours + resultColumn: count + sql: > + SELECT COUNT(DISTINCT subscriberId) as count FROM `pantel-2decb.purchases.raw_purchases` + WHERE TIMESTAMP_MILLIS(timestamp) > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 24 HOUR) diff --git a/bq-metrics-extractor/cronjob/extractor.yaml b/bq-metrics-extractor/cronjob/extractor.yaml new file mode 100644 index 000000000..65bc91486 --- /dev/null +++ b/bq-metrics-extractor/cronjob/extractor.yaml @@ -0,0 +1,15 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: bq-metrics-extractor +spec: + schedule: "*/30 * * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: bq-metrics-extractor + image: eu.gcr.io/pantel-2decb/bq-metrics-extractor:latest + imagePullPolicy: Always + restartPolicy: Never diff --git a/bq-metrics-extractor/docker-compose.yml b/bq-metrics-extractor/docker-compose.yml new file mode 100644 index 000000000..33c79698a --- /dev/null +++ b/bq-metrics-extractor/docker-compose.yml @@ -0,0 +1,64 @@ +version: '3.3' +services: + # application: + # image: ... + # environment: + # - PUBSUB_EMULATOR_HOST="emulator:8085" + # # ...other configurations... + # depends_on: + # - emulator + # - push-gateway + + + metrics-extrator: + container_name: metrics-extrator + build: + context: . + dockerfile: Dockerfile.test + depends_on: + - pushgateway + environment: + - GOOGLE_APPLICATION_CREDENTIALS=/secret/pantel-prod.json + + emulator: + container_name: emulator + image: adilsoncarvalho/gcloud-pubsub-emulator + ports: + - "8085:8085" + + prometheus: + container_name: prometheus + image: prom/prometheus + volumes: + - './prometheus.yml:/etc/prometheus/prometheus.yml' + - 'prometheus_data:/prometheus' + ports: + - '9090:9090' + + # Pushgateway exposes external port 8080, since that is the port + # that is exposed by the pushgateway in the kubernetes clusters + pushgateway: + container_name: pushgateway + image: prom/pushgateway + ports: + - '8080:9091' + + grafana: + container_name: grafana + image: grafana/grafana + environment: + # Please note that setting the password only works the _FIRST_TIME_ + # the image is built. After that, it's cached and won't change + # if you change it in this docker-compose.yml file. You have + # been warned! + - GF_SECURITY_ADMIN_PASSWORD=pass + depends_on: + - prometheus + ports: + - '3000:3000' + volumes: + - 'grafana_data:/var/lib/grafana' + +volumes: + prometheus_data: {} + grafana_data: {} diff --git a/bq-metrics-extractor/prometheus.yml b/bq-metrics-extractor/prometheus.yml new file mode 100644 index 000000000..836bbfba3 --- /dev/null +++ b/bq-metrics-extractor/prometheus.yml @@ -0,0 +1,27 @@ +global: + scrape_interval: 15s # By default, scrape targets every 15 seconds. + + # Attach these labels to any time series or alerts when communicating with + # external systems (federation, remote storage, Alertmanager). + external_labels: + monitor: 'codelab-monitor' + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped from this config. + - job_name: 'prometheus' + + # Override the global default and scrape targets from this job every 5 seconds. + scrape_interval: 5s + + static_configs: + - targets: ['prometheus:9090'] + + - job_name: 'push-gateway' + + scrape_interval: 5s + honor_labels: true + + static_configs: + - targets: ['push-gateway:9091'] \ No newline at end of file diff --git a/bq-metrics-extractor/script/build-and-upload-docker-image.sh b/bq-metrics-extractor/script/build-and-upload-docker-image.sh new file mode 100755 index 000000000..f7844af69 --- /dev/null +++ b/bq-metrics-extractor/script/build-and-upload-docker-image.sh @@ -0,0 +1,49 @@ +#!/bin/sh + +## +## Build a new jar file, then a new docker image, then +## upload the docker image to a google docker +## repository. +## + + +# Exit on failure +set -e + +# Check for dependencies +DEPENDENCIES="gradle docker gcloud" +for dep in $DEPENDENCIES ; do + if [[ -z "$(type $dep)" ]] ; then + echo "Could not find dependency $dep, bailing out" + exit 1 + fi +done + +# Set destination + +GCLOUD_PROJECT_NAME="pantel-2decb" +CONTAINER_NAME="bq-metrics-extractor" +GCLOUD_REPO_NAME="eu.gcr.io" + + + +# Log into the appropriate google account and prepare to build&upload +# XXX Couldn't figure out how to make this work well in a script, but +# that should be solved, therefore I'm keeping the dead code instead +# of doing the right thing according to the project coding standard +# and killing it off. +# gcloud auth login +# gcloud auth configure-docker + +# Build the java .jar application from sources +gradle build + +# Build the docker container +CONTAINER_ID=$(docker build . | grep "Successfully built" | awk '{print $3}') +echo "Built container $CONTAINER_ID" + +# Tag and push the docker container to the google repo +echo "Tagging and pushing container" +THE_TAG="${GCLOUD_REPO_NAME}/${GCLOUD_PROJECT_NAME}/${CONTAINER_NAME}" +docker tag ${CONTAINER_ID} ${THE_TAG} +docker push ${THE_TAG} diff --git a/bq-metrics-extractor/script/start.sh b/bq-metrics-extractor/script/start.sh new file mode 100755 index 000000000..6081dbe61 --- /dev/null +++ b/bq-metrics-extractor/script/start.sh @@ -0,0 +1,7 @@ +#!/bin/bash -x + +# Start app +exec java \ + -Dfile.encoding=UTF-8 \ + -Xshare:on \ + -jar /bq-metrics-extractor.jar query --pushgateway pushgateway:8080 config/config.yaml diff --git a/bq-metrics-extractor/src/main/java/org/ostelco/bqmetrics/BqMetricsExtractorApplication.kt b/bq-metrics-extractor/src/main/java/org/ostelco/bqmetrics/BqMetricsExtractorApplication.kt new file mode 100644 index 000000000..2f821ac3e --- /dev/null +++ b/bq-metrics-extractor/src/main/java/org/ostelco/bqmetrics/BqMetricsExtractorApplication.kt @@ -0,0 +1,323 @@ +package org.ostelco.bqmetrics + + +import com.fasterxml.jackson.annotation.JsonProperty +import com.google.cloud.bigquery.* +import io.dropwizard.Application +import io.dropwizard.setup.Bootstrap +import io.dropwizard.setup.Environment +import io.prometheus.client.exporter.PushGateway +import io.prometheus.client.CollectorRegistry +import io.dropwizard.Configuration +import io.dropwizard.cli.ConfiguredCommand +import io.prometheus.client.Gauge +import io.prometheus.client.Summary +import net.sourceforge.argparse4j.inf.Namespace +import net.sourceforge.argparse4j.inf.Subparser +import org.slf4j.LoggerFactory +import java.util.* +import org.slf4j.Logger +import javax.validation.Valid +import javax.validation.constraints.NotNull + +/** + * Bridge between "latent metrics" stored in BigQuery and Prometheus + * metrics available for instrumentation ana alerting services. + * + * Common usecase: + * + * java -jar /bq-metrics-extractor.jar query --pushgateway pushgateway:8080 config/config.yaml + * + * the pushgateway:8080 is the hostname (dns resolvable) and portnumber of the + * Prometheus Push Gateway. + * + * The config.yaml file contains specifications of queries and how they map + * to metrics: + * + * bqmetrics: + * - type: summary + * name: active_users + * help: Number of active users + * resultColumn: count + * sql: > + * SELECT count(distinct user_pseudo_id) AS count FROM `pantel-2decb.analytics_160712959.events_*` + * WHERE event_name = "first_open" + * LIMIT 1000 + * + * Use standard SQL syntax (not legacy) for queries. + * See: https://cloud.google.com/bigquery/sql-reference/ + * + * If not running in a google kubernetes cluster (e.g. in docker compose, or from the command line), + * it's necessary to set the environment variable GOOGLE_APPLICATION_CREDENTIALS to point to + * a credentials file that will provide access for the BigQuery library. + * + */ + + +/** + * Main entry point, invoke dropwizard application. + */ +fun main(args: Array) { + BqMetricsExtractorApplication().run(*args) +} + +/** + * Config of a single metric that will be extracted using a BigQuery + * query. + */ +private class MetricConfig { + + /** + * Type of the metric. Currently the only permitted type is + * "summary", the intent is to extend this as more types + * of metrics (counters, gauges, ...) are added. + */ + @Valid + @NotNull + @JsonProperty + lateinit var type: String + + /** + * The name of the metric, as it will be seen by Prometheus. + */ + @Valid + @NotNull + @JsonProperty + lateinit var name: String + + /** + * A help string, used to describe the metric. + */ + @Valid + @NotNull + @JsonProperty + lateinit var help: String + + /** + * When running the query, the result should be placed in a named + * column, and this field contains the name of that column. + */ + @Valid + @NotNull + @JsonProperty + lateinit var resultColumn: String + + /** + * The SQL used to extract the value of the metric from BigQuery. + */ + @Valid + @NotNull + @JsonProperty + lateinit var sql: String +} + + +/** + * Configuration for the extractor, default config + * plus a list of metrics descriptions. + */ +private class BqMetricsExtractorConfig: Configuration() { + @Valid + @NotNull + @JsonProperty("bqmetrics") + lateinit var metrics: List +} + + +/** + * Main entry point to the bq-metrics-extractor API server. + */ +private class BqMetricsExtractorApplication : Application() { + + override fun initialize(bootstrap: Bootstrap) { + bootstrap.addCommand(CollectAndPushMetrics()) + } + + override fun run( + configuration: BqMetricsExtractorConfig, + environment: Environment) { + } +} + + +private interface MetricBuilder { + fun buildMetric(registry: CollectorRegistry) + + fun getNumberValueViaSql(sql: String, resultColumn: String): Long { + // Instantiate a client. If you don't specify credentials when constructing a client, the + // client library will look for credentials in the environment, such as the + // GOOGLE_APPLICATION_CREDENTIALS environment variable. + val bigquery = BigQueryOptions.getDefaultInstance().service + val queryConfig: QueryJobConfiguration = + QueryJobConfiguration.newBuilder( + sql.trimIndent()) + .setUseLegacySql(false) + .build(); + + // Create a job ID so that we can safely retry. + val jobId: JobId = JobId.of(UUID.randomUUID().toString()); + var queryJob: Job = bigquery.create(JobInfo.newBuilder(queryConfig).setJobId(jobId).build()); + + // Wait for the query to complete. + queryJob = queryJob.waitFor(); + + // Check for errors + if (queryJob == null) { + throw BqMetricsExtractionException("Job no longer exists"); + } else if (queryJob.getStatus().getError() != null) { + // You can also look at queryJob.getStatus().getExecutionErrors() for all + // errors, not just the latest one. + throw BqMetricsExtractionException(queryJob.getStatus().getError().toString()); + } + val result = queryJob.getQueryResults() + if (result.totalRows != 1L) { + throw BqMetricsExtractionException("Number of results was ${result.totalRows} which is different from the expected single row") + } + + val count = result.iterateAll().iterator().next().get(resultColumn).longValue + + return count + } +} + +private class SummaryMetricBuilder( + val metricName: String, + val help: String, + val sql: String, + val resultColumn: String) : MetricBuilder { + + private val log: Logger = LoggerFactory.getLogger(SummaryMetricBuilder::class.java) + + + override fun buildMetric(registry: CollectorRegistry) { + val summary: Summary = Summary.build() + .name(metricName) + .help(help).register(registry) + val value: Long = getNumberValueViaSql(sql, resultColumn) + + log.info("Summarizing metric $metricName to be $value") + + summary.observe(value * 1.0) + } +} + +private class GaugeMetricBuilder( + val metricName: String, + val help: String, + val sql: String, + val resultColumn: String) : MetricBuilder { + + private val log: Logger = LoggerFactory.getLogger(SummaryMetricBuilder::class.java) + + override fun buildMetric(registry: CollectorRegistry) { + val gauge: Gauge = Gauge.build() + .name(metricName) + .help(help).register(registry) + val value: Long = getNumberValueViaSql(sql, resultColumn) + + log.info("Gauge metric $metricName = $value") + + gauge.set(value * 1.0) + } +} + +/** + * Thrown when something really bad is detected and it's necessary to terminate + * execution immediately. No cleanup of anything will be done. + */ +private class BqMetricsExtractionException: RuntimeException { + constructor(message: String, ex: Exception?): super(message, ex) + constructor(message: String): super(message) + constructor(ex: Exception): super(ex) +} + + +/** + * Adapter class that will push metrics to the Prometheus push gateway. + */ +private class PrometheusPusher(val pushGateway: String, val job: String) { + + private val log: Logger = LoggerFactory.getLogger(PrometheusPusher::class.java) + + val registry = CollectorRegistry() + + @Throws(Exception::class) + fun publishMetrics(metrics: List) { + + val metricSources: MutableList = mutableListOf() + metrics.forEach { + val typeString: String = it.type.trim().toUpperCase() + when (typeString) { + "SUMMARY" -> { + metricSources.add(SummaryMetricBuilder( + it.name, + it.help, + it.sql, + it.resultColumn)) + } + "GAUGE" -> { + metricSources.add(GaugeMetricBuilder( + it.name, + it.help, + it.sql, + it.resultColumn)) + } + else -> { + log.error("Unknown metrics type '${it.type}'") + } + } + } + + log.info("Querying bigquery for metric values") + val pg = PushGateway(pushGateway) + metricSources.forEach({ it.buildMetric(registry) }) + + log.info("Pushing metrics to pushgateway") + pg.pushAdd(registry, job) + log.info("Done transmitting metrics to pushgateway") + } +} + +private class CollectAndPushMetrics : ConfiguredCommand( + "query", + "query BigQuery for a metric") { + override fun run(bootstrap: Bootstrap?, namespace: Namespace?, configuration: BqMetricsExtractorConfig?) { + + if (configuration == null) { + throw BqMetricsExtractionException("Configuration is null") + } + + + if (namespace == null) { + throw BqMetricsExtractionException("Namespace from config is null") + } + + val pgw = namespace.get(pushgatewayKey) + PrometheusPusher(pgw, + "bq_metrics_extractor").publishMetrics(configuration.metrics) + } + + val pushgatewayKey = "pushgateway" + + override fun configure(subparser: Subparser?) { + super.configure(subparser) + if (subparser == null) { + throw BqMetricsExtractionException("subparser is null") + } + subparser.addArgument("-p", "--pushgateway") + .dest(pushgatewayKey) + .type(String::class.java) + .required(true) + .help("The pushgateway to report metrics to, format is hostname:portnumber") + } + + private class CollectAndPushMetrics : ConfiguredCommand( + "quit", + "Do nothing, only used to prime caches") { + override fun run(bootstrap: Bootstrap?, + namespace: Namespace?, + configuration: BqMetricsExtractorConfig?) { + // Doing nothing, as advertised. + } + } +} diff --git a/build.gradle b/build.gradle index e890c8522..9e0df77e7 100644 --- a/build.gradle +++ b/build.gradle @@ -30,8 +30,12 @@ subprojects { ext { kotlinVersion = "1.2.61" dropwizardVersion = "1.3.5" - googleCloudVersion = "1.41.0" + googleCloudVersion = "1.43.0" jacksonVersion = "2.9.6" + stripeVersion = "6.12.0" + guavaVersion = "26.0-jre" + assertJVersion = "3.11.1" + mockitoVersion = "2.21.0" } } diff --git a/client-api/build.gradle b/client-api/build.gradle index c41c249d3..1f355da83 100644 --- a/client-api/build.gradle +++ b/client-api/build.gradle @@ -11,25 +11,22 @@ dependencies { implementation "io.dropwizard:dropwizard-auth:$dropwizardVersion" implementation "io.dropwizard:dropwizard-client:$dropwizardVersion" - implementation 'com.google.guava:guava:25.1-jre' + implementation "com.google.guava:guava:$guavaVersion" implementation 'io.jsonwebtoken:jjwt:0.9.1' testImplementation "io.dropwizard:dropwizard-client:$dropwizardVersion" testImplementation "io.dropwizard:dropwizard-testing:$dropwizardVersion" testImplementation "com.fasterxml.jackson.module:jackson-module-kotlin:$jacksonVersion" - testImplementation "org.mockito:mockito-core:2.18.3" - testImplementation 'org.assertj:assertj-core:3.10.0' - - // from filter - // https://mvnrepository.com/artifact/org.glassfish.jersey.test-framework.providers/jersey-test-framework-provider-grizzly2 - // Updating from 2.25.1 to 2.27 causes error - testCompile (group: 'org.glassfish.jersey.test-framework.providers', name: 'jersey-test-framework-provider-grizzly2', version: '2.25.1') { - // 2.26 (latest) + testImplementation "org.mockito:mockito-core:$mockitoVersion" + testImplementation "org.assertj:assertj-core:$assertJVersion" + + testImplementation (group: 'org.glassfish.jersey.test-framework.providers', name: 'jersey-test-framework-provider-grizzly2', version: '2.25.1') { + because "Updating from 2.25.1 to 2.27.1 causes error. Keep the version matched with 'jersey-server' version from dropwizard." exclude group: 'javax.servlet', module: 'javax.servlet-api' exclude group: 'junit', module: 'junit' } - testCompile "com.nhaarman:mockito-kotlin:1.6.0" + testImplementation "com.nhaarman:mockito-kotlin:1.6.0" } tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).all { diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/ClientApiModule.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/ClientApiModule.kt index 8795b4970..b1d3f7105 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/ClientApiModule.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/ClientApiModule.kt @@ -10,6 +10,7 @@ import io.dropwizard.auth.CachingAuthenticator import io.dropwizard.auth.oauth.OAuthCredentialAuthFilter.Builder import io.dropwizard.client.JerseyClientBuilder import io.dropwizard.setup.Environment +import org.eclipse.jetty.servlets.CrossOriginFilter import org.ostelco.prime.client.api.auth.AccessTokenPrincipal import org.ostelco.prime.client.api.auth.OAuthAuthenticator import org.ostelco.prime.client.api.metrics.reportMetricsAtStartUp @@ -28,8 +29,11 @@ import org.ostelco.prime.module.PrimeModule import org.ostelco.prime.module.getResource import org.ostelco.prime.ocs.OcsSubscriberService import org.ostelco.prime.storage.ClientDataSource +import java.util.* +import javax.servlet.DispatcherType import javax.ws.rs.client.Client + /** * Provides API for client. * @@ -45,6 +49,16 @@ class ClientApiModule : PrimeModule { override fun init(env: Environment) { + // Allow CORS + val corsFilterRegistration = env.servlets().addFilter("CORS", CrossOriginFilter::class.java) + // Configure CORS parameters + corsFilterRegistration.setInitParameter("allowedOrigins", "*") + corsFilterRegistration.setInitParameter("allowedHeaders", + "Cache-Control,If-Modified-Since,Pragma,Content-Type,Authorization,X-Requested-With,Content-Length,Accept,Origin") + corsFilterRegistration.setInitParameter("allowedMethods", "OPTIONS,GET,PUT,POST,DELETE,HEAD") + corsFilterRegistration.addMappingForUrlPatterns(EnumSet.allOf(DispatcherType::class.java), true, "/*") + + val dao = SubscriberDAOImpl(storage, ocsSubscriberService) val jerseyEnv = env.jersey() diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/metrics/Metrics.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/metrics/Metrics.kt index 51bd24737..9bd73a7c8 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/metrics/Metrics.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/metrics/Metrics.kt @@ -19,7 +19,4 @@ fun reportMetricsAtStartUp() { fun updateMetricsOnNewSubscriber() { analyticsService.reportMetric(TOTAL_USERS, adminStore.getSubscriberCount()) analyticsService.reportMetric(USERS_ACQUIRED_THROUGH_REFERRALS, adminStore.getReferredSubscriberCount()) -} -fun updateMetricsOnPurchase() { - analyticsService.reportMetric(USERS_PAID_AT_LEAST_ONCE, adminStore.getPaidSubscriberCount()) } \ No newline at end of file diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/resources/PaymentResource.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/resources/PaymentResource.kt index 657ed4710..52f1b0ac0 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/resources/PaymentResource.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/resources/PaymentResource.kt @@ -52,7 +52,7 @@ class PaymentResource(private val dao: SubscriberDAO) { return dao.listSources(token.name) .fold( { apiError -> Response.status(apiError.status).entity(asJson(apiError.description)) }, - { sourceList -> Response.status(Response.Status.CREATED).entity(sourceList)} + { sourceList -> Response.status(Response.Status.OK).entity(sourceList)} ).build() } @@ -69,7 +69,7 @@ class PaymentResource(private val dao: SubscriberDAO) { return dao.setDefaultSource(token.name, sourceId) .fold( { apiError -> Response.status(apiError.status).entity(asJson(apiError.description)) }, - { sourceInfo -> Response.status(Response.Status.CREATED).entity(sourceInfo)} + { sourceInfo -> Response.status(Response.Status.OK).entity(sourceInfo)} ).build() } } diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt index 1d9d2bca0..6e617bfa6 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt @@ -1,12 +1,10 @@ package org.ostelco.prime.client.api.store import arrow.core.Either -import arrow.core.Tuple4 import arrow.core.flatMap import org.ostelco.prime.analytics.AnalyticsService import org.ostelco.prime.analytics.PrimeMetric.REVENUE import org.ostelco.prime.client.api.metrics.updateMetricsOnNewSubscriber -import org.ostelco.prime.client.api.metrics.updateMetricsOnPurchase import org.ostelco.prime.client.api.model.Consent import org.ostelco.prime.client.api.model.Person import org.ostelco.prime.client.api.model.SubscriptionStatus @@ -55,7 +53,7 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu BadRequestError("Incomplete profile description. ${it.message}") } } catch (e: Exception) { - logger.error("Failed to fetch profile", e) + logger.error("Failed to fetch profile for subscriberId $subscriberId", e) Either.left(NotFoundError("Failed to fetch profile")) } } @@ -73,7 +71,7 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu getProfile(subscriberId) } } catch (e: Exception) { - logger.error("Failed to create profile", e) + logger.error("Failed to create profile for subscriberId $subscriberId", e) Either.left(ForbiddenError("Failed to create profile")) } } @@ -87,7 +85,7 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu try { storage.addNotificationToken(msisdn, applicationToken) } catch (e: Exception) { - logger.error("Failed to store ApplicationToken", e) + logger.error("Failed to store ApplicationToken for msisdn $msisdn", e) return Either.left(InsuffientStorageError("Failed to store ApplicationToken")) } return getNotificationToken(msisdn, applicationToken.applicationID) @@ -99,7 +97,7 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu ?.let { Either.right(it) } ?: return Either.left(NotFoundError("Failed to get ApplicationToken")) } catch (e: Exception) { - logger.error("Failed to get ApplicationToken", e) + logger.error("Failed to get ApplicationToken for msisdn $msisdn", e) return Either.left(NotFoundError("Failed to get ApplicationToken")) } } @@ -111,7 +109,7 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu try { storage.updateSubscriber(profile) } catch (e: Exception) { - logger.error("Failed to update profile", e) + logger.error("Failed to update profile for subscriberId $subscriberId", e) return Either.left(NotFoundError("Failed to update profile")) } @@ -128,7 +126,7 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu } .mapLeft { NotFoundError(it.message) } } catch (e: Exception) { - logger.error("Failed to get balance", e) + logger.error("Failed to get balance for subscriber $subscriberId", e) return Either.left(NotFoundError("Failed to get balance")) } } @@ -139,7 +137,7 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu NotFoundError("Failed to get subscriptions. ${it.message}") } } catch (e: Exception) { - logger.error("Failed to get subscriptions", e) + logger.error("Failed to get subscriptions for subscriberId $subscriberId", e) return Either.left(NotFoundError("Failed to get subscriptions")) } } @@ -156,7 +154,7 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu { NotFoundError("Failed to get purchase history. ${it.message}") }, { it.toList() }) } catch (e: Exception) { - logger.error("Failed to get purchase history", e) + logger.error("Failed to get purchase history for subscriberId $subscriberId", e) Either.left(NotFoundError("Failed to get purchase history")) } } @@ -167,7 +165,7 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu NotFoundError("Did not find msisdn for this subscription. ${it.message}") } } catch (e: Exception) { - logger.error("Did not find msisdn for this subscription", e) + logger.error("Did not find msisdn for subscriberId $subscriberId", e) Either.left(NotFoundError("Did not find subscription")) } } @@ -178,7 +176,7 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu { NotFoundError(it.message) }, { products -> products.values }) } catch (e: Exception) { - logger.error("Failed to get Products", e) + logger.error("Failed to get Products for subscriberId $subscriberId", e) Either.left(NotFoundError("Failed to get Products")) } @@ -192,13 +190,13 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu private fun createAndStorePaymentProfile(name: String): Either { return paymentProcessor.createPaymentProfile(name) + .mapLeft { ForbiddenError(it.description) } .flatMap { profileInfo -> setPaymentProfile(name, profileInfo) .map { profileInfo } } } - @Deprecated("use purchaseProduct", ReplaceWith("purchaseProduct")) override fun purchaseProductWithoutPayment(subscriberId: String, sku: String): Either { return getProduct(subscriberId, sku) @@ -218,6 +216,10 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu } // Notify OCS .flatMap { + analyticsReporter.reportPurchaseInfo( + purchaseRecord = purchaseRecord, + subscriberId = subscriberId, + status = "success") //TODO: Handle errors (when it becomes available) ocsSubscriberService.topup(subscriberId, sku) // TODO vihang: handle currency conversion @@ -227,85 +229,16 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu } } - override fun purchaseProduct(subscriberId: String, sku: String, sourceId: String?, saveCard: Boolean): Either { - return getProduct(subscriberId, sku) - // If we can't find the product, return not-found - .mapLeft { NotFoundError("Product unavailable") } - .flatMap { product: Product -> - // Fetch/Create stripe payment profile for the subscriber. - getPaymentProfile(subscriberId) - .fold( - { createAndStorePaymentProfile(subscriberId) }, - { profileInfo -> Either.right(profileInfo) } - ) - .map { profileInfo -> Pair(product, profileInfo) } - } - .flatMap { (product, profileInfo) -> - // Add payment source - if (sourceId != null) { - paymentProcessor.addSource(profileInfo.id, sourceId). - map {sourceInfo -> Triple(product, profileInfo, sourceInfo.id)} - } else { - Either.right(Triple(product, profileInfo, null)) - } - } - .flatMap { (product, profileInfo, savedSourceId) -> - // Authorize stripe charge for this purchase - val price = product.price - paymentProcessor.authorizeCharge(profileInfo.id, savedSourceId, price.amount, price.currency) - .mapLeft { apiError -> - logger.error("failed to authorize purchase for customerId ${profileInfo.id}, sourceId $savedSourceId, sku $sku") - apiError - } - .map { chargeId -> Tuple4(profileInfo, savedSourceId, chargeId, product) } - } - .flatMap { (profileInfo, savedSourceId, chargeId, product) -> - val purchaseRecord = PurchaseRecord( - id = chargeId, - product = product, - timestamp = Instant.now().toEpochMilli(), - msisdn = "") - // Create purchase record - storage.addPurchaseRecord(subscriberId, purchaseRecord) - .mapLeft { storeError -> - logger.error("failed to save purchase record, for customerId ${profileInfo.id}, chargeId $chargeId, payment will be unclaimed in Stripe") - BadGatewayError(storeError.message) - } - // Notify OCS - .flatMap { - //TODO: Handle errors (when it becomes available) - ocsSubscriberService.topup(subscriberId, sku) - Either.right(Tuple4(profileInfo, savedSourceId, chargeId, product)) - } - } - .flatMap { (profileInfo, savedSourceId, chargeId, product) -> - // Capture the charge, our database have been updated. - paymentProcessor.captureCharge(chargeId, profileInfo.id, sourceId) - .mapLeft { apiError -> - logger.error("Capture failed for customerId ${profileInfo.id}, chargeId $chargeId, Fix this in Stripe Dashborad") - apiError - } - .map { - // TODO vihang: handle currency conversion - analyticsReporter.reportMetric(REVENUE, product.price.amount.toLong()) - updateMetricsOnPurchase() - Triple(profileInfo, savedSourceId, ProductInfo(product.sku)) - } - } - .flatMap { (profileInfo, savedSourceId, productInfo) -> - // Remove the payment source - if (!saveCard && savedSourceId != null) { - paymentProcessor.removeSource(profileInfo.id, savedSourceId) - .mapLeft { apiError -> - logger.error("Failed to remove card, for customerId ${profileInfo.id}, sourceId $sourceId") - apiError - } - .map { productInfo } - } else { - Either.Right(productInfo) - } - } - } + override fun purchaseProduct( + subscriberId: String, + sku: String, + sourceId: String?, + saveCard: Boolean): Either = + storage.purchaseProduct( + subscriberId, + sku, + sourceId, + saveCard).mapLeft { NotFoundError(it.description) } override fun getReferrals(subscriberId: String): Either> { return try { @@ -313,7 +246,7 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu { NotFoundError("Failed to get referral list. ${it.message}") }, { list -> list.map { Person(it) } }) } catch (e: Exception) { - logger.error("Failed to get referral list", e) + logger.error("Failed to get referral list for subscriberId $subscriberId", e) Either.left(NotFoundError("Failed to get referral list")) } } @@ -324,7 +257,7 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu { NotFoundError("Failed to get referred-by. ${it.message}") }, { Person(name = it) }) } catch (e: Exception) { - logger.error("Failed to get referred-by", e) + logger.error("Failed to get referred-by for subscriberId $subscriberId", e) Either.left(NotFoundError("Failed to get referred-by")) } } @@ -369,7 +302,7 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu { createAndStorePaymentProfile(subscriberId) }, { profileInfo -> Either.right(profileInfo) } ) - .flatMap { profileInfo -> paymentProcessor.addSource(profileInfo.id, sourceId) } + .flatMap { profileInfo -> paymentProcessor.addSource(profileInfo.id, sourceId).mapLeft { NotFoundError(it.description) } } } override fun setDefaultSource(subscriberId: String, sourceId: String): Either { @@ -378,7 +311,7 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu { createAndStorePaymentProfile(subscriberId) }, { profileInfo -> Either.right(profileInfo) } ) - .flatMap { profileInfo -> paymentProcessor.setDefaultSource(profileInfo.id, sourceId) } + .flatMap { profileInfo -> paymentProcessor.setDefaultSource(profileInfo.id, sourceId).mapLeft { NotFoundError(it.description) } } } override fun listSources(subscriberId: String): Either> { @@ -387,7 +320,8 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu { createAndStorePaymentProfile(subscriberId) }, { profileInfo -> Either.right(profileInfo) } ) - .flatMap { profileInfo -> paymentProcessor.getSavedSources(profileInfo.id) } + .flatMap { profileInfo -> paymentProcessor.getSavedSources(profileInfo.id).mapLeft { NotFoundError(it.description) } } + } diff --git a/client-api/src/test/kotlin/org/ostelco/prime/client/api/auth/GetUserInfoTest.kt b/client-api/src/test/kotlin/org/ostelco/prime/client/api/auth/GetUserInfoTest.kt index ff497953a..b8bd22a42 100644 --- a/client-api/src/test/kotlin/org/ostelco/prime/client/api/auth/GetUserInfoTest.kt +++ b/client-api/src/test/kotlin/org/ostelco/prime/client/api/auth/GetUserInfoTest.kt @@ -71,7 +71,7 @@ class GetUserInfoTest { } if (counter == 0) { - fail("Couldn't connect to RULE server") + fail("Couldn't connect to RULE server") } } diff --git a/dataflow-pipelines/README.md b/dataflow-pipelines/README.md index 87c729277..87aa7b4e8 100644 --- a/dataflow-pipelines/README.md +++ b/dataflow-pipelines/README.md @@ -7,11 +7,7 @@ ## Package - gradle clean shadowJar - -With unit testing: - - gradle clean test shadowJar + gradle clean build ## Deploy to GCP diff --git a/dataflow-pipelines/build.gradle b/dataflow-pipelines/build.gradle index ea2eb9480..59461ef65 100644 --- a/dataflow-pipelines/build.gradle +++ b/dataflow-pipelines/build.gradle @@ -6,12 +6,18 @@ plugins { } dependencies { - implementation project(':analytics-grpc-api') + implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion" + + implementation project(':analytics-grpc-api') + implementation "com.google.cloud:google-cloud-pubsub:$googleCloudVersion" + implementation 'com.google.cloud.dataflow:google-cloud-dataflow-java-sdk-all:2.5.0' runtimeOnly 'org.apache.beam:beam-runners-google-cloud-dataflow-java:2.5.0' + implementation 'ch.qos.logback:logback-classic:1.2.3' + testImplementation "org.jetbrains.kotlin:kotlin-test-junit:$kotlinVersion" testRuntimeOnly 'org.hamcrest:hamcrest-all:1.3' } diff --git a/dataflow-pipelines/docker-compose.yaml b/dataflow-pipelines/docker-compose.yaml index c892a375e..d5a8058b9 100644 --- a/dataflow-pipelines/docker-compose.yaml +++ b/dataflow-pipelines/docker-compose.yaml @@ -1,7 +1,7 @@ version: "3.7" services: - analytics: + dataflow-pipelines: container_name: dataflow-pipelines build: . environment: diff --git a/dataflow-pipelines/script/start.sh b/dataflow-pipelines/script/start.sh index c98400eb0..9ee58596d 100755 --- a/dataflow-pipelines/script/start.sh +++ b/dataflow-pipelines/script/start.sh @@ -3,4 +3,4 @@ # Start app exec java \ -Dfile.encoding=UTF-8 \ - -jar /analytics.jar + -jar /dataflow-pipelines.jar diff --git a/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/definitions/DataConsumptionPipelineDefinition.kt b/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/definitions/DataConsumptionPipelineDefinition.kt index 34b0abf3c..0eaa0cc39 100644 --- a/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/definitions/DataConsumptionPipelineDefinition.kt +++ b/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/definitions/DataConsumptionPipelineDefinition.kt @@ -5,6 +5,7 @@ import org.apache.beam.sdk.Pipeline import org.apache.beam.sdk.coders.KvCoder import org.apache.beam.sdk.coders.VarLongCoder import org.apache.beam.sdk.extensions.protobuf.ProtoCoder +import org.apache.beam.sdk.io.gcp.bigquery.TableRowJsonCoder import org.apache.beam.sdk.transforms.Combine import org.apache.beam.sdk.transforms.Filter import org.apache.beam.sdk.transforms.GroupByKey @@ -27,9 +28,6 @@ import org.ostelco.dataflow.pipelines.io.Table.RAW_CONSUMPTION import org.ostelco.dataflow.pipelines.io.convertToHourlyTableRows import org.ostelco.dataflow.pipelines.io.convertToRawTableRows import org.ostelco.dataflow.pipelines.io.readFromPubSub -import java.time.ZoneOffset -import java.time.ZonedDateTime -import java.time.format.DateTimeFormatter object DataConsumptionPipelineDefinition : PipelineDefinition { @@ -53,12 +51,14 @@ object DataConsumptionPipelineDefinition : PipelineDefinition { // PubSubEvents -> raw_consumption big-query dataTrafficInfoEvents .apply("convertToRawTableRows", convertToRawTableRows) + .setCoder(TableRowJsonCoder.of()) .apply("saveRawEventsToBigQuery", saveToBigQuery(RAW_CONSUMPTION)) // PubSubEvents -> aggregate by hour -> hourly_consumption big-query dataTrafficInfoEvents .apply("TotalDataConsumptionGroupByMsisdn", consumptionPerMsisdn) .apply("convertToHourlyTableRows", convertToHourlyTableRows) + .setCoder(TableRowJsonCoder.of()) .apply("saveToBigQueryGroupedByHour", saveToBigQuery(HOURLY_CONSUMPTION)) } } @@ -80,16 +80,11 @@ val consumptionPerMsisdn = object : PTransform, PCo .discardingFiredPanes() val toKeyValuePair = ParDoFn.transform> { - val zonedDateTime = ZonedDateTime - .ofInstant(java.time.Instant.ofEpochMilli(Timestamps.toMillis(it.timestamp)), ZoneOffset.UTC) - .withMinute(0) - .withSecond(0) - .withNano(0) - val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:SS") + val hoursSinceEpoch: Long = it.timestamp.seconds / 3600 KV.of( AggregatedDataTrafficInfo.newBuilder() .setMsisdn(it.msisdn) - .setDateTime(formatter.format(zonedDateTime)) + .setTimestamp(Timestamps.fromSeconds(hoursSinceEpoch * 3600)) .setDataBytes(0) .build(), it.bucketBytes) @@ -100,7 +95,7 @@ val consumptionPerMsisdn = object : PTransform, PCo val kvToSingleObject = ParDoFn.transform, AggregatedDataTrafficInfo> { AggregatedDataTrafficInfo.newBuilder() .setMsisdn(it.key?.msisdn) - .setDateTime(it.key?.dateTime) + .setTimestamp(it.key?.timestamp) .setDataBytes(it.value) .build() } @@ -119,5 +114,6 @@ val consumptionPerMsisdn = object : PTransform, PCo // sum for each group .apply("reduceToSumOfBucketBytes", reduceToSumOfBucketBytes) .apply("kvToSingleObject", kvToSingleObject) + .setCoder(ProtoCoder.of(AggregatedDataTrafficInfo::class.java)) } } \ No newline at end of file diff --git a/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/io/BigQuery.kt b/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/io/BigQuery.kt index b65e8e4c7..6316583e7 100644 --- a/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/io/BigQuery.kt +++ b/dataflow-pipelines/src/main/kotlin/org/ostelco/dataflow/pipelines/io/BigQuery.kt @@ -3,6 +3,7 @@ package org.ostelco.dataflow.pipelines.io import com.google.api.services.bigquery.model.TableFieldSchema import com.google.api.services.bigquery.model.TableRow import com.google.api.services.bigquery.model.TableSchema +import com.google.protobuf.Timestamp import com.google.protobuf.util.Timestamps import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO import org.ostelco.analytics.api.AggregatedDataTrafficInfo @@ -58,7 +59,7 @@ private object TableSchemas { val fields = ArrayList() fields.add(TableFieldSchema().setName("msisdn").setType("STRING")) fields.add(TableFieldSchema().setName("bytes").setType("INTEGER")) - fields.add(TableFieldSchema().setName("timestamp").setType("DATETIME")) + fields.add(TableFieldSchema().setName("timestamp").setType("TIMESTAMP")) TableSchema().setFields(fields) } } @@ -73,18 +74,19 @@ val convertToRawTableRows = ParDoFn.transform { .set("msisdn", it.msisdn) .set("bucketBytes", it.bucketBytes) .set("bundleBytes", it.bundleBytes) - .set("timestamp", ZonedDateTime.ofInstant( - Instant.ofEpochMilli(Timestamps.toMillis(it.timestamp)), - ZoneOffset.UTC).toString()) + .set("timestamp", protobufTimestampToZonedDateTime(it.timestamp)) } val convertToHourlyTableRows = ParDoFn.transform { TableRow() .set("msisdn", it.msisdn) .set("bytes", it.dataBytes) - .set("timestamp", it.dateTime) + .set("timestamp", protobufTimestampToZonedDateTime(it.timestamp)) } +fun protobufTimestampToZonedDateTime(timestamp: Timestamp) = ZonedDateTime.ofInstant( + Instant.ofEpochMilli(Timestamps.toMillis(timestamp)), + ZoneOffset.UTC).toString() // // Save to BigQuery Table // @@ -102,7 +104,6 @@ object BigQueryIOUtils { return BigQueryIO.writeTableRows() .to("$project:$dataset.${table.name.toLowerCase()}") .withSchema(TableSchemas.getTableSchema(table)) - .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_APPEND) } } diff --git a/dataflow-pipelines/src/main/resources/table_schema.ddl b/dataflow-pipelines/src/main/resources/table_schema.ddl new file mode 100644 index 000000000..2e6d229b3 --- /dev/null +++ b/dataflow-pipelines/src/main/resources/table_schema.ddl @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS +`pantel-2decb.data_consumption.hourly_consumption` +( + msisdn STRING NOT NULL, + bytes INT64 NOT NULL, + timestamp TIMESTAMP NOT NULL +) +PARTITION BY DATE(timestamp); + + +CREATE TABLE IF NOT EXISTS +`pantel-2decb.data_consumption.raw_consumption` +( + msisdn STRING NOT NULL, + bucketBytes INT64 NOT NULL, + bundleBytes INT64 NOT NULL, + timestamp TIMESTAMP NOT NULL +) +PARTITION BY DATE(timestamp); \ No newline at end of file diff --git a/dataflow-pipelines/src/test/kotlin/org/ostelco/dataflow/pipelines/ConsumptionPerMsisdnTest.kt b/dataflow-pipelines/src/test/kotlin/org/ostelco/dataflow/pipelines/ConsumptionPerMsisdnTest.kt index 0b26d7bc7..b8ac4121c 100644 --- a/dataflow-pipelines/src/test/kotlin/org/ostelco/dataflow/pipelines/ConsumptionPerMsisdnTest.kt +++ b/dataflow-pipelines/src/test/kotlin/org/ostelco/dataflow/pipelines/ConsumptionPerMsisdnTest.kt @@ -1,5 +1,6 @@ package org.ostelco.dataflow.pipelines +import com.google.protobuf.Timestamp import com.google.protobuf.util.Timestamps import org.apache.beam.sdk.extensions.protobuf.ProtoCoder import org.apache.beam.sdk.testing.NeedsRunner @@ -14,9 +15,6 @@ import org.junit.experimental.categories.Category import org.ostelco.analytics.api.AggregatedDataTrafficInfo import org.ostelco.analytics.api.DataTrafficInfo import org.ostelco.dataflow.pipelines.definitions.consumptionPerMsisdn -import java.time.ZoneOffset -import java.time.ZonedDateTime -import java.time.format.DateTimeFormatter class ConsumptionPerMsisdnTest { @@ -80,21 +78,13 @@ class ConsumptionPerMsisdnTest { .setCoder(ProtoCoder.of(AggregatedDataTrafficInfo::class.java)) PAssert.that(out).containsInAnyOrder( - AggregatedDataTrafficInfo.newBuilder().setMsisdn("123").setDataBytes(300).setDateTime(currentHourDateTime).build(), - AggregatedDataTrafficInfo.newBuilder().setMsisdn("456").setDataBytes(200).setDateTime(currentHourDateTime).build(), - AggregatedDataTrafficInfo.newBuilder().setMsisdn("789").setDataBytes(100).setDateTime(currentHourDateTime).build()) + AggregatedDataTrafficInfo.newBuilder().setMsisdn("123").setDataBytes(300).setTimestamp(currentHourDateTime).build(), + AggregatedDataTrafficInfo.newBuilder().setMsisdn("456").setDataBytes(200).setTimestamp(currentHourDateTime).build(), + AggregatedDataTrafficInfo.newBuilder().setMsisdn("789").setDataBytes(100).setTimestamp(currentHourDateTime).build()) pipeline.run().waitUntilFinish() } } - private fun getCurrentHourDateTime(): String { - val zonedDateTime = ZonedDateTime - .ofInstant(java.time.Instant.now(), ZoneOffset.UTC) - .withMinute(0) - .withSecond(0) - .withNano(0) - val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:SS") - return formatter.format(zonedDateTime) - } + private fun getCurrentHourDateTime(): Timestamp = Timestamps.fromSeconds((java.time.Instant.now().epochSecond / 3600) * 3600) } \ No newline at end of file diff --git a/diameter-test/src/main/kotlin/org/ostelco/diameter/test/TestHelper.kt b/diameter-test/src/main/kotlin/org/ostelco/diameter/test/TestHelper.kt index ddaeb83a2..8b0bb8dde 100644 --- a/diameter-test/src/main/kotlin/org/ostelco/diameter/test/TestHelper.kt +++ b/diameter-test/src/main/kotlin/org/ostelco/diameter/test/TestHelper.kt @@ -153,4 +153,11 @@ object TestHelper { addTerminateRequest(ccrAvps, ratingGroup = 10, serviceIdentifier = 1, bucketSize = bucketSize) addServiceInformation(ccrAvps, apn = APN, sgsnMncMcc = SGSN_MCC_MNC) } + + @JvmStatic + fun createTerminateRequest(ccrAvps: AvpSet, msisdn: String) { + buildBasicRequest(ccrAvps, RequestType.TERMINATION_REQUEST, requestNumber = 2) + addUser(ccrAvps, msisdn = msisdn, imsi = IMSI) + addServiceInformation(ccrAvps, apn = APN, sgsnMncMcc = SGSN_MCC_MNC) + } } \ No newline at end of file diff --git a/docker-compose.override.yaml b/docker-compose.override.yaml index 9d810bf56..16b2785f0 100644 --- a/docker-compose.override.yaml +++ b/docker-compose.override.yaml @@ -8,7 +8,6 @@ services: dockerfile: Dockerfile.test environment: - FIREBASE_ROOT_PATH=test - - GOOGLE_APPLICATION_CREDENTIALS=/secret/pantel-prod.json - PUBSUB_EMULATOR_HOST=pubsub-emulator:8085 - PUBSUB_PROJECT_ID=pantel-2decb - STRIPE_API_KEY=${STRIPE_API_KEY} @@ -93,6 +92,7 @@ services: environment: - PRIME_SOCKET=prime:8080 - STRIPE_API_KEY=${STRIPE_API_KEY} + - GOOGLE_APPLICATION_CREDENTIALS=/secret/pantel-prod.json networks: net: ipv4_address: 172.16.238.2 @@ -100,7 +100,7 @@ services: neo4j: container_name: "neo4j" - image: neo4j:3.4.4 + image: neo4j:3.4.7 environment: - NEO4J_AUTH=none ports: diff --git a/docs/NEO4J.md b/docs/NEO4J.md index bf7c42976..881d7ba3c 100644 --- a/docs/NEO4J.md +++ b/docs/NEO4J.md @@ -32,7 +32,7 @@ kubectl config get-contexts If name of the cluster, where neo4j is deployed, is `private-cluster`, then change `kubectl config`. ```bash -kubectl config set-context $(kubectl config get-contexts --output name | grep private-cluster) +kubectl config use-context $(kubectl config get-contexts --output name | grep private-cluster) ``` ### Port forward from neo4j pods diff --git a/docs/TEST.md b/docs/TEST.md index 4b93d4df8..b24b9098f 100644 --- a/docs/TEST.md +++ b/docs/TEST.md @@ -29,7 +29,7 @@ openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ./nginx.key -out ./n cp nginx.crt ../../ocsgw/config/metrics.crt ``` -### Test ext-pgw -- ocsgw -- prime --firebase +### Test acceptance-tests ```bash gradle clean build @@ -40,12 +40,6 @@ docker-compose up --build --abort-on-container-exit ```bash gradle prime:integration -``` - - * Test pubsub -- pseudonymiser(--datastore) -- pubsub - -```bash -docker-compose up --build -f docker-compose.yaml -f docker-compose.pseu.yaml --abort-on-container-exit ``` ## Configuring emulators diff --git a/docs/test-deployment/deployment.png b/docs/test-deployment/deployment.png index 964ec59db..4e2d59309 100644 Binary files a/docs/test-deployment/deployment.png and b/docs/test-deployment/deployment.png differ diff --git a/docs/test-deployment/deployment.puml b/docs/test-deployment/deployment.puml index 89aa3bd09..a083e744c 100644 --- a/docs/test-deployment/deployment.puml +++ b/docs/test-deployment/deployment.puml @@ -1,19 +1,16 @@ @startuml package "Docker" { [Acceptance Test Runner] - [Ext-pgw] - [Ocsgw] + [ocsgw] [ESP] [Prime] database DB } -[Acceptance Test Runner] -> [Ext-pgw] : http -[Acceptance Test Runner] -> [Prime] : http -[Ext-pgw] -> [Ocsgw] : diameter -[Ocsgw] -> [ESP] : gRPC -[Ocsgw] -> [Prime] : 8082 -[ESP] -> [Prime] : 8080 +[Acceptance Test Runner] --> [ocsgw] : diameter +[Acceptance Test Runner] -> [Prime] : http @ 8080 +[ocsgw] -> [ESP] : gRPC +[ESP] -> [Prime] : gRPC @ 8082 [Prime] -> DB @enduml diff --git a/embedded-graph-store/src/main/kotlin/org/ostelco/prime/storage/embeddedgraph/GraphStore.kt b/embedded-graph-store/src/main/kotlin/org/ostelco/prime/storage/embeddedgraph/GraphStore.kt deleted file mode 100644 index 35822dc88..000000000 --- a/embedded-graph-store/src/main/kotlin/org/ostelco/prime/storage/embeddedgraph/GraphStore.kt +++ /dev/null @@ -1,183 +0,0 @@ -package org.ostelco.prime.storage.embeddedgraph - -import org.ostelco.prime.model.ApplicationToken -import org.ostelco.prime.model.Entity -import org.ostelco.prime.model.Offer -import org.ostelco.prime.model.Product -import org.ostelco.prime.model.ProductClass -import org.ostelco.prime.model.PurchaseRecord -import org.ostelco.prime.model.Segment -import org.ostelco.prime.model.Subscriber -import org.ostelco.prime.model.Subscription -import org.ostelco.prime.storage.AdminDataStore -import org.ostelco.prime.storage.legacy.Storage -import org.ostelco.prime.storage.legacy.StorageException -import java.util.* -import java.util.stream.Collectors - -class GraphStore : Storage by GraphStoreSingleton, AdminDataStore by GraphStoreSingleton - -object GraphStoreSingleton : Storage, AdminDataStore { - - private val subscriberEntity = EntityType("Subscriber", Subscriber::class.java) - private val subscriberStore = EntityStore(subscriberEntity) - - private val productEntity = EntityType("Product", Product::class.java) - private val productStore = EntityStore(productEntity) - - private val subscriptionEntity = EntityType("Subscription", Subscription::class.java) - private val subscriptionStore = EntityStore(subscriptionEntity) - - private val notificationTokenEntity = EntityType("NotificationToken", ApplicationToken::class.java) - private val notificationTokenStore = EntityStore(notificationTokenEntity) - - private val subscriptionRelation = RelationType( - name = "HAS_SUBSCRIPTION", - from = subscriberEntity, - to = subscriptionEntity, - dataClass = Void::class.java) - private val subscriptionRelationStore = RelationStore(subscriptionRelation) - - private val purchaseRecordRelation = RelationType( - name = "PURCHASED", - from = subscriberEntity, - to = productEntity, - dataClass = PurchaseRecord::class.java) - private val purchaseRecordStore = RelationStore(purchaseRecordRelation) - - override val balances: Map - get() = subscriptionStore.getAll().mapValues { it.value.balance } - - override fun getSubscriber(id: String): Subscriber? = subscriberStore.get(id) - - override fun addSubscriber(subscriber: Subscriber): Boolean = subscriberStore.create(subscriber.id, subscriber) - - override fun updateSubscriber(subscriber: Subscriber): Boolean = subscriberStore.update(subscriber.id, subscriber) - - override fun removeSubscriber(id: String) = subscriberStore.delete(id) - - override fun addSubscription(id: String, msisdn: String): Boolean { - val from = subscriberStore.get(id) ?: return false - subscriptionStore.create(msisdn, Subscription(msisdn, 0L)) - val to = subscriptionStore.get(msisdn) ?: return false - return subscriptionRelationStore.create(from, null, to) - } - - override fun getProducts(subscriberId: String): Map { - val result = GraphServer.graphDb.execute( - """ - MATCH (:${subscriberEntity.name} {id: '$subscriberId'}) - <-[:${segmentToSubscriberRelation.name}]-(:${segmentEntity.name}) - <-[:${offerToSegmentRelation.name}]-(:${offerEntity.name}) - -[:${offerToProductRelation.name}]->(product:${productEntity.name}) - RETURN properties(product) AS product - """.trimIndent()) - - return result.stream() - .map { ObjectHandler.getObject(it["product"] as Map, Product::class.java) } - .collect(Collectors.toMap({ it?.sku }, { it })) - } - - override fun getProduct(subscriberId: String?, sku: String): Product? = productStore.get(sku) - - override fun getBalance(id: String): Long? { - return subscriberStore.getRelated(id, subscriptionRelation, subscriptionEntity) - .first() - .balance - } - - override fun setBalance(msisdn: String, noOfBytes: Long): Boolean = - subscriptionStore.update(msisdn, Subscription(msisdn, balance = noOfBytes)) - - override fun getMsisdn(subscriptionId: String): String? { - return subscriberStore.getRelated(subscriptionId, subscriptionRelation, subscriptionEntity) - .first() - .msisdn - } - - override fun getPurchaseRecords(id: String): Collection { - return subscriberStore.getRelations(id, purchaseRecordRelation) - } - - override fun addPurchaseRecord(id: String, purchase: PurchaseRecord): String? { - val subscriber = subscriberStore.get(id) ?: throw StorageException("Subscriber not found") - val product = productStore.get(purchase.product.sku) ?: throw StorageException("Product not found") - purchase.id = UUID.randomUUID().toString() - purchaseRecordStore.create(subscriber, purchase, product) - return purchase.id - } - - override fun getNotificationTokens(msisdn: String): Collection = notificationTokenStore.getAll().values - - override fun addNotificationToken(msisdn: String, token: ApplicationToken): Boolean = notificationTokenStore.create("$msisdn.${token.applicationID}", token) - - override fun getNotificationToken(msisdn: String, applicationID: String): ApplicationToken? = notificationTokenStore.get("$msisdn.$applicationID") - - override fun removeNotificationToken(msisdn: String, applicationID: String): Boolean = notificationTokenStore.delete("$msisdn.$applicationID") - // - // Admin Store - // - - private val offerEntity = EntityType("Offer", Entity::class.java) - private val offerStore = EntityStore(offerEntity) - - private val segmentEntity = EntityType("Segment", Entity::class.java) - private val segmentStore = EntityStore(segmentEntity) - - private val offerToSegmentRelation = RelationType("offerHasSegment", offerEntity, segmentEntity, Void::class.java) - private val offerToSegmentStore = RelationStore(offerToSegmentRelation) - - private val offerToProductRelation = RelationType("offerHasProduct", offerEntity, productEntity, Void::class.java) - private val offerToProductStore = RelationStore(offerToProductRelation) - - private val segmentToSubscriberRelation = RelationType("segmentToSubscriber", segmentEntity, subscriberEntity, Void::class.java) - private val segmentToSubscriberStore = RelationStore(segmentToSubscriberRelation) - - private val productClassEntity = EntityType("ProductClass", ProductClass::class.java) - private val productClassStore = EntityStore(productClassEntity) - - override fun createProductClass(productClass: ProductClass): Boolean = productClassStore.create(productClass.id, productClass) - - override fun createProduct(product: Product): Boolean = productStore.create(product.sku, product) - - override fun createSegment(segment: Segment) { - segmentStore.create(segment.id, segment) - updateSegment(segment) - } - - override fun createOffer(offer: Offer) { - offerStore.create(offer.id, offer) - offerToSegmentStore.create(offer.id, offer.segments) - offerToProductStore.create(offer.id, offer.products) - } - - override fun updateSegment(segment: Segment) { - segmentToSubscriberStore.create(segment.id, segment.subscribers) - } - - override fun getPaymentId(id: String): String? { - TODO("not implemented") - } - - override fun deletePaymentId(id: String): Boolean { - TODO("not implemented") - } - - override fun createPaymentId(id: String, paymentId: String): Boolean { - TODO("not implemented") - } - - override fun getCustomerId(id: String): String? { - TODO("not implemented") //To change body of created functions use File | Settings | File Templates. - } - - // override fun getOffers(): Collection = offerStore.getAll().values.map { Offer().apply { id = it.id } } - - // override fun getSegments(): Collection = segmentStore.getAll().values.map { Segment().apply { id = it.id } } - - // override fun getOffer(id: String): Offer? = offerStore.get(id)?.let { Offer().apply { this.id = it.id } } - - // override fun getSegment(id: String): Segment? = segmentStore.get(id)?.let { Segment().apply { this.id = it.id } } - - // override fun getProductClass(id: String): ProductClass? = productClassStore.get(id) -} \ No newline at end of file diff --git a/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt b/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt index fd7659c2b..d02412ace 100644 --- a/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt +++ b/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt @@ -84,12 +84,25 @@ data class PurchaseRecord( val product: Product, val timestamp: Long) : HasId +data class PurchaseRecordInfo(override val id: String, + val subscriberId: String, + val product: Product, + val timestamp: Long, + val status: String) : HasId { + constructor(purchaseRecord: PurchaseRecord, subscriberId: String, status: String) : this( + purchaseRecord.id, + subscriberId, + purchaseRecord.product, + purchaseRecord.timestamp, + status) +} + data class PseudonymEntity( - val msisdn: String, + val sourceId: String, val pseudonym: String, val start: Long, val end: Long) data class ActivePseudonyms( val current: PseudonymEntity, - val next: PseudonymEntity) \ No newline at end of file + val next: PseudonymEntity) diff --git a/neo4j-store/build.gradle b/neo4j-store/build.gradle index 8ebf3be0e..a3db2a884 100644 --- a/neo4j-store/build.gradle +++ b/neo4j-store/build.gradle @@ -32,7 +32,7 @@ dependencies { testImplementation "org.jetbrains.kotlin:kotlin-test:$kotlinVersion" testImplementation "org.jetbrains.kotlin:kotlin-test-junit:$kotlinVersion" - testImplementation "org.mockito:mockito-core:2.18.3" + testImplementation "org.mockito:mockito-core:$mockitoVersion" } apply from: '../jacoco.gradle' \ No newline at end of file diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index b2f109993..08cbc4451 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -1,8 +1,13 @@ package org.ostelco.prime.storage.graph import arrow.core.Either +import arrow.core.Tuple4 import arrow.core.flatMap import org.neo4j.driver.v1.Transaction +import org.ostelco.prime.analytics.AnalyticsService +import org.ostelco.prime.analytics.PrimeMetric.REVENUE +import org.ostelco.prime.analytics.PrimeMetric.USERS_PAID_AT_LEAST_ONCE +import org.ostelco.prime.core.ApiError import org.ostelco.prime.logger import org.ostelco.prime.model.Bundle import org.ostelco.prime.model.Offer @@ -14,6 +19,13 @@ import org.ostelco.prime.model.Subscriber import org.ostelco.prime.model.Subscription import org.ostelco.prime.module.getResource import org.ostelco.prime.ocs.OcsAdminService +import org.ostelco.prime.ocs.OcsSubscriberService +import org.ostelco.prime.paymentprocessor.PaymentProcessor +import org.ostelco.prime.paymentprocessor.core.BadGatewayError +import org.ostelco.prime.paymentprocessor.core.PaymentError +import org.ostelco.prime.paymentprocessor.core.ProductInfo +import org.ostelco.prime.paymentprocessor.core.ProfileInfo +import org.ostelco.prime.storage.DocumentStore import org.ostelco.prime.storage.GraphStore import org.ostelco.prime.storage.NotFoundError import org.ostelco.prime.storage.StoreError @@ -47,7 +59,7 @@ class Neo4jStore : GraphStore by Neo4jStoreSingleton object Neo4jStoreSingleton : GraphStore { - private val ocs: OcsAdminService by lazy { getResource() } + private val ocsAdminService: OcsAdminService by lazy { getResource() } private val logger by logger() // @@ -130,6 +142,7 @@ object Neo4jStoreSingleton : GraphStore { readTransaction { subscriberStore.get(subscriberId, transaction) } // TODO vihang: Move this logic to DSL + Rule Engine + Triggers, when they are ready + // >> BEGIN override fun addSubscriber(subscriber: Subscriber, referredBy: String?): Either = writeTransaction { if (subscriber.id == referredBy) { @@ -158,7 +171,7 @@ object Neo4jStoreSingleton : GraphStore { } } .flatMap { - ocs.addBundle(Bundle(bundleId, 1_000_000_000)) + ocsAdminService.addBundle(Bundle(bundleId, 1_000_000_000)) Either.right(Unit) } } else { @@ -176,14 +189,15 @@ object Neo4jStoreSingleton : GraphStore { } } .flatMap { - ocs.addBundle(Bundle(bundleId, 100_000_000)) + ocsAdminService.addBundle(Bundle(bundleId, 100_000_000)) Either.right(Unit) } }.flatMap { subscriberToBundleStore.create(subscriber.id, bundleId, transaction) } .flatMap { subscriberToSegmentStore.create(subscriber.id, "all", transaction) } .ifFailedThenRollback(transaction) } - + // << END + override fun updateSubscriber(subscriber: Subscriber): Either = writeTransaction { subscriberStore.update(subscriber, transaction) .ifFailedThenRollback(transaction) @@ -232,7 +246,7 @@ object Neo4jStoreSingleton : GraphStore { either.flatMap { _ -> subscriptionToBundleStore.create(subscription, bundle, transaction) .flatMap { - ocs.addMsisdnToBundleMapping(msisdn, bundle.id) + ocsAdminService.addMsisdnToBundleMapping(msisdn, bundle.id) Either.right(Unit) } } @@ -282,30 +296,152 @@ object Neo4jStoreSingleton : GraphStore { override fun getProduct(subscriberId: String, sku: String): Either { return readTransaction { - subscriberStore.exists(subscriberId, transaction) - .flatMap { - read(""" + getProduct(subscriberId, sku, transaction) + } + } + + private fun getProduct(subscriberId: String, sku: String, transaction: Transaction): Either { + return subscriberStore.exists(subscriberId, transaction) + .flatMap { + read(""" MATCH (:${subscriberEntity.name} {id: '$subscriberId'}) -[:${subscriberToSegmentRelation.relation.name}]->(:${segmentEntity.name}) <-[:${offerToSegmentRelation.relation.name}]-(:${offerEntity.name}) -[:${offerToProductRelation.relation.name}]->(product:${productEntity.name} {sku: '$sku'}) RETURN product; """.trimIndent(), - transaction) { statementResult -> - if (statementResult.hasNext()) { - Either.right(productEntity.createEntity(statementResult.single().get("product").asMap())) - } else { - Either.left(NotFoundError(type = productEntity.name, id = sku)) - } + transaction) { statementResult -> + if (statementResult.hasNext()) { + Either.right(productEntity.createEntity(statementResult.single().get("product").asMap())) + } else { + Either.left(NotFoundError(type = productEntity.name, id = sku)) } } - } + } } // // Purchase Records // + // TODO vihang: Move this logic to DSL + Rule Engine + Triggers, when they are ready + // >> BEGIN + private val documentStore by lazy { getResource() } + private val paymentProcessor by lazy { getResource() } + private val ocs by lazy { getResource() } + private val analyticsReporter by lazy { getResource() } + + private fun getPaymentProfile(name: String): Either = + documentStore.getPaymentId(name) + ?.let { profileInfoId -> Either.right(ProfileInfo(profileInfoId)) } + ?: Either.left(BadGatewayError("Failed to fetch payment customer ID")) + + private fun createAndStorePaymentProfile(name: String): Either { + return paymentProcessor.createPaymentProfile(name) + .flatMap { profileInfo -> + setPaymentProfile(name, profileInfo) + .map { profileInfo } + } + } + + private fun setPaymentProfile(name: String, profileInfo: ProfileInfo): Either = + Either.cond( + test = documentStore.createPaymentId(name, profileInfo.id), + ifTrue = { Unit }, + ifFalse = { BadGatewayError("Failed to save payment customer ID") }) + + override fun purchaseProduct( + subscriberId: String, + sku: String, + sourceId: String?, + saveCard: Boolean): Either = writeTransaction { + + val result = getProduct(subscriberId, sku, transaction) + // If we can't find the product, return not-found + .mapLeft { org.ostelco.prime.paymentprocessor.core.NotFoundError("Product unavailable") } + .flatMap { product: Product -> + // Fetch/Create stripe payment profile for the subscriber. + getPaymentProfile(subscriberId) + .fold( + { createAndStorePaymentProfile(subscriberId) }, + { profileInfo -> Either.right(profileInfo) } + ) + .map { profileInfo -> Pair(product, profileInfo) } + } + .flatMap { (product, profileInfo) -> + // Add payment source + if (sourceId != null) { + paymentProcessor.addSource(profileInfo.id, sourceId).map { sourceInfo -> Triple(product, profileInfo, sourceInfo.id) } + } else { + Either.right(Triple(product, profileInfo, null)) + } + } + .flatMap { (product, profileInfo, savedSourceId) -> + // Authorize stripe charge for this purchase + val price = product.price + //TODO: If later steps fail, then refund the authorized charge + paymentProcessor.authorizeCharge(profileInfo.id, savedSourceId, price.amount, price.currency) + .mapLeft { apiError -> + logger.error("failed to authorize purchase for customerId ${profileInfo.id}, sourceId $savedSourceId, sku $sku") + apiError + } + .map { chargeId -> Tuple4(profileInfo, savedSourceId, chargeId, product) } + } + .flatMap { (profileInfo, savedSourceId, chargeId, product) -> + val purchaseRecord = PurchaseRecord( + id = chargeId, + product = product, + timestamp = Instant.now().toEpochMilli(), + msisdn = "") + // Create purchase record + createPurchaseRecordRelation(subscriberId, purchaseRecord, transaction) + .mapLeft { storeError -> + paymentProcessor.refundCharge(chargeId) + logger.error("failed to save purchase record, for customerId ${profileInfo.id}, chargeId $chargeId, payment will be unclaimed in Stripe") + BadGatewayError(storeError.message) + } + // Notify OCS + .flatMap { + //TODO: While aborting transactions, send a record with "reverted" status + analyticsReporter.reportPurchaseInfo( + purchaseRecord = purchaseRecord, + subscriberId = subscriberId, + status = "success") + //TODO vihang: Handle errors (when it becomes available) + ocs.topup(subscriberId, sku) + // TODO vihang: handle currency conversion + analyticsReporter.reportMetric(REVENUE, product.price.amount.toLong()) + analyticsReporter.reportMetric(USERS_PAID_AT_LEAST_ONCE, getPaidSubscriberCount(transaction)) + Either.right(Tuple4(profileInfo, savedSourceId, chargeId, product)) + } + } + .mapLeft { error -> + transaction.failure() + error + } + + result.map { (profileInfo, _, chargeId, _) -> + // Capture the charge, our database have been updated. + paymentProcessor.captureCharge(chargeId, profileInfo.id) + .mapLeft { + // TODO payment: retry capture charge + logger.error("Capture failed for customerId ${profileInfo.id}, chargeId $chargeId, Fix this in Stripe Dashboard") + } + } + result.map { (profileInfo, savedSourceId, _, _) -> + // Remove the payment source + if (!saveCard && savedSourceId != null) { + paymentProcessor.removeSource(profileInfo.id, savedSourceId) + .mapLeft { paymentError -> + logger.error("Failed to remove card, for customerId ${profileInfo.id}, sourceId $sourceId") + paymentError + } + } + } + result.map { (_, _, _, product) -> ProductInfo(product.sku) } + } + // << END + override fun getPurchaseRecords(subscriberId: String): Either> { return readTransaction { subscriberStore.getRelations(subscriberId, purchaseRecordRelation, transaction) @@ -429,7 +565,11 @@ object Neo4jStoreSingleton : GraphStore { } override fun getPaidSubscriberCount(): Long = readTransaction { - read(""" + getPaidSubscriberCount(transaction) + } + + private fun getPaidSubscriberCount(transaction: Transaction): Long { + return read(""" MATCH (subscriber:${subscriberEntity.name})-[:${purchaseRecordRelation.relation.name}]->(product:${productEntity.name}) WHERE product.`price/amount` > 0 RETURN count(subscriber) AS count @@ -438,7 +578,6 @@ object Neo4jStoreSingleton : GraphStore { result.single().get("count").asLong() } } - // // Stores // diff --git a/neo4j-store/src/test/resources/docker-compose.yaml b/neo4j-store/src/test/resources/docker-compose.yaml index 28aff66dc..633d77427 100644 --- a/neo4j-store/src/test/resources/docker-compose.yaml +++ b/neo4j-store/src/test/resources/docker-compose.yaml @@ -3,7 +3,7 @@ version: "3.3" services: neo4j: container_name: "neo4j" - image: neo4j:3.4.4 + image: neo4j:3.4.7 environment: - NEO4J_AUTH=none ports: diff --git a/ocs/src/main/kotlin/org/ostelco/prime/analytics/DataConsumptionInfo.kt b/ocs/src/main/kotlin/org/ostelco/prime/analytics/DataConsumptionInfo.kt index 09f3d6366..268660faa 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/analytics/DataConsumptionInfo.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/analytics/DataConsumptionInfo.kt @@ -2,8 +2,9 @@ package org.ostelco.prime.analytics import com.lmax.disruptor.EventHandler import org.ostelco.prime.analytics.PrimeMetric.MEGABYTES_CONSUMED -import org.ostelco.prime.disruptor.EventMessageType.CREDIT_CONTROL_REQUEST +import org.ostelco.ocs.api.CreditControlRequestType import org.ostelco.prime.disruptor.OcsEvent +import org.ostelco.prime.disruptor.EventMessageType.CREDIT_CONTROL_REQUEST import org.ostelco.prime.logger import org.ostelco.prime.module.getResource @@ -29,11 +30,26 @@ class DataConsumptionInfo() : EventHandler { logger.info("Sent DataConsumptionInfo event to analytics") analyticsReporter.reportTrafficInfo( msisdn = event.msisdn!!, - usedBytes = event.usedBucketBytes, + usedBytes = event.request?.msccList?.firstOrNull()?.used?.totalOctets ?: 0L, bundleBytes = event.bundleBytes) analyticsReporter.reportMetric( primeMetric = MEGABYTES_CONSUMED, - value = event.usedBucketBytes / 1_000_000) + value = (event.request?.msccList?.firstOrNull()?.used?.totalOctets ?: 0L) / 1_000_000) + + //ToDo: Send to analytics and build pipeline + event.request?.let { request -> + if(request.type == CreditControlRequestType.INITIAL_REQUEST) { + logger.info("MSISDN : {} connected apn {} sgsn_mcc_mnc {}", + request.msisdn, + request.serviceInformation.psInformation.calledStationId, + request.serviceInformation.psInformation.sgsnMccMnc) + } else if (request.type == CreditControlRequestType.TERMINATION_REQUEST) { + logger.info("MSISDN : {} disconnected apn {} sgsn_mcc_mnc", + request.msisdn, + request.serviceInformation.psInformation.calledStationId, + request.serviceInformation.psInformation.sgsnMccMnc) + } + } } } } diff --git a/ocs/src/main/kotlin/org/ostelco/prime/disruptor/EventProducerImpl.kt b/ocs/src/main/kotlin/org/ostelco/prime/disruptor/EventProducerImpl.kt index 47edee8e0..644d4fe70 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/disruptor/EventProducerImpl.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/disruptor/EventProducerImpl.kt @@ -46,14 +46,10 @@ class EventProducerImpl(private val ringBuffer: RingBuffer) : EventPro msisdn: String? = null, bundleId: String? = null, bundleBytes: Long = 0, - requestedBytes: Long = 0, - usedBytes: Long = 0, reservedBytes: Long = 0, - serviceId: Long = 0, - ratingGroup: Long = 0, - reportingReason: ReportingReason = ReportingReason.UNRECOGNIZED, streamId: String? = null, - requestId: String? = null) { + request: CreditControlRequestInfo? = null, + topUpBytes: Long? = 0) { processNextEventOnTheRingBuffer( Consumer { event -> @@ -62,14 +58,10 @@ class EventProducerImpl(private val ringBuffer: RingBuffer) : EventPro bundleId, emptyList(), bundleBytes, - requestedBytes, - usedBytes, reservedBytes, - serviceId, - ratingGroup, - reportingReason, streamId, - requestId) + request, + topUpBytes) }) } @@ -80,7 +72,7 @@ class EventProducerImpl(private val ringBuffer: RingBuffer) : EventPro injectIntoRingBuffer( type = TOPUP_DATA_BUNDLE_BALANCE, bundleId = bundleId, - requestedBytes = bytes) + topUpBytes = bytes) } override fun releaseReservedDataBucketEvent( @@ -89,32 +81,18 @@ class EventProducerImpl(private val ringBuffer: RingBuffer) : EventPro injectIntoRingBuffer( type = RELEASE_RESERVED_BUCKET, - msisdn = msisdn, - requestedBytes = bytes) + msisdn = msisdn) } override fun injectCreditControlRequestIntoRingbuffer( request: CreditControlRequestInfo, streamId: String) { - if (request.msccList.isEmpty()) { - injectIntoRingBuffer(CREDIT_CONTROL_REQUEST, - request.msisdn, - streamId = streamId, - requestId = request.requestId) - } else { - // FIXME vihang: For now we assume that there is only 1 MSCC in the Request. - injectIntoRingBuffer(CREDIT_CONTROL_REQUEST, - msisdn = request.msisdn, - requestedBytes = request.getMscc(0).requested.totalOctets, - usedBytes = request.getMscc(0).used.totalOctets, - reservedBytes = 0, - serviceId = request.getMscc(0).serviceIdentifier, - ratingGroup = request.getMscc(0).ratingGroup, - reportingReason = request.getMscc(0).reportingReason, - streamId = streamId, - requestId = request.requestId) - } + injectIntoRingBuffer(CREDIT_CONTROL_REQUEST, + msisdn = request.msisdn, + reservedBytes = 0, + streamId = streamId, + request = request) } override fun addBundle(bundle: Bundle) { diff --git a/ocs/src/main/kotlin/org/ostelco/prime/disruptor/OcsEvent.kt b/ocs/src/main/kotlin/org/ostelco/prime/disruptor/OcsEvent.kt index 815a76669..3ee85c00c 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/disruptor/OcsEvent.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/disruptor/OcsEvent.kt @@ -1,5 +1,6 @@ package org.ostelco.prime.disruptor +import org.ostelco.ocs.api.CreditControlRequestInfo import org.ostelco.ocs.api.ReportingReason class OcsEvent { @@ -25,19 +26,6 @@ class OcsEvent { */ var msisdnToppedUp: List? = null - /** - * Origin of word 'bucket' - P-GW consumes data in `buckets` of 10 MB ~ 100 MB at a time - * This field is used in. - * Request to reserve a new bucket of bytes - */ - var requestedBucketBytes: Long = 0 - - /** - * Bytes that has been used from the bucket (previously reserved). - */ - var usedBucketBytes: Long = 0 - - /** * Buckets that has been reserved from the bundle. */ @@ -55,46 +43,25 @@ class OcsEvent { var ocsgwStreamId: String? = null /** - * Request ID used by OCS gateway to correlate response with requests + * Credit-Control-Request from OCS */ - var ocsgwRequestId: String? = null - + var request: CreditControlRequestInfo? = null; /** - * Service-Identifier is used to classify traffic + * Topup amount for bundle */ - var serviceIdentifier: Long = 0 - - /** - * Rating-Group is used to classify traffic - */ - var ratingGroup: Long = 0 - - /** - * Reporting-Reason - * // FIXME martin: This is the Reporting-Reason for the MSCC. The PrimeEvent might be to generic since there is also Reporting-Reason used on ServiceUnit level - */ - var reportingReason: ReportingReason = ReportingReason.UNRECOGNIZED + var topUpBytes: Long? = 0; fun clear() { messageType = null - msisdn = null bundleId = null - msisdnToppedUp = null - bundleBytes = 0 - requestedBucketBytes = 0 - usedBucketBytes = 0 reservedBucketBytes = 0 - bundleBytes = 0 - ocsgwStreamId = null - ocsgwRequestId = null - serviceIdentifier = 0 - ratingGroup = 0 - reportingReason = ReportingReason.UNRECOGNIZED + request = null + topUpBytes = 0; } //FIXME vihang: We need to think about roaming!!! @@ -105,26 +72,18 @@ class OcsEvent { bundleId: String?, msisdnToppedUp: List, bundleBytes: Long, - requestedBytes: Long, - usedBytes: Long, reservedBucketBytes: Long, - serviceIdentifier: Long, - ratingGroup: Long, - reportingReason: ReportingReason, ocsgwStreamId: String?, - ocsgwRequestId: String?) { + request: CreditControlRequestInfo?, + topUpBytes: Long?) { this.messageType = messageType this.msisdn = msisdn this.bundleId = bundleId this.msisdnToppedUp = msisdnToppedUp this.bundleBytes = bundleBytes - this.requestedBucketBytes = requestedBytes - this.usedBucketBytes = usedBytes this.reservedBucketBytes = reservedBucketBytes - this.serviceIdentifier = serviceIdentifier - this.ratingGroup = ratingGroup - this.reportingReason = reportingReason this.ocsgwStreamId = ocsgwStreamId - this.ocsgwRequestId = ocsgwRequestId + this.request = request + this.topUpBytes = topUpBytes } } diff --git a/ocs/src/main/kotlin/org/ostelco/prime/events/EventProcessor.kt b/ocs/src/main/kotlin/org/ostelco/prime/events/EventProcessor.kt index 59f8fc6a0..13131a944 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/events/EventProcessor.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/events/EventProcessor.kt @@ -31,8 +31,8 @@ class EventProcessor( || event.messageType == RELEASE_RESERVED_BUCKET || event.messageType == TOPUP_DATA_BUNDLE_BALANCE || event.messageType == REMOVE_MSISDN_TO_BUNDLE_MAPPING) { - logger.info("Updating data bundle balance for {} : {} to {} bytes", - event.msisdn, event.bundleId, event.bundleBytes) + logger.info("Updating data bundle balance for bundleId : {} to {} bytes", + event.bundleId, event.bundleBytes) val bundleId = event.bundleId if (bundleId != null) { storage.updateBundle(Bundle(bundleId, event.bundleBytes)) diff --git a/ocs/src/main/kotlin/org/ostelco/prime/ocs/EventHandlerImpl.kt b/ocs/src/main/kotlin/org/ostelco/prime/ocs/EventHandlerImpl.kt index 2402c9727..a7f2ba282 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/ocs/EventHandlerImpl.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/ocs/EventHandlerImpl.kt @@ -50,14 +50,17 @@ internal class EventHandlerImpl(private val ocsService: OcsService) : EventHandl } private fun logEventProcessing(msg: String, event: OcsEvent) { - logger.info("{}", msg) - logger.info("MSISDN: {}", event.msisdn) - logger.info("requested bytes: {}", event.requestedBucketBytes) - logger.info("reserved bytes: {}", event.reservedBucketBytes) - logger.info("used bytes: {}", event.usedBucketBytes) - logger.info("bundle bytes: {}", event.bundleBytes) - logger.info("Reporting reason: {}", event.reportingReason) - logger.info("request id: {} ",event.ocsgwRequestId) + val logString = """ + $msg + Msisdn: ${event.msisdn} + Requested bytes: ${event.request?.msccList?.firstOrNull()?.requested?.totalOctets ?: 0L} + Used bytes: ${event.request?.msccList?.firstOrNull()?.used?.totalOctets ?: 0L} + Bundle bytes: ${event.bundleBytes} + Topup bytes: ${event.topUpBytes} + Request id: ${event.request?.requestId} + """.trimIndent() + + logger.info(logString) } private fun handleCreditControlRequest(event: OcsEvent) { @@ -70,33 +73,33 @@ internal class EventHandlerImpl(private val ocsService: OcsService) : EventHandl try { val creditControlAnswer = CreditControlAnswerInfo.newBuilder() .setMsisdn(event.msisdn) - .setRequestId(event.ocsgwRequestId) - - // This is a hack to know when we have received an MSCC in the request or not. - // For Terminate request we might not have any MSCC and therefore no serviceIdentifier. - if (event.serviceIdentifier > 0) { - val msccBuilder = MultipleServiceCreditControl.newBuilder() - msccBuilder.setServiceIdentifier(event.serviceIdentifier) - .setRatingGroup(event.ratingGroup) - .setValidityTime(86400) - - if ((event.reportingReason != ReportingReason.FINAL) && (event.requestedBucketBytes > 0)) { - msccBuilder.granted = ServiceUnit.newBuilder() - .setTotalOctets(event.reservedBucketBytes) - .build() - if (event.reservedBucketBytes < event.requestedBucketBytes) { - msccBuilder.finalUnitIndication = FinalUnitIndication.newBuilder() - .setFinalUnitAction(FinalUnitAction.TERMINATE) - .setIsSet(true) + + event.request?.let { request -> + if (request.msccCount > 0) { + val msccBuilder = MultipleServiceCreditControl.newBuilder() + msccBuilder.setServiceIdentifier(request.getMscc(0).serviceIdentifier) + .setRatingGroup(request.getMscc(0).ratingGroup) + .setValidityTime(86400) + + if ((request.getMscc(0).reportingReason != ReportingReason.FINAL) && (request.getMscc(0).requested.totalOctets > 0)) { + msccBuilder.granted = ServiceUnit.newBuilder() + .setTotalOctets(event.reservedBucketBytes) + .build() + if (event.reservedBucketBytes < request.getMscc(0).requested.totalOctets) { + msccBuilder.finalUnitIndication = FinalUnitIndication.newBuilder() + .setFinalUnitAction(FinalUnitAction.TERMINATE) + .setIsSet(true) + .build() + } + } else { + // Use -1 to indicate no granted service unit should be included in the answer + msccBuilder.granted = ServiceUnit.newBuilder() + .setTotalOctets(-1) .build() } - } else { - // Use -1 to indicate no granted service unit should be included in the answer - msccBuilder.granted = ServiceUnit.newBuilder() - .setTotalOctets(-1) - .build() + creditControlAnswer.addMscc(msccBuilder.build()) } - creditControlAnswer.addMscc(msccBuilder.build()) + creditControlAnswer.setRequestId(request.requestId) } val streamId = event.ocsgwStreamId diff --git a/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsState.kt b/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsState.kt index b44a14084..6e03b6820 100644 --- a/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsState.kt +++ b/ocs/src/main/kotlin/org/ostelco/prime/ocs/OcsState.kt @@ -41,10 +41,10 @@ class OcsState(val loadSubscriberInfo:Boolean = true) : EventHandler { logger.error("Received null as msisdn") return } - consumeDataBytes(msisdn, event.usedBucketBytes) + consumeDataBytes(msisdn, event.request?.msccList?.firstOrNull()?.used?.totalOctets ?: 0L) event.reservedBucketBytes = reserveDataBytes( msisdn, - event.requestedBucketBytes) + event.request?.msccList?.firstOrNull()?.requested?.totalOctets ?: 0L) event.bundleId = msisdnToBundleIdMap[msisdn] event.bundleBytes = bundleBalanceMap[event.bundleId] ?: 0 } @@ -54,7 +54,7 @@ class OcsState(val loadSubscriberInfo:Boolean = true) : EventHandler { logger.error("Received null as bundleId") return } - event.bundleBytes = addDataBundleBytes(bundleId, event.requestedBucketBytes) + event.bundleBytes = addDataBundleBytes(bundleId, event.topUpBytes ?: 0L) event.msisdnToppedUp = bundleIdToMsisdnMap[bundleId]?.toList() } RELEASE_RESERVED_BUCKET -> { diff --git a/ocs/src/test/kotlin/org/ostelco/prime/disruptor/PrimeEventProducerTest.kt b/ocs/src/test/kotlin/org/ostelco/prime/disruptor/PrimeEventProducerTest.kt index e410bae73..63f04a32e 100644 --- a/ocs/src/test/kotlin/org/ostelco/prime/disruptor/PrimeEventProducerTest.kt +++ b/ocs/src/test/kotlin/org/ostelco/prime/disruptor/PrimeEventProducerTest.kt @@ -77,7 +77,7 @@ class PrimeEventProducerTest { // Verify some behavior assertEquals(BUNDLE_ID, event.bundleId) - assertEquals(NO_OF_TOPUP_BYTES, event.requestedBucketBytes) + assertEquals(NO_OF_TOPUP_BYTES, event.topUpBytes) assertEquals(TOPUP_DATA_BUNDLE_BALANCE, event.messageType) } @@ -89,8 +89,8 @@ class PrimeEventProducerTest { .setTotalOctets(REQUESTED_BYTES) .build()) .setUsed(ServiceUnit.newBuilder().setTotalOctets(USED_BYTES).build()) - .setRatingGroup(10) - .setServiceIdentifier(1) + .setRatingGroup(RATING_GROUP) + .setServiceIdentifier(SERVICE_IDENTIFIER) .build() ).build() @@ -98,10 +98,10 @@ class PrimeEventProducerTest { val event = collectedEvent assertEquals(MSISDN, event.msisdn) - assertEquals(REQUESTED_BYTES, event.requestedBucketBytes) - assertEquals(USED_BYTES, event.usedBucketBytes) - assertEquals(10, event.ratingGroup) - assertEquals(1, event.serviceIdentifier) + assertEquals(REQUESTED_BYTES, event.request?.msccList?.firstOrNull()?.requested?.totalOctets ?: 0L) + assertEquals(USED_BYTES, event.request?.msccList?.firstOrNull()?.used?.totalOctets ?: 0L) + assertEquals(RATING_GROUP, event.request?.msccList?.firstOrNull()?.ratingGroup) + assertEquals(SERVICE_IDENTIFIER, event.request?.msccList?.firstOrNull()?.serviceIdentifier) assertEquals(STREAM_ID, event.ocsgwStreamId) assertEquals(CREDIT_CONTROL_REQUEST, event.messageType) } @@ -123,6 +123,10 @@ class PrimeEventProducerTest { private const val RING_BUFFER_SIZE = 256 private const val TIMEOUT = 10 + + private const val RATING_GROUP = 10L; + + private const val SERVICE_IDENTIFIER = 1L; } } diff --git a/ocsgw/build.gradle b/ocsgw/build.gradle index 3a307fed5..0ce19f6f4 100644 --- a/ocsgw/build.gradle +++ b/ocsgw/build.gradle @@ -3,6 +3,8 @@ plugins { id "com.github.johnrengelman.shadow" version "2.0.4" } +ext.junit5Version = "5.3.0" + dependencies { implementation project(':ocs-grpc-api') implementation project(':analytics-grpc-api') @@ -13,14 +15,14 @@ dependencies { implementation 'ch.qos.logback:logback-classic:1.2.3' // log to gcp stack-driver - implementation 'com.google.cloud:google-cloud-logging-logback:0.59.0-alpha' + implementation 'com.google.cloud:google-cloud-logging-logback:0.61.0-alpha' testImplementation project(':diameter-test') - testImplementation 'org.junit.jupiter:junit-jupiter-api:5.2.0' - testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.2.0' + testImplementation "org.junit.jupiter:junit-jupiter-api:$junit5Version" + testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:$junit5Version" testImplementation 'junit:junit:4.12' - testRuntimeOnly 'org.junit.vintage:junit-vintage-engine:5.2.0' + testRuntimeOnly "org.junit.vintage:junit-vintage-engine:$junit5Version" } test { @@ -76,6 +78,12 @@ task packDev(type: Zip, dependsOn: 'shadowJar') { from ('script/') { into(project.name + '/script') } + from ('config/logback.dev.xml') { + into (project.name + '/config/') + rename { String fileName -> + fileName.replace('dev.', '') + } + } from ('config/dictionary.xml') { into (project.name + '/config/') } diff --git a/ocsgw/config/logback.dev.xml b/ocsgw/config/logback.dev.xml new file mode 100644 index 000000000..d268423e5 --- /dev/null +++ b/ocsgw/config/logback.dev.xml @@ -0,0 +1,36 @@ + + + + + + %d{dd MMM yyyy HH:mm:ss,SSS} %-5p %c{1} - %m%n + + + + + + INFO + + ocsgw + global + INFO + + + + + 1000 + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ocsgw/config/server-jdiameter-config.dev.xml b/ocsgw/config/server-jdiameter-config.dev.xml index af551d1ee..e197b71ac 100644 --- a/ocsgw/config/server-jdiameter-config.dev.xml +++ b/ocsgw/config/server-jdiameter-config.dev.xml @@ -5,7 +5,7 @@ - + diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java index 361021785..1b615aa1b 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/GrpcDataSource.java @@ -229,7 +229,7 @@ public void onNext(CreditControlAnswerInfo answer) { private void handleGrpcCcrAnswer(CreditControlAnswerInfo answer) { try { - LOG.info("[<<] Received data bucket for {}", answer.getMsisdn()); + LOG.info("[<<] CreditControlAnswer for {}", answer.getMsisdn()); final CreditControlContext ccrContext = ccrMap.remove(answer.getRequestId()); if (ccrContext != null) { final ServerCCASession session = OcsServer.getInstance().getStack().getSession(ccrContext.getSessionId(), ServerCCASession.class); @@ -329,7 +329,7 @@ private void updateBlockedList(CreditControlAnswerInfo answer, CreditControlRequ public void handleRequest(final CreditControlContext context) { ccrMap.put(context.getSessionId(), context); addToSessionMap(context); - LOG.info("[>>] Requesting bytes for {}", context.getCreditControlRequest().getMsisdn()); + LOG.info("[>>] creditControlRequest for {}", context.getCreditControlRequest().getMsisdn()); if (creditControlRequest != null) { try { diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java index 6facc6699..3b863e8dd 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/data/grpc/OcsgwMetrics.java @@ -24,7 +24,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; -public class OcsgwMetrics { +class OcsgwMetrics { private static final Logger LOG = LoggerFactory.getLogger(OcsgwMetrics.class); @@ -40,9 +40,11 @@ public class OcsgwMetrics { private ScheduledFuture initAnalyticsFuture = null; + private ScheduledFuture keepAliveFuture = null; + private int lastActiveSessions = 0; - public OcsgwMetrics(String metricsServerHostname, ServiceAccountJwtAccessCredentials credentials) { + OcsgwMetrics(String metricsServerHostname, ServiceAccountJwtAccessCredentials credentials) { try { final NettyChannelBuilder nettyChannelBuilder = NettyChannelBuilder @@ -82,6 +84,13 @@ public final void onCompleted() { } } + private void reconnectKeepAlive() { + LOG.info("reconnectKeepAlive called"); + if (keepAliveFuture != null) { + keepAliveFuture.cancel(true); + } + } + private void reconnectAnalyticsReport() { LOG.info("reconnectAnalyticsReport called"); @@ -91,6 +100,7 @@ private void reconnectAnalyticsReport() { LOG.info("Schedule new Callable initAnalyticsRequest"); initAnalyticsFuture = executorService.schedule((Callable) () -> { + reconnectKeepAlive(); LOG.info("Calling initAnalyticsRequest"); initAnalyticsRequest(); sendAnalytics(lastActiveSessions); @@ -100,7 +110,7 @@ private void reconnectAnalyticsReport() { TimeUnit.SECONDS); } - public void initAnalyticsRequest() { + void initAnalyticsRequest() { ocsgwAnalyticsReport = ocsgwAnalyticsServiceStub.ocsgwAnalyticsEvent( new AnalyticsRequestObserver() { @@ -110,11 +120,21 @@ public void onNext(OcsgwAnalyticsReply value) { } } ); + initKeepAlive(); + } + + private void initKeepAlive() { + // this is used to keep connection alive + keepAliveFuture = executorService.scheduleWithFixedDelay(() -> { + sendAnalytics(lastActiveSessions); + }, + 15, + 50, + TimeUnit.SECONDS); } - public void sendAnalytics(int size) { + void sendAnalytics(int size) { ocsgwAnalyticsReport.onNext(OcsgwAnalyticsReport.newBuilder().setActiveSessions(size).build()); lastActiveSessions = size; } - } \ No newline at end of file diff --git a/ostelco-lib/build.gradle b/ostelco-lib/build.gradle index a320261f4..3c54eb8da 100644 --- a/ostelco-lib/build.gradle +++ b/ostelco-lib/build.gradle @@ -9,11 +9,11 @@ dependencies { // Match netty via ocs-api implementation 'com.google.firebase:firebase-admin:6.4.0' implementation 'com.lmax:disruptor:3.4.2' - implementation 'com.google.guava:guava:25.1-jre' + implementation "com.google.guava:guava:$guavaVersion" testImplementation "io.dropwizard:dropwizard-testing:$dropwizardVersion" testImplementation "org.mockito:mockito-core:2.18.3" - testImplementation 'org.assertj:assertj-core:3.10.0' + testImplementation "org.assertj:assertj-core:$assertJVersion" // https://mvnrepository.com/artifact/org.glassfish.jersey.test-framework.providers/jersey-test-framework-provider-grizzly2 testCompile("org.glassfish.jersey.test-framework.providers:jersey-test-framework-provider-grizzly2:2.27") { diff --git a/payment-processor/build.gradle b/payment-processor/build.gradle index 727e5e3e0..5d018028a 100644 --- a/payment-processor/build.gradle +++ b/payment-processor/build.gradle @@ -21,7 +21,7 @@ dependencies { implementation project(":prime-api") - implementation "com.stripe:stripe-java:6.8.0" + implementation "com.stripe:stripe-java:$stripeVersion" testImplementation "org.jetbrains.kotlin:kotlin-test:$kotlinVersion" testImplementation "org.jetbrains.kotlin:kotlin-test-junit:$kotlinVersion" diff --git a/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt b/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt index 074efa20b..cea785c71 100644 --- a/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt +++ b/payment-processor/src/integration-tests/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt @@ -90,6 +90,22 @@ class StripePaymentProcessorTest { assertEquals(true, resultRemoveDefault.isRight()) } + @Test + fun createAuthorizeChargeAndRefund() { + val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + assertEquals(true, resultAddSource.isRight()) + + val resultAuthorizeCharge = paymentProcessor.authorizeCharge(stripeCustomerId, resultAddSource.fold({ "" }, { it.id }), 1000, "nok") + assertEquals(true, resultAuthorizeCharge.isRight()) + + val resultRefundCharge = paymentProcessor.refundCharge(resultAuthorizeCharge.fold({ "" }, { it } )) + assertEquals(true, resultRefundCharge.isRight()) + assertEquals(resultAuthorizeCharge.fold({ "" }, { it } ), resultRefundCharge.fold({ "" }, { it } )) + + val resultRemoveSource = paymentProcessor.removeSource(stripeCustomerId, resultAddSource.fold({ "" }, { it.id })) + assertEquals(true, resultRemoveSource.isRight()) + } + @Test fun createAndRemoveProduct() { val resultCreateProduct = paymentProcessor.createProduct("TestSku") diff --git a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt index f8afb287b..ca34b6aef 100644 --- a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt +++ b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt @@ -2,129 +2,144 @@ package org.ostelco.prime.paymentprocessor import arrow.core.Either import arrow.core.flatMap -import com.stripe.model.Charge -import com.stripe.model.Customer -import com.stripe.model.Plan -import com.stripe.model.Product -import com.stripe.model.Subscription -import org.ostelco.prime.core.ApiError -import org.ostelco.prime.core.BadGatewayError -import org.ostelco.prime.core.ForbiddenError -import org.ostelco.prime.core.NotFoundError import org.ostelco.prime.logger +import com.stripe.model.* import org.ostelco.prime.paymentprocessor.core.* + class StripePaymentProcessor : PaymentProcessor { - private val LOG by logger() + private val logger by logger() - override fun getSavedSources(customerId: String): Either> = - either (NotFoundError("Failed to get sources for customer ${customerId}")) { + override fun getSavedSources(customerId: String): Either> = + either(NotFoundError("Failed to retrieve sources for customer $customerId")) { val sources = mutableListOf() val customer = Customer.retrieve(customerId) customer.sources.data.forEach { - sources.add(SourceInfo(it.id)) + sources.add(SourceInfo(it.id, getAccountDetails(it))) } sources } - override fun createPaymentProfile(userEmail: String): Either = - either(ForbiddenError("Failed to create profile for user ${userEmail}")) { - val customerParams = HashMap() - customerParams.put("email", userEmail) + /* Returns detailed 'account details' for the given Stripe source/account. + Note that the fields 'id' and 'accountType' are manadatory. */ + private fun getAccountDetails(accountInfo: ExternalAccount) : Map { + when (accountInfo) { + is Card -> { + return mapOf("id" to accountInfo.id, + "accountType" to "card", + "addressLine1" to accountInfo.addressLine1, + "addressLine2" to accountInfo.addressLine2, + "zip" to accountInfo.addressZip, + "city" to accountInfo.addressCity, + "state" to accountInfo.addressState, + "country" to accountInfo.country, + "currency" to accountInfo.currency, + "brand" to accountInfo.brand, // "Visa", "Mastercard" etc. + "last4" to accountInfo.last4, + "expireMonth" to accountInfo.expMonth, + "expireYear" to accountInfo.expYear, + "funding" to accountInfo.funding) // Typ.: "credit" or "debit" + .filterValues { it != null } // Unfortunately the 'swagger' def. will removed fields back again. + } + // To add support for other Stripe source/account types, see + // https://stripe.com/docs/api/java#sources + else -> { + logger.error("Received unsupported Stripe source/account type: {}", accountInfo) + return mapOf("id" to accountInfo.id, + "accountType" to "unsupported") + } + } + } + + override fun createPaymentProfile(userEmail: String): Either = + either(ForbiddenError("Failed to create profile for user $userEmail")) { + val customerParams = mapOf("email" to userEmail) ProfileInfo(Customer.create(customerParams).id) } - override fun createPlan(productId: String, amount: Int, currency: String, interval: PaymentProcessor.Interval): Either = - either(ForbiddenError("Failed to create plan with product id ${productId} amount ${amount} currency ${currency} interval ${interval.value}")) { - val planParams = HashMap() - planParams["amount"] = amount - planParams["interval"] = interval.value - planParams["product"] = productId - planParams["currency"] = currency + override fun createPlan(productId: String, amount: Int, currency: String, interval: PaymentProcessor.Interval): Either = + either(ForbiddenError("Failed to create plan with product id $productId amount $amount currency $currency interval ${interval.value}")) { + val planParams = mapOf( + "amount" to amount, + "interval" to interval.value, + "product" to productId, + "currency" to currency) PlanInfo(Plan.create(planParams).id) } - override fun removePlan(planId: String): Either = - either(NotFoundError("Failed to delete plan ${planId}")) { + override fun removePlan(planId: String): Either = + either(NotFoundError("Failed to delete plan $planId")) { val plan = Plan.retrieve(planId) PlanInfo(plan.delete().id) } - override fun createProduct(sku: String): Either = - either(ForbiddenError("Failed to create product with sku ${sku}")) { - val productParams = HashMap() - productParams["name"] = sku - productParams["type"] = "service" + override fun createProduct(sku: String): Either = + either(ForbiddenError("Failed to create product with sku $sku")) { + val productParams = mapOf( + "name" to sku, + "type" to "service") ProductInfo(Product.create(productParams).id) } - override fun removeProduct(productId: String): Either = - either(NotFoundError("Failed to delete product ${productId}")) { + override fun removeProduct(productId: String): Either = + either(NotFoundError("Failed to delete product $productId")) { val product = Product.retrieve(productId) ProductInfo(product.delete().id) } - override fun addSource(customerId: String, sourceId: String): Either = - either(ForbiddenError("Failed to add source ${sourceId} to customer ${customerId}")) { + override fun addSource(customerId: String, sourceId: String): Either = + either(NotFoundError("Failed to add source $sourceId to customer $customerId")) { val customer = Customer.retrieve(customerId) - val params = HashMap() - params["source"] = sourceId + val params = mapOf("source" to sourceId) SourceInfo(customer.sources.create(params).id) } - override fun setDefaultSource(customerId: String, sourceId: String): Either = - either(ForbiddenError("Failed to set default source ${sourceId} for customer ${customerId}")) { + override fun setDefaultSource(customerId: String, sourceId: String): Either = + either(NotFoundError("Failed to set default source $sourceId for customer $customerId")) { val customer = Customer.retrieve(customerId) - val updateParams = HashMap() - updateParams.put("default_source", sourceId) + val updateParams = mapOf("default_source" to sourceId) val customerUpdated = customer.update(updateParams) SourceInfo(customerUpdated.defaultSource) } - override fun getDefaultSource(customerId: String): Either = - either(NotFoundError( "Failed to get default source for customer ${customerId}")) { + override fun getDefaultSource(customerId: String): Either = + either(NotFoundError("Failed to get default source for customer $customerId")) { SourceInfo(Customer.retrieve(customerId).defaultSource) } - override fun deletePaymentProfile(customerId: String): Either = - either(NotFoundError("Failed to delete customer ${customerId}")) { + override fun deletePaymentProfile(customerId: String): Either = + either(NotFoundError("Failed to delete customer $customerId")) { val customer = Customer.retrieve(customerId) ProfileInfo(customer.delete().id) } - override fun subscribeToPlan(planId: String, customerId: String): Either = - either(ForbiddenError("Failed to subscribe customer ${customerId} to plan ${planId}")) { - val item = HashMap() - item["plan"] = planId - - val items = HashMap() - items["0"] = item - - val params = HashMap() - params["customer"] = customerId - params["items"] = items + override fun subscribeToPlan(planId: String, customerId: String): Either = + either(ForbiddenError("Failed to subscribe customer $customerId to plan $planId")) { + val item = mapOf("plan" to planId) + val params = mapOf( + "customer" to customerId, + "items" to mapOf("0" to item)) SubscriptionInfo(Subscription.create(params).id) } - override fun cancelSubscription(subscriptionId: String, atIntervalEnd: Boolean): Either = - either(ForbiddenError("Failed to unsubscribe subscription Id : ${subscriptionId} atIntervalEnd ${atIntervalEnd}")) { + override fun cancelSubscription(subscriptionId: String, atIntervalEnd: Boolean): Either = + either(NotFoundError("Failed to unsubscribe subscription Id : $subscriptionId atIntervalEnd $atIntervalEnd")) { val subscription = Subscription.retrieve(subscriptionId) - val subscriptionParams = HashMap() - subscriptionParams["at_period_end"] = atIntervalEnd + val subscriptionParams = mapOf("at_period_end" to atIntervalEnd) SubscriptionInfo(subscription.cancel(subscriptionParams).id) } - override fun authorizeCharge(customerId: String, sourceId: String?, amount: Int, currency: String): Either { + override fun authorizeCharge(customerId: String, sourceId: String?, amount: Int, currency: String): Either { val errorMessage = "Failed to authorize the charge for customerId $customerId sourceId $sourceId amount $amount currency $currency" return either(ForbiddenError(errorMessage)) { - val chargeParams = HashMap() - chargeParams["amount"] = amount - chargeParams["currency"] = currency - chargeParams["customer"] = customerId - chargeParams["capture"] = false + val chargeParams = mutableMapOf( + "amount" to amount, + "currency" to currency, + "customer" to customerId, + "capture" to false) if (sourceId != null) { chargeParams["source"] = sourceId } @@ -139,9 +154,9 @@ class StripePaymentProcessor : PaymentProcessor { } } - override fun captureCharge(chargeId: String, customerId: String, sourceId: String?): Either { + override fun captureCharge(chargeId: String, customerId: String): Either { val errorMessage = "Failed to capture charge for customerId $customerId chargeId $chargeId" - return either(ForbiddenError(errorMessage)) { + return either(NotFoundError(errorMessage)) { Charge.retrieve(chargeId) }.flatMap { charge: Charge -> val review = charge.review @@ -155,23 +170,31 @@ class StripePaymentProcessor : PaymentProcessor { charge.capture() Either.right(charge.id) } catch (e: Exception) { - LOG.warn(errorMessage, e) + logger.warn(errorMessage, e) Either.left(BadGatewayError(errorMessage)) } } } - override fun removeSource(customerId: String, sourceId: String): Either = - either(ForbiddenError("Failed to remove source ${sourceId} from customer ${customerId}")) { + override fun refundCharge(chargeId: String): Either = + either(NotFoundError("Failed to refund charge $chargeId")) { + val refundParams = mapOf("charge" to chargeId) + Refund.create(refundParams).charge + } + + override fun removeSource(customerId: String, sourceId: String): Either = + either(ForbiddenError("Failed to remove source $sourceId from customer $customerId")) { Customer.retrieve(customerId).sources.retrieve(sourceId).delete().id } - private fun either(apiError: ApiError, action: () -> RETURN): Either { + private fun either(paymentError: PaymentError, action: () -> RETURN): Either { return try { Either.right(action()) } catch (e: Exception) { - LOG.warn(apiError.description, e) - Either.left(apiError) + paymentError.externalErrorMessage = e.message + logger.warn(paymentError.description, e) + Either.left(paymentError) } } } + diff --git a/prime-api/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt b/prime-api/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt index 466f9f0be..ddfcac34a 100644 --- a/prime-api/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt +++ b/prime-api/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsService.kt @@ -2,10 +2,12 @@ package org.ostelco.prime.analytics import org.ostelco.prime.analytics.MetricType.COUNTER import org.ostelco.prime.analytics.MetricType.GAUGE +import org.ostelco.prime.model.PurchaseRecord interface AnalyticsService { fun reportTrafficInfo(msisdn: String, usedBytes: Long, bundleBytes: Long) fun reportMetric(primeMetric: PrimeMetric, value: Long) + fun reportPurchaseInfo(purchaseRecord: PurchaseRecord, subscriberId: String, status: String) } enum class PrimeMetric(val metricType: MetricType) { diff --git a/prime-api/src/main/kotlin/org/ostelco/prime/paymentprocessor/PaymentProcessor.kt b/prime-api/src/main/kotlin/org/ostelco/prime/paymentprocessor/PaymentProcessor.kt index d68c9ecf0..78d27867c 100644 --- a/prime-api/src/main/kotlin/org/ostelco/prime/paymentprocessor/PaymentProcessor.kt +++ b/prime-api/src/main/kotlin/org/ostelco/prime/paymentprocessor/PaymentProcessor.kt @@ -18,19 +18,19 @@ interface PaymentProcessor { * @param sourceId Stripe source id * @return Stripe sourceId if created */ - fun addSource(customerId: String, sourceId: String): Either + fun addSource(customerId: String, sourceId: String): Either /** * @param userEmail: user email (Prime unique identifier for customer) * @return Stripe customerId if created */ - fun createPaymentProfile(userEmail: String): Either + fun createPaymentProfile(userEmail: String): Either /** * @param customerId Stripe customer id * @return Stripe customerId if deleted */ - fun deletePaymentProfile(customerId: String): Either + fun deletePaymentProfile(customerId: String): Either /** * @param productId Stripe product id @@ -39,58 +39,58 @@ interface PaymentProcessor { * @param interval The frequency with which a subscription should be billed. * @return Stripe planId if created */ - fun createPlan(productId: String, amount: Int, currency: String, interval: Interval): Either + fun createPlan(productId: String, amount: Int, currency: String, interval: Interval): Either /** * @param Stripe Plan Id * @param Stripe Customer Id * @return Stripe SubscriptionId if subscribed */ - fun subscribeToPlan(planId: String, customerId: String): Either + fun subscribeToPlan(planId: String, customerId: String): Either /** * @param Stripe Plan Id * @return Stripe PlanId if deleted */ - fun removePlan(planId: String): Either + fun removePlan(planId: String): Either /** * @param Stripe Subscription Id * @param Stripe atIntervalEnd set to true if the subscription shall remain active until the end of the Plan interval * @return Stripe SubscriptionId if unsubscribed */ - fun cancelSubscription(subscriptionId: String, atIntervalEnd: Boolean = true): Either + fun cancelSubscription(subscriptionId: String, atIntervalEnd: Boolean = true): Either /** * @param sku Prime product SKU * @return Stripe productId if created */ - fun createProduct(sku: String): Either + fun createProduct(sku: String): Either /** * @param productId Stripe product Id * @return Stripe productId if removed */ - fun removeProduct(productId: String): Either + fun removeProduct(productId: String): Either /** * @param customerId Stripe customer id * @return List of Stripe sourceId */ - fun getSavedSources(customerId: String): Either> + fun getSavedSources(customerId: String): Either> /** * @param customerId Stripe customer id * @return Stripe default sourceId */ - fun getDefaultSource(customerId: String): Either + fun getDefaultSource(customerId: String): Either /** * @param customerId Stripe customer id * @param sourceId Stripe source id * @return SourceInfo if created */ - fun setDefaultSource(customerId: String, sourceId: String): Either + fun setDefaultSource(customerId: String, sourceId: String): Either /** * @param customerId Customer id in the payment system @@ -99,21 +99,26 @@ interface PaymentProcessor { * @param currency Three-letter ISO currency code in lowercase * @return id of the charge if authorization was successful */ - fun authorizeCharge(customerId: String, sourceId: String?, amount: Int, currency: String): Either + fun authorizeCharge(customerId: String, sourceId: String?, amount: Int, currency: String): Either /** * @param chargeId ID of the of the authorized charge from authorizeCharge() * @param customerId Customer id in the payment system - * @param sourceId id of the payment source * @return id of the charge if authorization was successful */ - fun captureCharge(chargeId: String, customerId: String, sourceId: String?): Either + fun captureCharge(chargeId: String, customerId: String): Either + + /** + * @param chargeId ID of the of the authorized charge to refund from authorizeCharge() + * @return id of the charge + */ + fun refundCharge(chargeId: String): Either /** * @param customerId Customer id in the payment system * @param sourceId id of the payment source * @return id if removed */ - fun removeSource(customerId: String, sourceId: String): Either + fun removeSource(customerId: String, sourceId: String): Either } \ No newline at end of file diff --git a/prime-api/src/main/kotlin/org/ostelco/prime/paymentprocessor/core/Model.kt b/prime-api/src/main/kotlin/org/ostelco/prime/paymentprocessor/core/Model.kt index e890b127f..c87fc14cf 100644 --- a/prime-api/src/main/kotlin/org/ostelco/prime/paymentprocessor/core/Model.kt +++ b/prime-api/src/main/kotlin/org/ostelco/prime/paymentprocessor/core/Model.kt @@ -6,6 +6,6 @@ class ProductInfo(val id: String) class ProfileInfo(val id: String) -class SourceInfo(val id: String) +class SourceInfo(val id: String, val details: Map? = null) class SubscriptionInfo(val id: String) diff --git a/prime-api/src/main/kotlin/org/ostelco/prime/paymentprocessor/core/PaymentError.kt b/prime-api/src/main/kotlin/org/ostelco/prime/paymentprocessor/core/PaymentError.kt new file mode 100644 index 000000000..42b2f3945 --- /dev/null +++ b/prime-api/src/main/kotlin/org/ostelco/prime/paymentprocessor/core/PaymentError.kt @@ -0,0 +1,20 @@ +package org.ostelco.prime.paymentprocessor.core + +import javax.ws.rs.core.Response + +sealed class PaymentError(val description: String) { + open var status : Int = 0 + var externalErrorMessage : String? = null +} + +class ForbiddenError(description: String) : PaymentError(description) { + override var status : Int = Response.Status.FORBIDDEN.statusCode +} + +class NotFoundError(description: String) : PaymentError(description) { + override var status : Int = Response.Status.NOT_FOUND.statusCode +} + +class BadGatewayError(description: String) : PaymentError(description) { + override var status : Int = Response.Status.BAD_REQUEST.statusCode +} \ No newline at end of file diff --git a/prime-api/src/main/kotlin/org/ostelco/prime/pseudonymizer/PseudonymizerService.kt b/prime-api/src/main/kotlin/org/ostelco/prime/pseudonymizer/PseudonymizerService.kt index f48b214c7..f89f0f327 100644 --- a/prime-api/src/main/kotlin/org/ostelco/prime/pseudonymizer/PseudonymizerService.kt +++ b/prime-api/src/main/kotlin/org/ostelco/prime/pseudonymizer/PseudonymizerService.kt @@ -7,5 +7,8 @@ interface PseudonymizerService { fun getActivePseudonymsForMsisdn(msisdn: String): ActivePseudonyms - fun getPseudonymEntityFor(msisdn: String, timestamp: Long): PseudonymEntity + fun getMsisdnPseudonym(msisdn: String, timestamp: Long): PseudonymEntity + + fun getSubscriberIdPseudonym(subscriberId: String, timestamp: Long): PseudonymEntity + } \ No newline at end of file diff --git a/prime-api/src/main/kotlin/org/ostelco/prime/storage/Variants.kt b/prime-api/src/main/kotlin/org/ostelco/prime/storage/Variants.kt index a5fd68d89..c4b0d491d 100644 --- a/prime-api/src/main/kotlin/org/ostelco/prime/storage/Variants.kt +++ b/prime-api/src/main/kotlin/org/ostelco/prime/storage/Variants.kt @@ -10,6 +10,8 @@ import org.ostelco.prime.model.PurchaseRecord import org.ostelco.prime.model.Segment import org.ostelco.prime.model.Subscriber import org.ostelco.prime.model.Subscription +import org.ostelco.prime.paymentprocessor.core.PaymentError +import org.ostelco.prime.paymentprocessor.core.ProductInfo interface ClientDocumentStore { @@ -118,6 +120,11 @@ interface ClientGraphStore { * Get user who has referred this user. */ fun getReferredBy(subscriberId: String): Either + + /** + * Temporary method to perform purchase as atomic transaction + */ + fun purchaseProduct(subscriberId: String, sku: String, sourceId: String?, saveCard: Boolean): Either } interface AdminGraphStore { diff --git a/prime-client-api/build.gradle b/prime-client-api/build.gradle index 9bbc9c03d..d78bb4bd1 100644 --- a/prime-client-api/build.gradle +++ b/prime-client-api/build.gradle @@ -7,22 +7,12 @@ plugins { // gradle generateSwaggerCode swaggerSources { - - // if they ever fix kotlin -// 'kotlin-client' { -// inputFile = file("${projectDir}/../prime/infra/prod/prime-client-api.yaml") -// code { -// language = 'kotlin' -// configFile = file("${projectDir}/config.json") -// } -// } 'java-client' { inputFile = file("${projectDir}/../prime/infra/dev/prime-client-api.yaml") code { language = 'java' configFile = file("${projectDir}/config.json") - // for creating only model - // components = ["models"] +// components = ["models"] } } } @@ -31,22 +21,39 @@ compileJava.dependsOn swaggerSources.'java-client'.code sourceSets.main.java.srcDir "${swaggerSources.'java-client'.code.outputDir}/src/main/java" sourceSets.main.resources.srcDir "${swaggerSources.'java-client'.code.outputDir}/src/main/resources" +// if they ever fix kotlin +//swaggerSources { +// 'kotlin-client' { +// inputFile = file("${projectDir}/../prime/infra/prod/prime-client-api.yaml") +// code { +// language = 'kotlin' +// configFile = file("${projectDir}/config.json") +// } +// } +//} +// +//compileKotlin.dependsOn swaggerSources.'kotlin-client'.code +//sourceSets.main.kotlin.srcDir "${swaggerSources.'kotlin-client'.code.outputDir}/src/main/java" +//sourceSets.main.resources.srcDir "${swaggerSources.'kotlin-client'.code.outputDir}/src/main/resources" + dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion" swaggerCodegen 'io.swagger:swagger-codegen-cli:2.3.1' // taken from build/swagger-code-java-client/build.gradle - // for model implementation 'io.swagger:swagger-annotations:1.5.15' implementation 'com.google.code.gson:gson:2.8.5' - - // for all implementation 'com.squareup.okhttp:okhttp:2.7.5' implementation 'com.squareup.okhttp:logging-interceptor:2.7.5' implementation 'io.gsonfire:gson-fire:1.8.3' implementation 'org.threeten:threetenbp:1.3.7' testImplementation 'junit:junit:4.12' + + // taken from build/swagger-code-kotlin-client/build.gradle +// implementation "com.squareup.okhttp3:okhttp:3.8.0" +// implementation "com.squareup.moshi:moshi-kotlin:1.5.0" +// implementation "com.squareup.moshi:moshi-adapters:1.5.0" } idea { diff --git a/prime/build.gradle b/prime/build.gradle index c295eb5a3..d7f591f40 100644 --- a/prime/build.gradle +++ b/prime/build.gradle @@ -18,7 +18,7 @@ sourceSets { } } -version = "1.13.0" +version = "1.14.0" repositories { maven { @@ -43,8 +43,8 @@ dependencies { implementation "com.fasterxml.jackson.module:jackson-module-kotlin:$jacksonVersion" implementation "io.dropwizard:dropwizard-http2:$dropwizardVersion" - implementation "io.dropwizard:dropwizard-json-logging:$dropwizardVersion" - implementation 'com.google.guava:guava:25.1-jre' + runtimeOnly "io.dropwizard:dropwizard-json-logging:$dropwizardVersion" + implementation "com.google.guava:guava:$guavaVersion" implementation 'org.dhatim:dropwizard-prometheus:2.2.0' testImplementation "io.dropwizard:dropwizard-testing:$dropwizardVersion" diff --git a/prime/config/config.yaml b/prime/config/config.yaml index 1f4bf8d61..e12a6f1af 100644 --- a/prime/config/config.yaml +++ b/prime/config/config.yaml @@ -11,7 +11,8 @@ modules: - type: analytics config: projectId: pantel-2decb - topicId: data-traffic + dataTrafficTopicId: data-traffic + purchaseInfoTopicId: purchase-info - type: ocs config: lowBalanceThreshold: 100000000 @@ -34,13 +35,22 @@ server: port: 8080 maxConcurrentStreams: 1024 initialStreamRecvWindow: 65535 + requestLog: + appenders: + - type: console + layout: + type: json + customFieldNames: + level: severity logging: level: INFO loggers: org.ostelco: DEBUG + org.dhatim.dropwizard.prometheus.DropwizardMetricsExporter: ERROR appenders: - type: console layout: type: json - + customFieldNames: + level: severity \ No newline at end of file diff --git a/prime/config/test.yaml b/prime/config/test.yaml index 11def9a99..b67aa3372 100644 --- a/prime/config/test.yaml +++ b/prime/config/test.yaml @@ -13,7 +13,8 @@ modules: - type: analytics config: projectId: pantel-2decb - topicId: data-traffic + dataTrafficTopicId: data-traffic + purchaseInfoTopicId: purchase-info - type: ocs config: lowBalanceThreshold: 0 @@ -44,3 +45,5 @@ logging: level: INFO loggers: org.ostelco: DEBUG + # suppress exception logged while connecting to real bigQuery 3 times before connecting to emulator + com.google.auth.oauth2.ComputeEngineCredentials: ERROR \ No newline at end of file diff --git a/prime/infra/MONITORING.md b/prime/infra/MONITORING.md new file mode 100644 index 000000000..97e54c81b --- /dev/null +++ b/prime/infra/MONITORING.md @@ -0,0 +1,103 @@ +# Prometheus + +## Setup + +Anything that wants to be scraped by prometheus requires at least one annotation, see the [promotheus section](#prometheus) + +## Access + +### Grafana + +Through external ip: + +```bash +kubectl get services --namespace=monitoring | grep grafana | awk '{print $4}' +``` + +### Prometheus + +Through port forwarding: + +```bash +kubectl port-forward --namespace=monitoring $(kubectl get pods --namespace=monitoring | grep prometheus-core | awk '{print $1}') 9090 +``` + +## Discovery + +Prometheus is configured to do service discovery + +## Scrape + +Prometheus is configured to scrape metrics over https, thus the `ConfigMap` for `prometheus-core` requires path to ca_file from kubernetes secrets + +## [Pushgateway](https://github.com/prometheus/pushgateway) + +Used jobs that might not live long enough to be scraped by prometheus. + +Example usage: +```bash +# Push a metric to pushgateway:8080 (specified in the service declaration for pushgateway) +kubectl run curl-it --image=radial/busyboxplus:curl -i --tty --rm +echo "some_metric 4.71" | curl -v --data-binary @- http://pushgateway:8080/metrics/job/some_job +``` + +## [Monitoring.yaml](dev/monitoring.yaml) + +Is completely based on manifests-all.yaml from this (github repo)[https://github.com/giantswarm/kubernetes-prometheus] + +### Namespace +Defines monitoring namespace + +### [Alert Manager](https://prometheus.io/docs/alerting/alertmanager/) + +__`TODO: Add email config and / or slack api url for alerts to work`__ + +Defines alerts that can be sent by email or to slack. + +Contains two config maps, one defining the alert template and another to configure the alertmanager itself. +There is also a kubernetes Deployment and Service configuration. + +### [Grafana](https://grafana.com/) + +__`TODO: Discuss how grafana should be exposed. With LoadBalancer and / or through existing ingress using auth0 to authenticate users before they can access the dashboard.`__ +__`TODO: Figure out a better way / automatic way to backup dashboards and automatically import them on redeploy`__ + +Contains a Deployment and Service configuration and a ConfigMap with predefined dashboards. + +Grafana is exposed using a LoadBalancer. + +### [](#prometheus)[Prometheus](https://prometheus.io/) + +__`TODO: scraping happens over https thus requires a ca_file, figure out if this is automatically handled or if we need to add a ca file to kubernetes secrets`__ + +Contains a Deployment configuration and a ConfigMap. The ConfigMap defines how and what prometheus scrapes etc. + +There are more configurations at the end of the file, containing ConfigMap for prometheus rules, ClusterRoleBinding, ClusterRole, ServiceAccount and finally the prometheus Service itself. + +The following roles are being scraped: +nodes, endpoints, services, pods + +Given that they set the required annotations which tells prometheus that they should be scaped, see below: + +`add the annotations to the pods, that means the config with type Deployment, StatefulSet, DaemonSet` +```yaml +metadata: + annotations: + prometheus.io/scrape: 'true' # REQUIRED: has to be set for prometheus to scrape + prometheus.io/port: '9102' # OPTIONAL: defaults to '9102' + prometheus.io/path: '' # OPTIONAL: defaults to '/metrics' + prometheus.io/scheme: '' # OPTIONAL: http or https defaults to 'https' +``` + +### [Kube State Metrics](https://github.com/kubernetes/kube-state-metrics) + +See the above link for documentation. + +### Extra Prometheus Metrics + +Some DaemonSets that define different prometheus metrics, not sure if this is a general config or if its connected to any of the other configurations. + + + + + diff --git a/prime/infra/README.md b/prime/infra/README.md index 11d55723c..6c8208b21 100644 --- a/prime/infra/README.md +++ b/prime/infra/README.md @@ -223,6 +223,11 @@ kubectl create secret generic metrics-ostelco-ssl \ --from-file=certs/dev.ostelco.org/nginx.crt \ --from-file=certs/dev.ostelco.org/nginx.key ``` +### Cloud Pub/Sub + +```bash +gcloud pubsub topics create purchase-info +``` ### Endpoints @@ -263,6 +268,39 @@ gcloud endpoints services deploy prime/infra/dev/prime-client-api.yaml ## Deploy to Dev cluster +### Deploy monitoring + +```bash +kubectl apply -f prime/infra/dev/monitoring.yaml + +# If the above command fails on creating clusterroles / clusterbindings you need to add a role to the user you are using to deploy +# You can read more about it here https://github.com/coreos/prometheus-operator/issues/357 +kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user $(gcloud config get-value account) + +# +kubectl apply -f prime/infra/dev/monitoring-pushgateway.yaml +``` + +#### Prometheus dashboard +```bash +kubectl port-forward --namespace=monitoring $(kubectl get pods --namespace=monitoring | grep prometheus-core | awk '{print $1}') 9090 +``` + +#### Grafana dashboard +__`Has own its own load balancer and can be accessed directly. Discuss if this is OK or find and implement a different way of accessing the grafana dashboard.`__ + +Can be accessed directly from external ip +```bash +kubectl get services --namespace=monitoring | grep grafana | awk '{print $4}' +``` + +#### Push gateway +```bash +# Push a metric to pushgateway:8080 (specified in the service declaration for pushgateway) +kubectl run curl-it --image=radial/busyboxplus:curl -i --tty --rm +echo "some_metric 4.71" | curl -v --data-binary @- http://pushgateway:8080/metrics/job/some_job +``` + ### Setup Neo4j ```bash @@ -303,4 +341,4 @@ logName="projects/pantel-2decb/logs/prime" ## Connect using Neo4j Browser -Check [docs/NEO4J.md](../docs/NEO4J.md) \ No newline at end of file +Check [docs/NEO4J.md](../docs/NEO4J.md) diff --git a/prime/infra/dev/monitoring-pushgateway.yaml b/prime/infra/dev/monitoring-pushgateway.yaml new file mode 100644 index 000000000..4448c85a2 --- /dev/null +++ b/prime/infra/dev/monitoring-pushgateway.yaml @@ -0,0 +1,30 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: pushgateway-deployment +spec: + replicas: 1 + template: + metadata: + labels: + app: pushgateway-server + annotations: + prometheus.io/scrape: 'true' + spec: + containers: + - name: pushgateway + image: prom/pushgateway:v0.5.2 + ports: + - containerPort: 9091 +--- +apiVersion: v1 +kind: Service +metadata: + name: pushgateway +spec: + selector: + app: pushgateway-server + type: NodePort + ports: + - port: 8080 + targetPort: 9091 diff --git a/prime/infra/dev/monitoring.yaml b/prime/infra/dev/monitoring.yaml new file mode 100644 index 000000000..12376307d --- /dev/null +++ b/prime/infra/dev/monitoring.yaml @@ -0,0 +1,2517 @@ +# Based on https://github.com/giantswarm/kubernetes-prometheus +# According to Praqma this is an outdated way to setup prometheus stack +--- +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: grafana-core + namespace: monitoring + labels: + app: grafana + component: core +spec: + replicas: 1 + template: + metadata: + labels: + app: grafana + component: core + spec: + containers: + - image: grafana/grafana:4.2.0 + name: grafana-core + imagePullPolicy: IfNotPresent + ports: + - name: http-server + containerPort: 3000 + # env: + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 100m + memory: 100Mi + env: + # The following env variables set up basic auth twith the default admin user and admin password. + - name: GF_AUTH_BASIC_ENABLED + value: "true" + - name: GF_AUTH_ANONYMOUS_ENABLED + value: "false" + # - name: GF_AUTH_ANONYMOUS_ORG_ROLE + # value: Admin + # does not really work, because of template variables in exported dashboards: + # - name: GF_DASHBOARDS_JSON_ENABLED + # value: "true" + readinessProbe: + httpGet: + path: /login + port: 3000 + # initialDelaySeconds: 30 + # timeoutSeconds: 1 + volumeMounts: + - name: grafana-persistent-storage + mountPath: /var + volumes: + - name: grafana-persistent-storage + emptyDir: {} +--- +apiVersion: v1 +data: + grafana-net-2-dashboard.json: | + { + "__inputs": [{ + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + }], + "__requires": [{ + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + }, { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" + }, { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "3.1.0" + }, { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }], + "id": null, + "title": "Prometheus Stats", + "tags": [], + "style": "dark", + "timezone": "browser", + "editable": true, + "hideControls": true, + "sharedCrosshair": false, + "rows": [{ + "collapse": false, + "editable": true, + "height": 178, + "panels": [{ + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": ["rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)"], + "datasource": "${DS_PROMETHEUS}", + "decimals": 1, + "editable": true, + "error": false, + "format": "s", + "id": 5, + "interval": null, + "links": [], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [{ + "expr": "(time() - process_start_time_seconds{job=\"prometheus\"})", + "intervalFactor": 2, + "refId": "A", + "step": 4 + }], + "thresholds": "", + "title": "Uptime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [{ + "op": "=", + "text": "N/A", + "value": "null" + }], + "valueName": "current", + "mappingTypes": [{ + "name": "value to text", + "value": 1 + }, { + "name": "range to text", + "value": 2 + }], + "rangeMaps": [{ + "from": "null", + "to": "null", + "text": "N/A" + }], + "mappingType": 1, + "gauge": { + "show": false, + "minValue": 0, + "maxValue": 100, + "thresholdMarkers": true, + "thresholdLabels": false + } + }, { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": ["rgba(50, 172, 45, 0.97)", "rgba(237, 129, 40, 0.89)", "rgba(245, 54, 54, 0.9)"], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "id": 6, + "interval": null, + "links": [], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "targets": [{ + "expr": "prometheus_local_storage_memory_series", + "intervalFactor": 2, + "refId": "A", + "step": 4 + }], + "thresholds": "1,5", + "title": "Local Storage Memory Series", + "type": "singlestat", + "valueFontSize": "70%", + "valueMaps": [], + "valueName": "current", + "mappingTypes": [{ + "name": "value to text", + "value": 1 + }, { + "name": "range to text", + "value": 2 + }], + "rangeMaps": [{ + "from": "null", + "to": "null", + "text": "N/A" + }], + "mappingType": 1, + "gauge": { + "show": false, + "minValue": 0, + "maxValue": 100, + "thresholdMarkers": true, + "thresholdLabels": false + } + }, { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": ["rgba(50, 172, 45, 0.97)", "rgba(237, 129, 40, 0.89)", "rgba(245, 54, 54, 0.9)"], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "id": 7, + "interval": null, + "links": [], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "targets": [{ + "expr": "prometheus_local_storage_indexing_queue_length", + "intervalFactor": 2, + "refId": "A", + "step": 4 + }], + "thresholds": "500,4000", + "title": "Interal Storage Queue Length", + "type": "singlestat", + "valueFontSize": "70%", + "valueMaps": [{ + "op": "=", + "text": "Empty", + "value": "0" + }], + "valueName": "current", + "mappingTypes": [{ + "name": "value to text", + "value": 1 + }, { + "name": "range to text", + "value": 2 + }], + "rangeMaps": [{ + "from": "null", + "to": "null", + "text": "N/A" + }], + "mappingType": 1, + "gauge": { + "show": false, + "minValue": 0, + "maxValue": 100, + "thresholdMarkers": true, + "thresholdLabels": false + } + }, { + "content": "\"Prometheus\nPrometheus\n\n

You're using Prometheus, an open-source systems monitoring and alerting toolkit originally built at SoundCloud. For more information, check out the Grafana and Prometheus projects.

", + "editable": true, + "error": false, + "id": 9, + "links": [], + "mode": "html", + "span": 3, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }], + "title": "New row" + }, { + "collapse": false, + "editable": true, + "height": 227, + "panels": [{ + "aliasColors": { + "prometheus": "#C15C17", + "{instance=\"localhost:9090\",job=\"prometheus\"}": "#C15C17" + }, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 9, + "stack": false, + "steppedLine": false, + "targets": [{ + "expr": "rate(prometheus_local_storage_ingested_samples_total[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{job}}", + "metric": "", + "refId": "A", + "step": 2 + }], + "timeFrom": null, + "timeShift": null, + "title": "Samples ingested (rate-5m)", + "tooltip": { + "shared": true, + "value_type": "cumulative", + "ordering": "alphabetical", + "msResolution": false + }, + "type": "graph", + "yaxes": [{ + "show": true, + "min": null, + "max": null, + "logBase": 1, + "format": "short" + }, { + "show": true, + "min": null, + "max": null, + "logBase": 1, + "format": "short" + }], + "xaxis": { + "show": true + } + }, { + "content": "#### Samples Ingested\nThis graph displays the count of samples ingested by the Prometheus server, as measured over the last 5 minutes, per time series in the range vector. When troubleshooting an issue on IRC or Github, this is often the first stat requested by the Prometheus team. ", + "editable": true, + "error": false, + "id": 8, + "links": [], + "mode": "markdown", + "span": 2.995914043583536, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }], + "title": "New row" + }, { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [{ + "aliasColors": { + "prometheus": "#F9BA8F", + "{instance=\"localhost:9090\",interval=\"5s\",job=\"prometheus\"}": "#F9BA8F" + }, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [{ + "expr": "rate(prometheus_target_interval_length_seconds_count[5m])", + "intervalFactor": 2, + "legendFormat": "{{job}}", + "refId": "A", + "step": 2 + }], + "timeFrom": null, + "timeShift": null, + "title": "Target Scrapes (last 5m)", + "tooltip": { + "shared": true, + "value_type": "cumulative", + "ordering": "alphabetical", + "msResolution": false + }, + "type": "graph", + "yaxes": [{ + "show": true, + "min": null, + "max": null, + "logBase": 1, + "format": "short" + }, { + "show": true, + "min": null, + "max": null, + "logBase": 1, + "format": "short" + }], + "xaxis": { + "show": true + } + }, { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [{ + "expr": "prometheus_target_interval_length_seconds{quantile!=\"0.01\", quantile!=\"0.05\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{quantile}} ({{interval}})", + "metric": "", + "refId": "A", + "step": 2 + }], + "timeFrom": null, + "timeShift": null, + "title": "Scrape Duration", + "tooltip": { + "shared": true, + "value_type": "cumulative", + "ordering": "alphabetical", + "msResolution": false + }, + "type": "graph", + "yaxes": [{ + "show": true, + "min": null, + "max": null, + "logBase": 1, + "format": "short" + }, { + "show": true, + "min": null, + "max": null, + "logBase": 1, + "format": "short" + }], + "xaxis": { + "show": true + } + }, { + "content": "#### Scrapes\nPrometheus scrapes metrics from instrumented jobs, either directly or via an intermediary push gateway for short-lived jobs. Target scrapes will show how frequently targets are scraped, as measured over the last 5 minutes, per time series in the range vector. Scrape Duration will show how long the scrapes are taking, with percentiles available as series. ", + "editable": true, + "error": false, + "id": 11, + "links": [], + "mode": "markdown", + "span": 3, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }], + "title": "New row" + }, { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [{ + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 9, + "stack": false, + "steppedLine": false, + "targets": [{ + "expr": "prometheus_evaluator_duration_milliseconds{quantile!=\"0.01\", quantile!=\"0.05\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{quantile}}", + "refId": "A", + "step": 2 + }], + "timeFrom": null, + "timeShift": null, + "title": "Rule Eval Duration", + "tooltip": { + "shared": true, + "value_type": "cumulative", + "ordering": "alphabetical", + "msResolution": false + }, + "type": "graph", + "yaxes": [{ + "show": true, + "min": null, + "max": null, + "logBase": 1, + "format": "percentunit", + "label": "" + }, { + "show": true, + "min": null, + "max": null, + "logBase": 1, + "format": "short" + }], + "xaxis": { + "show": true + } + }, { + "content": "#### Rule Evaluation Duration\nThis graph panel plots the duration for all evaluations to execute. The 50th percentile, 90th percentile and 99th percentile are shown as three separate series to help identify outliers that may be skewing the data.", + "editable": true, + "error": false, + "id": 15, + "links": [], + "mode": "markdown", + "span": 3, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }], + "title": "New row" + }], + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"], + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] + }, + "templating": { + "list": [] + }, + "annotations": { + "list": [] + }, + "refresh": false, + "schemaVersion": 12, + "version": 0, + "links": [{ + "icon": "info", + "tags": [], + "targetBlank": true, + "title": "Grafana Docs", + "tooltip": "", + "type": "link", + "url": "http://www.grafana.org/docs" + }, { + "icon": "info", + "tags": [], + "targetBlank": true, + "title": "Prometheus Docs", + "type": "link", + "url": "http://prometheus.io/docs/introduction/overview/" + }], + "gnetId": 2, + "description": "The official, pre-built Prometheus Stats Dashboard." + } + grafana-net-737-dashboard.json: | + { + "__inputs": [{ + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + }], + "__requires": [{ + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + }, { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "3.1.0" + }, { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }], + "id": null, + "title": "Kubernetes Pod Resources", + "description": "Shows resource usage of Kubernetes pods.", + "tags": [ + "kubernetes" + ], + "style": "dark", + "timezone": "browser", + "editable": true, + "hideControls": false, + "sharedCrosshair": false, + "rows": [{ + "collapse": false, + "editable": true, + "height": "250px", + "panels": [{ + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "180px", + "id": 4, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [{ + "name": "value to text", + "value": 1 + }, { + "name": "range to text", + "value": 2 + }], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [{ + "from": "null", + "text": "N/A", + "to": "null" + }], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [{ + "expr": "sum (container_memory_working_set_bytes{id=\"/\",instance=~\"^$instance$\"}) / sum (machine_memory_bytes{instance=~\"^$instance$\"}) * 100", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 2 + }], + "thresholds": "65, 90", + "timeFrom": "1m", + "timeShift": null, + "title": "Memory Working Set", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [{ + "op": "=", + "text": "N/A", + "value": "null" + }], + "valueName": "current" + }, { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "180px", + "id": 6, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [{ + "name": "value to text", + "value": 1 + }, { + "name": "range to text", + "value": 2 + }], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [{ + "from": "null", + "text": "N/A", + "to": "null" + }], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [{ + "expr": "sum(rate(container_cpu_usage_seconds_total{id=\"/\",instance=~\"^$instance$\"}[1m])) / sum (machine_cpu_cores{instance=~\"^$instance$\"}) * 100", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + }], + "thresholds": "65, 90", + "timeFrom": "1m", + "timeShift": null, + "title": "Cpu Usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [{ + "op": "=", + "text": "N/A", + "value": "null" + }], + "valueName": "current" + }, { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "180px", + "id": 7, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [{ + "name": "value to text", + "value": 1 + }, { + "name": "range to text", + "value": 2 + }], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [{ + "from": "null", + "text": "N/A", + "to": "null" + }], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [{ + "expr": "sum(container_fs_usage_bytes{id=\"/\",instance=~\"^$instance$\"}) / sum(container_fs_limit_bytes{id=\"/\",instance=~\"^$instance$\"}) * 100", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10 + }], + "thresholds": "65, 90", + "timeFrom": "1m", + "timeShift": null, + "title": "Filesystem Usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [{ + "op": "=", + "text": "N/A", + "value": "null" + }], + "valueName": "current" + }, { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "1px", + "hideTimeOverride": true, + "id": 9, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [{ + "name": "value to text", + "value": 1 + }, { + "name": "range to text", + "value": 2 + }], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "20%", + "prefix": "", + "prefixFontSize": "20%", + "rangeMaps": [{ + "from": "null", + "text": "N/A", + "to": "null" + }], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [{ + "expr": "sum(container_memory_working_set_bytes{id=\"/\",instance=~\"^$instance$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + }], + "thresholds": "", + "timeFrom": "1m", + "title": "Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [{ + "op": "=", + "text": "N/A", + "value": "null" + }], + "valueName": "current" + }, { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "1px", + "hideTimeOverride": true, + "id": 10, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [{ + "name": "value to text", + "value": 1 + }, { + "name": "range to text", + "value": 2 + }], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [{ + "from": "null", + "text": "N/A", + "to": "null" + }], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [{ + "expr": "sum (machine_memory_bytes{instance=~\"^$instance$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + }], + "thresholds": "", + "timeFrom": "1m", + "title": "Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [{ + "op": "=", + "text": "N/A", + "value": "null" + }], + "valueName": "current" + }, { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "1px", + "hideTimeOverride": true, + "id": 11, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [{ + "name": "value to text", + "value": 1 + }, { + "name": "range to text", + "value": 2 + }], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [{ + "from": "null", + "text": "N/A", + "to": "null" + }], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [{ + "expr": "sum (rate (container_cpu_usage_seconds_total{id=\"/\",instance=~\"^$instance$\"}[1m]))", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + }], + "thresholds": "", + "timeFrom": "1m", + "timeShift": null, + "title": "Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [{ + "op": "=", + "text": "N/A", + "value": "null" + }], + "valueName": "current" + }, { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "1px", + "hideTimeOverride": true, + "id": 12, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [{ + "name": "value to text", + "value": 1 + }, { + "name": "range to text", + "value": 2 + }], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [{ + "from": "null", + "text": "N/A", + "to": "null" + }], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [{ + "expr": "sum (machine_cpu_cores{instance=~\"^$instance$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + }], + "thresholds": "", + "timeFrom": "1m", + "title": "Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [{ + "op": "=", + "text": "N/A", + "value": "null" + }], + "valueName": "current" + }, { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "1px", + "hideTimeOverride": true, + "id": 13, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [{ + "name": "value to text", + "value": 1 + }, { + "name": "range to text", + "value": 2 + }], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [{ + "from": "null", + "text": "N/A", + "to": "null" + }], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [{ + "expr": "sum(container_fs_usage_bytes{id=\"/\",instance=~\"^$instance$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + }], + "thresholds": "", + "timeFrom": "1m", + "title": "Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [{ + "op": "=", + "text": "N/A", + "value": "null" + }], + "valueName": "current" + }, { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "1px", + "hideTimeOverride": true, + "id": 14, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [{ + "name": "value to text", + "value": 1 + }, { + "name": "range to text", + "value": 2 + }], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [{ + "from": "null", + "text": "N/A", + "to": "null" + }], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [{ + "expr": "sum (container_fs_limit_bytes{id=\"/\",instance=~\"^$instance$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + }], + "thresholds": "", + "timeFrom": "1m", + "title": "Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [{ + "op": "=", + "text": "N/A", + "value": "null" + }], + "valueName": "current" + }, { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)", + "thresholdLine": false + }, + "height": "200px", + "id": 32, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [{ + "expr": "sum(rate(container_network_receive_bytes_total{instance=~\"^$instance$\",namespace=~\"^$namespace$\"}[1m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "receive", + "metric": "network", + "refId": "A", + "step": 240 + }, { + "expr": "- sum(rate(container_network_transmit_bytes_total{instance=~\"^$instance$\",namespace=~\"^$namespace$\"}[1m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "transmit", + "metric": "network", + "refId": "B", + "step": 240 + }], + "timeFrom": null, + "timeShift": null, + "title": "Network", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [{ + "format": "Bps", + "label": "transmit / receive", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + }] + }], + "showTitle": true, + "title": "all pods" + }, { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [{ + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 3, + "editable": true, + "error": false, + "fill": 0, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": "", + "id": 17, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [{ + "expr": "sum(rate(container_cpu_usage_seconds_total{image!=\"\",name=~\"^k8s_.*\",instance=~\"^$instance$\",namespace=~\"^$namespace$\"}[1m])) by (pod_name)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ pod_name }}", + "metric": "container_cpu", + "refId": "A", + "step": 240 + }], + "timeFrom": null, + "timeShift": null, + "title": "Cpu Usage", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [{ + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + }] + }, { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 33, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [{ + "expr": "sum (container_memory_working_set_bytes{image!=\"\",name=~\"^k8s_.*\",instance=~\"^$instance$\",namespace=~\"^$namespace$\"}) by (pod_name)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ pod_name }}", + "metric": "", + "refId": "A", + "step": 240 + }], + "timeFrom": null, + "timeShift": null, + "title": "Memory Working Set", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [{ + "format": "bytes", + "label": "used", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + }] + }, { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 16, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [{ + "expr": "sum (rate (container_network_receive_bytes_total{image!=\"\",name=~\"^k8s_.*\",instance=~\"^$instance$\",namespace=~\"^$namespace$\"}[1m])) by (pod_name)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ pod_name }} < in", + "metric": "network", + "refId": "A", + "step": 240 + }, { + "expr": "- sum (rate (container_network_transmit_bytes_total{image!=\"\",name=~\"^k8s_.*\",instance=~\"^$instance$\",namespace=~\"^$namespace$\"}[1m])) by (pod_name)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ pod_name }} > out", + "metric": "network", + "refId": "B", + "step": 240 + }], + "timeFrom": null, + "timeShift": null, + "title": "Network", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [{ + "format": "Bps", + "label": "transmit / receive", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + }] + }, { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 34, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [{ + "expr": "sum(container_fs_usage_bytes{image!=\"\",name=~\"^k8s_.*\",instance=~\"^$instance$\",namespace=~\"^$namespace$\"}) by (pod_name)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ pod_name }}", + "metric": "network", + "refId": "A", + "step": 240 + }], + "timeFrom": null, + "timeShift": null, + "title": "Filesystem", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [{ + "format": "bytes", + "label": "used", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + }] + }], + "showTitle": true, + "title": "each pod" + }], + "time": { + "from": "now-3d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "templating": { + "list": [{ + "allValue": ".*", + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(instance)", + "refresh": 1, + "regex": "", + "type": "query" + }, { + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "hide": 0, + "includeAll": true, + "label": "Namespace", + "multi": true, + "name": "namespace", + "options": [], + "query": "label_values(namespace)", + "refresh": 1, + "regex": "", + "type": "query" + }] + }, + "annotations": { + "list": [] + }, + "refresh": false, + "schemaVersion": 12, + "version": 8, + "links": [], + "gnetId": 737 + } + prometheus-datasource.json: | + { + "name": "prometheus", + "type": "prometheus", + "url": "http://prometheus:9090", + "access": "proxy", + "basicAuth": false + } +kind: ConfigMap +metadata: + creationTimestamp: null + name: grafana-import-dashboards + namespace: monitoring +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: grafana-import-dashboards + namespace: monitoring + labels: + app: grafana + component: import-dashboards +spec: + template: + metadata: + name: grafana-import-dashboards + labels: + app: grafana + component: import-dashboards + annotations: + pod.beta.kubernetes.io/init-containers: '[ + { + "name": "wait-for-endpoints", + "image": "giantswarm/tiny-tools", + "imagePullPolicy": "IfNotPresent", + "command": ["fish", "-c", "echo \"waiting for endpoints...\"; while true; set endpoints (curl -s --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt --header \"Authorization: Bearer \"(cat /var/run/secrets/kubernetes.io/serviceaccount/token) https://kubernetes.default.svc/api/v1/namespaces/monitoring/endpoints/grafana); echo $endpoints | jq \".\"; if test (echo $endpoints | jq -r \".subsets[]?.addresses // [] | length\") -gt 0; exit 0; end; echo \"waiting...\";sleep 1; end"], + "args": ["monitoring", "grafana"] + } + ]' + spec: + serviceAccountName: prometheus-k8s + containers: + - name: grafana-import-dashboards + image: giantswarm/tiny-tools + command: ["/bin/sh", "-c"] + workingDir: /opt/grafana-import-dashboards + args: + - > + for file in *-datasource.json ; do + if [ -e "$file" ] ; then + echo "importing $file" && + curl --silent --fail --show-error \ + --request POST http://admin:admin@grafana:80/api/datasources \ + --header "Content-Type: application/json" \ + --data-binary "@$file" ; + echo "" ; + fi + done ; + for file in *-dashboard.json ; do + if [ -e "$file" ] ; then + echo "importing $file" && + ( echo '{"dashboard":'; \ + cat "$file"; \ + echo ',"overwrite":true,"inputs":[{"name":"DS_PROMETHEUS","type":"datasource","pluginId":"prometheus","value":"prometheus"}]}' ) \ + | jq -c '.' \ + | curl --silent --fail --show-error \ + --request POST http://admin:admin@grafana:80/api/dashboards/import \ + --header "Content-Type: application/json" \ + --data-binary "@-" ; + echo "" ; + fi + done + + volumeMounts: + - name: config-volume + mountPath: /opt/grafana-import-dashboards + restartPolicy: Never + volumes: + - name: config-volume + configMap: + name: grafana-import-dashboards +--- +# apiVersion: extensions/v1beta1 +# kind: Ingress +# metadata: +# name: grafana +# namespace: monitoring +# spec: +# rules: +# - host: ..k8s.gigantic.io +# http: +# paths: +# - path: / +# backend: +# serviceName: grafana +# servicePort: 3000 +--- +apiVersion: v1 +kind: Service +metadata: + name: grafana + namespace: monitoring + labels: + app: grafana + component: core +spec: + type: LoadBalancer + ports: + - port: 80 + targetPort: http-server + selector: + app: grafana + component: core +#spec: +# type: NodePort +# ports: +# - port: 3000 +# selector: +# app: grafana +# component: core +--- +apiVersion: v1 +data: + prometheus.yaml: | + global: + scrape_interval: 10s + scrape_timeout: 10s + evaluation_interval: 10s + rule_files: + - "/etc/prometheus-rules/*.rules" + scrape_configs: + + # https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml#L37 + - job_name: 'kubernetes-nodes' + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - source_labels: [__address__] + regex: '(.*):10250' + replacement: '${1}:10255' + target_label: __address__ + + # https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml#L79 + - job_name: 'kubernetes-endpoints' + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] + action: replace + target_label: __scheme__ + regex: (https?) + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: (.+)(?::\d+);(\d+) + replacement: $1:$2 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_service_name] + action: replace + target_label: kubernetes_name + + # https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml#L119 + - job_name: 'kubernetes-services' + metrics_path: /probe + params: + module: [http_2xx] + kubernetes_sd_configs: + - role: service + relabel_configs: + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe] + action: keep + regex: true + - source_labels: [__address__] + target_label: __param_target + - target_label: __address__ + replacement: blackbox + - source_labels: [__param_target] + target_label: instance + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_service_name] + target_label: kubernetes_name + + - job_name: 'kubernetes-cadvisor' + metrics_path: /metrics/cadvisor + #metrics_path: /cadvisor + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - source_labels: [__address__] + regex: '(.*):10250' + replacement: '${1}:10255' + #replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor + target_label: __address__ + + # https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml#L156 + - job_name: 'kubernetes-pods' + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: (.+):(?:\d+);(\d+) + replacement: ${1}:${2} + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name + - source_labels: [__meta_kubernetes_pod_container_port_number] + action: keep + regex: 9\d{3} +kind: ConfigMap +metadata: + creationTimestamp: null + name: prometheus-core + namespace: monitoring +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: prometheus-core + namespace: monitoring + labels: + app: prometheus + component: core + triggerUpdate: "2" +spec: + strategy: + rollingUpdate: + maxSurge: 1 + type: RollingUpdate + replicas: 1 + template: + metadata: + name: prometheus-main + labels: + app: prometheus + component: core + spec: + serviceAccountName: prometheus-k8s + containers: + - name: watch + image: weaveworks/watch:master-5b2a6e5 + imagePullPolicy: IfNotPresent + args: ["-v", "-t", "-p=/etc/prometheus-rules", "curl", "-X", "POST", "--fail", "-o", "-", "-sS", "--max-time", "3602", "http://prometheus:9090/-/reload"] + volumeMounts: + - name: config-volume + mountPath: /etc/prometheus-rules + - name: prometheus + image: prom/prometheus:v1.7.0 + args: + - '-storage.local.retention=12h' + - '-storage.local.memory-chunks=500000' + - '-config.file=/etc/prometheus/prometheus.yaml' + ports: + - name: webui + containerPort: 9090 + resources: + requests: + cpu: 500m + memory: 500M + limits: + cpu: 500m + memory: 500M + volumeMounts: + - name: config-volume + mountPath: /etc/prometheus + - name: rules-volume + mountPath: /etc/prometheus-rules + volumes: + - name: config-volume + configMap: + name: prometheus-core + - name: rules-volume + configMap: + name: prometheus-rules +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: kube-state-metrics + namespace: monitoring +spec: + replicas: 2 + template: + metadata: + labels: + app: kube-state-metrics + spec: + serviceAccountName: kube-state-metrics + containers: + - name: kube-state-metrics + image: gcr.io/google_containers/kube-state-metrics:v0.5.0 + ports: + - containerPort: 8080 +--- +# --- +# apiVersion: rbac.authorization.k8s.io/v1beta1 +# kind: ClusterRoleBinding +# metadata: +# name: kube-state-metrics +# roleRef: +# apiGroup: rbac.authorization.k8s.io +# kind: ClusterRole +# name: kube-state-metrics +# subjects: +# - kind: ServiceAccount +# name: kube-state-metrics +# namespace: monitoring +# --- +# apiVersion: rbac.authorization.k8s.io/v1beta1 +# kind: ClusterRole +# metadata: +# name: kube-state-metrics +# rules: +# - apiGroups: [""] +# resources: +# - nodes +# - pods +# - services +# - resourcequotas +# - replicationcontrollers +# - limitranges +# verbs: ["list", "watch"] +# - apiGroups: ["extensions"] +# resources: +# - daemonsets +# - deployments +# - replicasets +# verbs: ["list", "watch"] +# --- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-state-metrics + namespace: monitoring +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/scrape: 'true' + name: kube-state-metrics + namespace: monitoring + labels: + app: kube-state-metrics +spec: + ports: + - name: kube-state-metrics + port: 8080 + protocol: TCP + selector: + app: kube-state-metrics + +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: node-directory-size-metrics + namespace: monitoring + annotations: + description: | + This `DaemonSet` provides metrics in Prometheus format about disk usage on the nodes. + The container `read-du` reads in sizes of all directories below /mnt and writes that to `/tmp/metrics`. It only reports directories larger then `100M` for now. + The other container `caddy` just hands out the contents of that file on request via `http` on `/metrics` at port `9102` which are the defaults for Prometheus. + These are scheduled on every node in the Kubernetes cluster. + To choose directories from the node to check, just mount them on the `read-du` container below `/mnt`. +spec: + template: + metadata: + labels: + app: node-directory-size-metrics + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '9102' + description: | + This `Pod` provides metrics in Prometheus format about disk usage on the node. + The container `read-du` reads in sizes of all directories below /mnt and writes that to `/tmp/metrics`. It only reports directories larger then `100M` for now. + The other container `caddy` just hands out the contents of that file on request on `/metrics` at port `9102` which are the defaults for Prometheus. + This `Pod` is scheduled on every node in the Kubernetes cluster. + To choose directories from the node to check just mount them on `read-du` below `/mnt`. + spec: + containers: + - name: read-du + image: giantswarm/tiny-tools + imagePullPolicy: Always + # FIXME threshold via env var + # The + command: + - fish + - --command + - | + touch /tmp/metrics-temp + while true + for directory in (du --bytes --separate-dirs --threshold=100M /mnt) + echo $directory | read size path + echo "node_directory_size_bytes{path=\"$path\"} $size" \ + >> /tmp/metrics-temp + end + mv /tmp/metrics-temp /tmp/metrics + sleep 300 + end + volumeMounts: + - name: host-fs-var + mountPath: /mnt/var + readOnly: true + - name: metrics + mountPath: /tmp + - name: caddy + image: dockermuenster/caddy:0.9.3 + command: + - "caddy" + - "-port=9102" + - "-root=/var/www" + ports: + - containerPort: 9102 + volumeMounts: + - name: metrics + mountPath: /var/www + volumes: + - name: host-fs-var + hostPath: + path: /var + - name: metrics + emptyDir: + medium: Memory +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: prometheus-node-exporter + namespace: monitoring + labels: + app: prometheus + component: node-exporter +spec: + template: + metadata: + name: prometheus-node-exporter + labels: + app: prometheus + component: node-exporter + spec: + containers: + - image: prom/node-exporter:v0.14.0 + name: prometheus-node-exporter + ports: + - name: prom-node-exp + #^ must be an IANA_SVC_NAME (at most 15 characters, ..) + containerPort: 9100 + hostPort: 9100 + hostNetwork: true + hostPID: true +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/scrape: 'true' + name: prometheus-node-exporter + namespace: monitoring + labels: + app: prometheus + component: node-exporter +spec: + clusterIP: None + ports: + - name: prometheus-node-exporter + port: 9100 + protocol: TCP + selector: + app: prometheus + component: node-exporter + type: ClusterIP +--- +apiVersion: v1 +data: + cpu-usage.rules: | + ALERT NodeCPUUsage + IF (100 - (avg by (instance) (irate(node_cpu{kubernetes_name="prometheus-node-exporter",mode="idle"}[5m])) * 100)) > 75 + FOR 2m + LABELS { + severity="page" + } + ANNOTATIONS { + SUMMARY = "{{$labels.instance}}: High CPU usage detected", + DESCRIPTION = "{{$labels.instance}}: CPU usage is above 75% (current value is: {{ $value }})" + } + instance-availability.rules: | + ALERT InstanceDown + IF up == 0 + FOR 1m + LABELS { severity = "page" } + ANNOTATIONS { + summary = "Instance {{ $labels.instance }} down", + description = "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 1 minute.", + } + low-disk-space.rules: | + + ALERT NodeLowDisk + IF ((node_filesystem_size{mountpoint="/"} - node_filesystem_free{mountpoint="/"} ) / node_filesystem_size{mountpoint="/"} * 100) > 75 + FOR 2m + LABELS { + severity="page" + } + ANNOTATIONS { + SUMMARY = "{{$labels.instance}}: Low disk space", + DESCRIPTION = "{{$labels.instance}}: Disk usage is above 75% (current value is: {{ $value }})" + } + mem-usage.rules: | + ALERT NodeSwapUsage + IF (((node_memory_SwapTotal-node_memory_SwapFree)/node_memory_SwapTotal)*100) > 7 + FOR 2m + LABELS { + severity="page" + } + ANNOTATIONS { + SUMMARY = "{{$labels.instance}}: Swap usage detected", + DESCRIPTION = "{{$labels.instance}}: Swap usage usage is above 75% (current value is: {{ $value }})" + } + + ALERT NodeMemoryUsage + IF (((node_memory_MemTotal-node_memory_MemFree-node_memory_Cached)/(node_memory_MemTotal)*100)) > 75 + FOR 2m + LABELS { + severity="page" + } + ANNOTATIONS { + SUMMARY = "{{$labels.instance}}: High memory usage detected", + DESCRIPTION = "{{$labels.instance}}: Memory usage is above 75% (current value is: {{ $value }})" + } +kind: ConfigMap +metadata: + creationTimestamp: null + name: prometheus-rules + namespace: monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: +- kind: ServiceAccount + name: prometheus-k8s + namespace: monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: prometheus +rules: +- apiGroups: [""] + resources: + - nodes + - services + - endpoints + - pods + - nodes/metrics + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: + - configmaps + verbs: ["get"] +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus-k8s + namespace: monitoring +--- +apiVersion: v1 +kind: Service +metadata: + name: prometheus + namespace: monitoring + labels: + app: prometheus + component: core + annotations: + prometheus.io/scrape: 'true' +spec: + type: NodePort + ports: + - port: 9090 + protocol: TCP + name: webui + selector: + app: prometheus + component: core diff --git a/prime/infra/dev/neo4j.yaml b/prime/infra/dev/neo4j.yaml index e6b720b77..0ec4cc2f8 100644 --- a/prime/infra/dev/neo4j.yaml +++ b/prime/infra/dev/neo4j.yaml @@ -41,7 +41,7 @@ spec: spec: containers: - name: neo4j - image: "neo4j:3.3.5-enterprise" + image: "neo4j:3.4.7-enterprise" imagePullPolicy: "IfNotPresent" env: - name: NEO4J_dbms_mode diff --git a/prime/infra/dev/prime-client-api.yaml b/prime/infra/dev/prime-client-api.yaml index b6e682c36..eea0a83a7 100644 --- a/prime/infra/dev/prime-client-api.yaml +++ b/prime/infra/dev/prime-client-api.yaml @@ -4,6 +4,9 @@ info: description: "The client API for Panacea." version: "1.0.0" host: "api.dev.ostelco.org" +x-google-endpoints: + - name: "api.dev.ostelco.org" + allowCors: true schemes: - "https" paths: @@ -30,11 +33,15 @@ paths: - application/json operationId: "createProfile" parameters: - - in: body - name: profile + - name: profile + in: body description: The profile to create. schema: $ref: '#/definitions/Profile' + - name: referred_by + in: query + description: "Referral ID of user who has invited this user" + type: string responses: 201: description: "Successfully created the profile." @@ -435,6 +442,41 @@ definitions: id: description: "The identifier for the source" type: string + details: + description: "All information stored with the source" + type: object + properties: + id: + type: string + accountType: + type: string + addressLine1: + type: string + addressLine2: + type: string + zip: + type: string + city: + type: string + state: + type: string + country: + type: string + currency: + type: string + brand: + type: string + last4: + type: string + expireMonth: + type: integer + expireYear: + type: integer + funding: + type: string + required: + - id + - accountType ConsentList: type: array items: @@ -482,7 +524,7 @@ definitions: PseudonymEntity: type: object properties: - msisdn: + sourceId: type: string pseudonym: type: string @@ -495,7 +537,7 @@ definitions: type: integer format: int64 required: - - msisdn + - sourceId - pseudonym - start - end @@ -527,4 +569,4 @@ securityDefinitions: type: "oauth2" x-google-issuer: "https://ostelco.eu.auth0.com/" x-google-jwks_uri: "https://ostelco.eu.auth0.com/.well-known/jwks.json" - x-google-audiences: "http://google_api" + x-google-audiences: "http://google_api" \ No newline at end of file diff --git a/prime/infra/dev/prime.yaml b/prime/infra/dev/prime.yaml index e78ea3e16..f2ad741c5 100644 --- a/prime/infra/dev/prime.yaml +++ b/prime/infra/dev/prime.yaml @@ -127,6 +127,7 @@ spec: readOnly: true - name: prime image: gcr.io/pantel-2decb/prime:PRIME_VERSION + imagePullPolicy: Always env: - name: FIREBASE_ROOT_PATH value: dev diff --git a/prime/infra/prod/neo4j.yaml b/prime/infra/prod/neo4j.yaml index 2fc8eea4a..0ec4cc2f8 100644 --- a/prime/infra/prod/neo4j.yaml +++ b/prime/infra/prod/neo4j.yaml @@ -41,7 +41,7 @@ spec: spec: containers: - name: neo4j - image: "neo4j:3.3.4-enterprise" + image: "neo4j:3.4.7-enterprise" imagePullPolicy: "IfNotPresent" env: - name: NEO4J_dbms_mode diff --git a/prime/infra/prod/prime-client-api.yaml b/prime/infra/prod/prime-client-api.yaml index 2d0234ba1..1b4e1b0dd 100644 --- a/prime/infra/prod/prime-client-api.yaml +++ b/prime/infra/prod/prime-client-api.yaml @@ -4,6 +4,9 @@ info: description: "The client API for Panacea." version: "1.0.0" host: "api.ostelco.org" +x-google-endpoints: + - name: "api.ostelco.org" + allowCors: true schemes: - "https" paths: @@ -30,11 +33,15 @@ paths: - application/json operationId: "createProfile" parameters: - - in: body - name: profile + - name: profile + in: body description: The profile to create. schema: $ref: '#/definitions/Profile' + - name: referred_by + in: query + description: "Referral ID of user who has invited this user" + type: string responses: 201: description: "Successfully created the profile." @@ -482,7 +489,7 @@ definitions: PseudonymEntity: type: object properties: - msisdn: + sourceId: type: string pseudonym: type: string @@ -495,7 +502,7 @@ definitions: type: integer format: int64 required: - - msisdn + - sourceId - pseudonym - start - end @@ -527,4 +534,4 @@ securityDefinitions: type: "oauth2" x-google-issuer: "https://ostelco.eu.auth0.com/" x-google-jwks_uri: "https://ostelco.eu.auth0.com/.well-known/jwks.json" - x-google-audiences: "http://google_api" + x-google-audiences: "http://google_api" \ No newline at end of file diff --git a/prime/infra/prod/prime.yaml b/prime/infra/prod/prime.yaml index de74f8fc2..725015ddc 100644 --- a/prime/infra/prod/prime.yaml +++ b/prime/infra/prod/prime.yaml @@ -91,6 +91,7 @@ spec: readOnly: true - name: prime image: gcr.io/pantel-2decb/prime:PRIME_VERSION + imagePullPolicy: Always env: - name: FIREBASE_ROOT_PATH value: v2 diff --git a/prime/infra/raw_purchases_schema.ddl b/prime/infra/raw_purchases_schema.ddl new file mode 100644 index 000000000..dabb4fd95 --- /dev/null +++ b/prime/infra/raw_purchases_schema.ddl @@ -0,0 +1,23 @@ + CREATE TABLE purchases.raw_purchases + ( + id STRING NOT NULL, + subscriberId STRING NOT NULL, + timestamp INT64 NOT NULL, + status STRING NOT NULL, + product STRUCT< + sku STRING NOT NULL, + price STRUCT< + amount INT64 NOT NULL, + currency STRING NOT NULL + > NOT NULL, + properties ARRAY< STRUCT< + key STRING NOT NULL, + value STRING NOT NULL + > >, + presentation ARRAY< STRUCT< + key STRING NOT NULL, + value STRING NOT NULL + > > + > NOT NULL +) +PARTITION BY DATE(_PARTITIONTIME) diff --git a/prime/script/deploy-dev-direct.sh b/prime/script/deploy-dev-direct.sh index 408df2c44..f25997bbd 100755 --- a/prime/script/deploy-dev-direct.sh +++ b/prime/script/deploy-dev-direct.sh @@ -7,7 +7,7 @@ if [ ! -f prime/script/deploy.sh ]; then exit 1 fi -# TODO vihang: check if the kubectl config points to dev cluster +kubectl config use-context $(kubectl config get-contexts --output name | grep dev-cluster) PROJECT_ID="$(gcloud config get-value project -q)" PRIME_VERSION="$(gradle prime:properties -q | grep "version:" | awk '{print $2}' | tr -d '[:space:]')" diff --git a/prime/script/deploy-direct.sh b/prime/script/deploy-direct.sh index e4416a1ab..6568413dd 100755 --- a/prime/script/deploy-direct.sh +++ b/prime/script/deploy-direct.sh @@ -14,6 +14,8 @@ if [ ! -f ${CHECK_REPO} ]; then exit 1 fi +kubectl config use-context $(kubectl config get-contexts --output name | grep private-cluster) + BRANCH_NAME=$(git branch | grep \* | cut -d ' ' -f2) echo BRANCH_NAME=${BRANCH_NAME} ${CHECK_REPO} ${BRANCH_NAME} diff --git a/prime/script/wait.sh b/prime/script/wait.sh index f5450a2cb..b3bfd9d1e 100755 --- a/prime/script/wait.sh +++ b/prime/script/wait.sh @@ -38,6 +38,8 @@ echo "Creating topics and subscriptions...." curl -X PUT pubsub-emulator:8085/v1/projects/pantel-2decb/topics/data-traffic curl -X PUT pubsub-emulator:8085/v1/projects/pantel-2decb/topics/pseudo-traffic curl -X PUT -H "Content-Type: application/json" -d '{"topic":"projects/pantel-2decb/topics/data-traffic","ackDeadlineSeconds":10}' pubsub-emulator:8085/v1/projects/pantel-2decb/subscriptions/test-pseudo +curl -X PUT pubsub-emulator:8085/v1/projects/pantel-2decb/topics/purchase-info +curl -X PUT -H "Content-Type: application/json" -d '{"topic":"projects/pantel-2decb/topics/purchase-info","ackDeadlineSeconds":10}' pubsub-emulator:8085/v1/projects/pantel-2decb/subscriptions/purchase-info-sub echo "Done creating topics and subscriptions" diff --git a/prime/src/integration-tests/resources/config.yaml b/prime/src/integration-tests/resources/config.yaml index a8475ea4c..6e4faf8eb 100644 --- a/prime/src/integration-tests/resources/config.yaml +++ b/prime/src/integration-tests/resources/config.yaml @@ -11,7 +11,8 @@ modules: - type: analytics config: projectId: pantel-2decb - topicId: data-traffic + dataTrafficTopicId: data-traffic + purchaseInfoTopicId: purchase-info - type: ocs config: lowBalanceThreshold: 0 diff --git a/prime/src/integration-tests/resources/docker-compose.yaml b/prime/src/integration-tests/resources/docker-compose.yaml index 28aff66dc..633d77427 100644 --- a/prime/src/integration-tests/resources/docker-compose.yaml +++ b/prime/src/integration-tests/resources/docker-compose.yaml @@ -3,7 +3,7 @@ version: "3.3" services: neo4j: container_name: "neo4j" - image: neo4j:3.4.4 + image: neo4j:3.4.7 environment: - NEO4J_AUTH=none ports: diff --git a/pseudonym-server/build.gradle b/pseudonym-server/build.gradle index 6f5af1d2f..926ea958c 100644 --- a/pseudonym-server/build.gradle +++ b/pseudonym-server/build.gradle @@ -12,7 +12,7 @@ dependencies { implementation project(':analytics-grpc-api') implementation "io.dropwizard:dropwizard-client:$dropwizardVersion" - implementation 'com.google.guava:guava:25.1-jre' + implementation "com.google.guava:guava:$guavaVersion" // Match with grpc-netty-shaded via PubSub // removing io.grpc:grpc-netty-shaded:1.14.0 causes ALPN error implementation 'io.grpc:grpc-netty-shaded:1.14.0' diff --git a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/Model.kt b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/Model.kt index 81d6334f1..e02687f3c 100644 --- a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/Model.kt +++ b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/Model.kt @@ -1,10 +1,12 @@ package org.ostelco.pseudonym -const val PseudonymEntityKind = "Pseudonym" +const val MsisdnPseudonymEntityKind = "Pseudonym" const val msisdnPropertyName = "msisdn" const val pseudonymPropertyName = "pseudonym" const val startPropertyName = "start" const val endPropertyName = "end" +const val SubscriberIdPseudonymEntityKind = "SubscriberPseudonym" +const val subscriberIdPropertyName = "subscriberId" const val ExportTaskKind = "ExportTask" const val exportIdPropertyName = "exportId" diff --git a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/resources/PseudonymResource.kt b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/resources/PseudonymResource.kt index 054bdabf2..85a6f37c9 100644 --- a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/resources/PseudonymResource.kt +++ b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/resources/PseudonymResource.kt @@ -37,7 +37,7 @@ class PseudonymResource { fun getPseudonym(@NotBlank @PathParam("msisdn") msisdn: String, @NotBlank @PathParam("timestamp") timestamp: String): Response { logger.info("GET pseudonym for Msisdn = $msisdn at timestamp = $timestamp") - val entity = PseudonymizerServiceSingleton.getPseudonymEntityFor(msisdn, timestamp.toLong()) + val entity = PseudonymizerServiceSingleton.getMsisdnPseudonym(msisdn, timestamp.toLong()) return Response.ok(entity, MediaType.APPLICATION_JSON).build() } @@ -51,7 +51,7 @@ class PseudonymResource { fun getPseudonym(@NotBlank @PathParam("msisdn") msisdn: String): Response { val timestamp = Instant.now().toEpochMilli() logger.info("GET pseudonym for Msisdn = $msisdn at current time, timestamp = $timestamp") - val entity = PseudonymizerServiceSingleton.getPseudonymEntityFor(msisdn, timestamp) + val entity = PseudonymizerServiceSingleton.getMsisdnPseudonym(msisdn, timestamp) return Response.ok(entity, MediaType.APPLICATION_JSON).build() } @@ -76,7 +76,7 @@ class PseudonymResource { @Path("/find/{pseudonym}") fun findPseudonym(@NotBlank @PathParam("pseudonym") pseudonym: String): Response { logger.info("Find details for pseudonym = $pseudonym") - return PseudonymizerServiceSingleton.findPseudonym(pseudonym = pseudonym) + return PseudonymizerServiceSingleton.findMsisdnPseudonym(pseudonym = pseudonym) ?.let { Response.ok(it, MediaType.APPLICATION_JSON).build() } ?: Response.status(Status.NOT_FOUND).build() } @@ -90,7 +90,7 @@ class PseudonymResource { @Path("/delete/{msisdn}") fun deleteAllPseudonyms(@NotBlank @PathParam("msisdn") msisdn: String): Response { logger.info("delete all pseudonyms for Msisdn = $msisdn") - val count = PseudonymizerServiceSingleton.deleteAllPseudonyms(msisdn = msisdn) + val count = PseudonymizerServiceSingleton.deleteAllMsisdnPseudonyms(msisdn = msisdn) // Return a Json object with number of records deleted. val countMap = mapOf("count" to count) logger.info("deleted $count records for Msisdn = $msisdn") @@ -106,7 +106,7 @@ class PseudonymResource { @Path("/export/{exportId}") fun exportPseudonyms(@NotBlank @PathParam("exportId") exportId: String): Response { logger.info("GET export all pseudonyms to the table $exportId") - PseudonymizerServiceSingleton.exportPseudonyms(exportId = exportId) + PseudonymizerServiceSingleton.exportMsisdnPseudonyms(exportId = exportId) return Response.ok("Started Exporting", MediaType.TEXT_PLAIN).build() } diff --git a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/service/PseudonymExport.kt b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/service/PseudonymExport.kt index e94c9872d..242c7d0dd 100644 --- a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/service/PseudonymExport.kt +++ b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/service/PseudonymExport.kt @@ -17,7 +17,7 @@ import com.google.cloud.datastore.StructuredQuery import com.google.common.cache.Cache import com.google.common.cache.CacheBuilder import org.ostelco.pseudonym.ExportTaskKind -import org.ostelco.pseudonym.PseudonymEntityKind +import org.ostelco.pseudonym.MsisdnPseudonymEntityKind import org.ostelco.pseudonym.errorPropertyName import org.ostelco.pseudonym.exportIdPropertyName import org.ostelco.pseudonym.msisdnPropertyName @@ -83,7 +83,7 @@ class PseudonymExport(private val exportId: String, private val bigquery: BigQue // Dump pseudonyms to BQ, one page at a time. Since all records in a // page are inserted at once, use a small page size val queryBuilder = Query.newEntityQueryBuilder() - .setKind(PseudonymEntityKind) + .setKind(MsisdnPseudonymEntityKind) .setOrderBy(StructuredQuery.OrderBy.asc(msisdnPropertyName)) .setLimit(pageSize) if (cursor != null) { diff --git a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/service/PseudonymizerServiceSingleton.kt b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/service/PseudonymizerServiceSingleton.kt index 44509bb39..e82932c4c 100644 --- a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/service/PseudonymizerServiceSingleton.kt +++ b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/service/PseudonymizerServiceSingleton.kt @@ -18,17 +18,8 @@ import org.ostelco.prime.logger import org.ostelco.prime.model.ActivePseudonyms import org.ostelco.prime.model.PseudonymEntity import org.ostelco.prime.pseudonymizer.PseudonymizerService -import org.ostelco.pseudonym.ConfigRegistry -import org.ostelco.pseudonym.ExportTaskKind -import org.ostelco.pseudonym.PseudonymEntityKind -import org.ostelco.pseudonym.endPropertyName -import org.ostelco.pseudonym.errorPropertyName -import org.ostelco.pseudonym.exportIdPropertyName -import org.ostelco.pseudonym.msisdnPropertyName -import org.ostelco.pseudonym.pseudonymPropertyName +import org.ostelco.pseudonym.* import org.ostelco.pseudonym.resources.ExportTask -import org.ostelco.pseudonym.startPropertyName -import org.ostelco.pseudonym.statusPropertyName import org.ostelco.pseudonym.utils.WeeklyBounds import java.time.Instant import java.util.* @@ -67,9 +58,14 @@ object PseudonymizerServiceSingleton : PseudonymizerService { private var bigQuery: BigQuery? = null private val dateBounds: DateBounds = WeeklyBounds() + private val msisdnPseudonymiser: Pseudonymizer = Pseudonymizer(MsisdnPseudonymEntityKind, msisdnPropertyName) + private val subscriberIdPseudonymiser: Pseudonymizer = Pseudonymizer(SubscriberIdPseudonymEntityKind, subscriberIdPropertyName) private val executor = Executors.newFixedThreadPool(3) - val pseudonymCache: Cache = CacheBuilder.newBuilder() + val msisdnPseudonymCache: Cache = CacheBuilder.newBuilder() + .maximumSize(5000) + .build() + val subscriberIdPseudonymCache: Cache = CacheBuilder.newBuilder() .maximumSize(5000) .build() @@ -83,57 +79,46 @@ object PseudonymizerServiceSingleton : PseudonymizerService { logger.info("Local testing, BigQuery is not available...") null } + msisdnPseudonymiser.init(datastore, bigQuery, dateBounds) + subscriberIdPseudonymiser.init(datastore, bigQuery, dateBounds) } override fun getActivePseudonymsForMsisdn(msisdn: String): ActivePseudonyms { val currentTimestamp = Instant.now().toEpochMilli() val nextTimestamp = dateBounds.getNextPeriodStart(currentTimestamp) logger.info("GET pseudonym for Msisdn = $msisdn at timestamps = $currentTimestamp & $nextTimestamp") - val current = getPseudonymEntityFor(msisdn, currentTimestamp) - val next = getPseudonymEntityFor(msisdn, nextTimestamp) + val current = getMsisdnPseudonym(msisdn, currentTimestamp) + val next = getMsisdnPseudonym(msisdn, nextTimestamp) return ActivePseudonyms(current, next) } - override fun getPseudonymEntityFor(msisdn: String, timestamp: Long): PseudonymEntity { + override fun getMsisdnPseudonym(msisdn: String, timestamp: Long): PseudonymEntity { val (bounds, keyPrefix) = dateBounds.getBoundsNKeyPrefix(msisdn, timestamp) // Retrieves the element from cache. - return pseudonymCache.get(keyPrefix) { - getPseudonymEntity(keyPrefix) ?: createPseudonym(msisdn, bounds, keyPrefix) + return msisdnPseudonymCache.get(keyPrefix) { + msisdnPseudonymiser.getPseudonymEntity(keyPrefix) + ?: msisdnPseudonymiser.createPseudonym(msisdn, bounds, keyPrefix) } } - fun findPseudonym(pseudonym: String): PseudonymEntity? { - val query = Query.newEntityQueryBuilder() - .setKind(PseudonymEntityKind) - .setFilter(PropertyFilter.eq(pseudonymPropertyName, pseudonym)) - .setLimit(1) - .build() - val results = datastore.run(query) - if (results.hasNext()) { - val entity = results.next() - return convertToPseudonymEntity(entity) + override fun getSubscriberIdPseudonym(subscriberId: String, timestamp: Long): PseudonymEntity { + val (bounds, keyPrefix) = dateBounds.getBoundsNKeyPrefix(subscriberId, timestamp) + // Retrieves the element from cache. + return subscriberIdPseudonymCache.get(keyPrefix) { + subscriberIdPseudonymiser.getPseudonymEntity(keyPrefix) + ?: subscriberIdPseudonymiser.createPseudonym(subscriberId, bounds, keyPrefix) } - logger.info("Couldn't find, pseudonym = $pseudonym") - return null } - fun deleteAllPseudonyms(msisdn: String): Int { - val query = Query.newEntityQueryBuilder() - .setKind(PseudonymEntityKind) - .setFilter(PropertyFilter.eq(msisdnPropertyName, msisdn)) - .setLimit(1) - .build() - val results = datastore.run(query) - var count = 0 - while (results.hasNext()) { - val entity = results.next() - datastore.delete(entity.key) - count++ - } - return count + fun findMsisdnPseudonym(pseudonym: String): PseudonymEntity? { + return msisdnPseudonymiser.findPseudonym(pseudonym) + } + + fun deleteAllMsisdnPseudonyms(msisdn: String): Int { + return msisdnPseudonymiser.deleteAllPseudonyms(msisdn) } - fun exportPseudonyms(exportId: String) { + fun exportMsisdnPseudonyms(exportId: String) { bigQuery?.apply { logger.info("GET export all pseudonyms to the table $exportId") val exporter = PseudonymExport(exportId = exportId, bigquery = this, datastore = datastore) @@ -202,12 +187,57 @@ object PseudonymizerServiceSingleton : PseudonymizerService { } return null } +} + + +class Pseudonymizer(val entityKind: String, val sourcePropertyName: String) { + private val logger by logger() + private lateinit var datastore: Datastore + private var bigQuery: BigQuery? = null + private lateinit var dateBounds: DateBounds + + fun init(ds: Datastore, bq: BigQuery? = null, bounds: DateBounds) { + datastore = ds + bigQuery = bq + dateBounds = bounds + } + + fun findPseudonym(pseudonym: String): PseudonymEntity? { + val query = Query.newEntityQueryBuilder() + .setKind(entityKind) + .setFilter(PropertyFilter.eq(pseudonymPropertyName, pseudonym)) + .setLimit(1) + .build() + val results = datastore.run(query) + if (results.hasNext()) { + val entity = results.next() + return convertToPseudonymEntity(entity) + } + logger.info("Couldn't find, pseudonym = $pseudonym") + return null + } + + fun deleteAllPseudonyms(sourceId: String): Int { + val query = Query.newEntityQueryBuilder() + .setKind(entityKind) + .setFilter(PropertyFilter.eq(sourcePropertyName, sourceId)) + .setLimit(1) + .build() + val results = datastore.run(query) + var count = 0 + while (results.hasNext()) { + val entity = results.next() + datastore.delete(entity.key) + count++ + } + return count + } private fun getPseudonymKey(keyPrefix: String): Key { - return datastore.newKeyFactory().setKind(PseudonymEntityKind).newKey(keyPrefix) + return datastore.newKeyFactory().setKind(entityKind).newKey(keyPrefix) } - private fun getPseudonymEntity(keyPrefix: String): PseudonymEntity? { + fun getPseudonymEntity(keyPrefix: String): PseudonymEntity? { val pseudonymKey = getPseudonymKey(keyPrefix) val value = datastore.get(pseudonymKey) if (value != null) { @@ -217,9 +247,9 @@ object PseudonymizerServiceSingleton : PseudonymizerService { return null } - private fun createPseudonym(msisdn: String, bounds: Bounds, keyPrefix: String): PseudonymEntity { + fun createPseudonym(sourceId: String, bounds: Bounds, keyPrefix: String): PseudonymEntity { val uuid = UUID.randomUUID().toString() - var entity = PseudonymEntity(msisdn, uuid, bounds.start, bounds.end) + var entity = PseudonymEntity(sourceId, uuid, bounds.start, bounds.end) val pseudonymKey = getPseudonymKey(keyPrefix) val transaction = datastore.newTransaction() @@ -229,7 +259,7 @@ object PseudonymizerServiceSingleton : PseudonymizerService { if (currentEntity == null) { // Prepare the new datastore entity val pseudonym = Entity.newBuilder(pseudonymKey) - .set(msisdnPropertyName, entity.msisdn) + .set(sourcePropertyName, entity.sourceId) .set(pseudonymPropertyName, entity.pseudonym) .set(startPropertyName, entity.start) .set(endPropertyName, entity.end) @@ -250,9 +280,9 @@ object PseudonymizerServiceSingleton : PseudonymizerService { private fun convertToPseudonymEntity(entity: Entity): PseudonymEntity { return PseudonymEntity( - entity.getString(msisdnPropertyName), + entity.getString(sourcePropertyName), entity.getString(pseudonymPropertyName), entity.getLong(startPropertyName), entity.getLong(endPropertyName)) } -} \ No newline at end of file +} diff --git a/pseudonym-server/src/test/kotlin/org/ostelco/pseudonym/PseudonymResourceTest.kt b/pseudonym-server/src/test/kotlin/org/ostelco/pseudonym/PseudonymResourceTest.kt index 7c9d6ce98..288afdd53 100644 --- a/pseudonym-server/src/test/kotlin/org/ostelco/pseudonym/PseudonymResourceTest.kt +++ b/pseudonym-server/src/test/kotlin/org/ostelco/pseudonym/PseudonymResourceTest.kt @@ -92,7 +92,7 @@ class PseudonymResourceTest { assertEquals(Status.OK.statusCode, result.status) val json = result.readEntity(String::class.java) pseudonymEntity = mapper.readValue(json) - assertEquals(testMsisdn1, pseudonymEntity.msisdn) + assertEquals(testMsisdn1, pseudonymEntity.sourceId) } run { @@ -126,7 +126,7 @@ class PseudonymResourceTest { assertEquals(Status.OK.statusCode, result.status) val json = result.readEntity(String::class.java) pseudonymEntity = mapper.readValue(json) - assertEquals(testMsisdn1, pseudonymEntity.msisdn) + assertEquals(testMsisdn1, pseudonymEntity.sourceId) } run { @@ -168,7 +168,7 @@ class PseudonymResourceTest { assertEquals(Status.OK.statusCode, result.status) val json = result.readEntity(String::class.java) pseudonymEntity = mapper.readValue(json) - assertEquals(testMsisdn1, pseudonymEntity.msisdn) + assertEquals(testMsisdn1, pseudonymEntity.sourceId) } run { @@ -203,7 +203,7 @@ class PseudonymResourceTest { assertEquals(Status.OK.statusCode, result.status) val json = result.readEntity(String::class.java) pseudonymEntity = mapper.readValue(json) - assertEquals(testMsisdn1, pseudonymEntity.msisdn) + assertEquals(testMsisdn1, pseudonymEntity.sourceId) } run { @@ -216,7 +216,7 @@ class PseudonymResourceTest { assertEquals(Status.OK.statusCode, result.status) val json = result.readEntity(String::class.java) val pseudonymEntity2 = mapper.readValue(json) - assertEquals(testMsisdn1, pseudonymEntity2.msisdn) + assertEquals(testMsisdn1, pseudonymEntity2.sourceId) } } @@ -236,7 +236,7 @@ class PseudonymResourceTest { assertEquals(Status.OK.statusCode, result.status) val json = result.readEntity(String::class.java) pseudonymEntity = mapper.readValue(json) - assertEquals(testMsisdn2, pseudonymEntity.msisdn) + assertEquals(testMsisdn2, pseudonymEntity.sourceId) } run { diff --git a/scripts/deploy-ocsgw.sh b/scripts/deploy-ocsgw.sh index 30af284ab..ad84ea7c5 100755 --- a/scripts/deploy-ocsgw.sh +++ b/scripts/deploy-ocsgw.sh @@ -7,10 +7,6 @@ # ctr-c # -echo "Starting to deploy OCSGW to test installation" -echo "The last thing this script will do is to look at logs from the ocsgw" -echo "It will continue to do so until terminated by ^C" - variant=dev host_ip=192.168.0.124 if [ "$1" = prod ] ; then @@ -18,6 +14,11 @@ if [ "$1" = prod ] ; then variant=prod fi +echo "Starting to deploy OCSGW to $variant" +echo "The last thing this script will do is to look at logs from the ocsgw" +echo "It will continue to do so until terminated by ^C" + + scp -oProxyJump=loltel@10.6.101.1 build/deploy/ostelco-core-${variant}.zip ubuntu@${host_ip}:. ssh -A -Jloltel@10.6.101.1 ubuntu@${host_ip} < + bin/neo4j-admin backup + --backup-dir=/backup_dir + --name=graph.db-backup + --from=host.docker.internal + --cc-report-dir=/backup_dir + volumes: + - "./backup_dir:/backup_dir" + environment: + - NEO4J_ACCEPT_LICENSE_AGREEMENT=yes \ No newline at end of file diff --git a/tools/neo4j-admin-tools/docker-compose.neo4j.yaml b/tools/neo4j-admin-tools/docker-compose.neo4j.yaml new file mode 100644 index 000000000..4258e14aa --- /dev/null +++ b/tools/neo4j-admin-tools/docker-compose.neo4j.yaml @@ -0,0 +1,13 @@ +version: "3.7" + +services: + neo4j: + container_name: "neo4j" + image: neo4j:3.4.7 + environment: + - NEO4J_AUTH=none + ports: + - "7687:7687" + - "7474:7474" + volumes: + - "./data_dir:/data" \ No newline at end of file diff --git a/tools/neo4j-admin-tools/docker-compose.restore.yaml b/tools/neo4j-admin-tools/docker-compose.restore.yaml new file mode 100644 index 000000000..685876573 --- /dev/null +++ b/tools/neo4j-admin-tools/docker-compose.restore.yaml @@ -0,0 +1,16 @@ +version: "3.7" + +services: + neo4j-online-restore: + container_name: neo4j-online-restore + image: neo4j:3.4.7-enterprise + command: > + bin/neo4j-admin restore + --from=/backup_dir/graph.db-backup + --database=graph.db + --force + volumes: + - "./backup_dir:/backup_dir" + - "./data_dir:/data" + environment: + - NEO4J_ACCEPT_LICENSE_AGREEMENT=yes \ No newline at end of file diff --git a/tools/neo4j-admin-tools/docker-compose.yaml b/tools/neo4j-admin-tools/docker-compose.yaml new file mode 100644 index 000000000..8be1a6627 --- /dev/null +++ b/tools/neo4j-admin-tools/docker-compose.yaml @@ -0,0 +1,12 @@ +version: "3.7" + +services: + neo4j: + container_name: "neo4j" + image: neo4j:3.4.7 + environment: + - NEO4J_AUTH=none + ports: + - "7687:7687" + - "7474:7474" + tmpfs: "/data" diff --git a/tools/neo4j-admin-tools/src/main/kotlin/org/ostelco/tools/migration/MainKt.kt b/tools/neo4j-admin-tools/src/main/kotlin/org/ostelco/tools/migration/MainKt.kt index 860a31246..bb57ed0bd 100644 --- a/tools/neo4j-admin-tools/src/main/kotlin/org/ostelco/tools/migration/MainKt.kt +++ b/tools/neo4j-admin-tools/src/main/kotlin/org/ostelco/tools/migration/MainKt.kt @@ -1,53 +1,80 @@ package org.ostelco.tools.migration import org.neo4j.driver.v1.AccessMode +import java.nio.file.Files +import java.nio.file.Paths fun main(args: Array) { - cypherFileToNeo4jImporter() + neo4jExporterToCypherFile() + // cypherFileToNeo4jImporter() } -fun cypherFileAndFirebaseToNeo4jMigration() { - initFirebase() +fun neo4jExporterToCypherFile() { Neo4jClient.init() - Neo4jClient.driver.session(AccessMode.WRITE).use { + Neo4jClient.driver.session(AccessMode.READ).use { session -> - val txn = it.beginTransaction() + val txn = session.beginTransaction() - println("Import from file to Neo4j") + println("Import from Neo4j to file") - importFromCypherFile("src/main/resources/init.cypher") { - query -> txn.run(query) + importFromNeo4j(txn) { str -> + Files.write(Paths.get("src/main/resources/backup.cypher"), str.toByteArray()) } - println("Exporting from firebase and import it to Neo4j") - importFromFirebase { - createQuery -> txn.run(createQuery) + println("Done") + txn.success() + } + + Neo4jClient.stop() +} + +fun cypherFileToNeo4jImporter() { + + Neo4jClient.init() + + Neo4jClient.driver.session(AccessMode.WRITE).use { session -> + + val txn = session.beginTransaction() + + println("Import from file to Neo4j") + + importFromCypherFile("src/main/resources/backup.prod.cypher") { query -> + txn.run(query) } println("Done") txn.success() } + Neo4jClient.stop() } -fun cypherFileToNeo4jImporter() { +fun cypherFileAndFirebaseToNeo4jMigration() { + initFirebase() Neo4jClient.init() - Neo4jClient.driver.session(AccessMode.WRITE).use { + Neo4jClient.driver.session(AccessMode.WRITE).use { session -> - val txn = it.beginTransaction() + val txn = session.beginTransaction() println("Import from file to Neo4j") - importFromCypherFile("src/main/resources/init.cypher") { - query -> txn.run(query) + importFromCypherFile("src/main/resources/init.cypher") { query -> + txn.run(query) + } + + println("Exporting from firebase and import it to Neo4j") + importFromFirebase { createQuery -> + txn.run(createQuery) } println("Done") txn.success() } + Neo4jClient.stop() -} \ No newline at end of file +} + diff --git a/tools/neo4j-admin-tools/src/main/kotlin/org/ostelco/tools/migration/Neo4jExporter.kt b/tools/neo4j-admin-tools/src/main/kotlin/org/ostelco/tools/migration/Neo4jExporter.kt new file mode 100644 index 000000000..914ff70f0 --- /dev/null +++ b/tools/neo4j-admin-tools/src/main/kotlin/org/ostelco/tools/migration/Neo4jExporter.kt @@ -0,0 +1,53 @@ +package org.ostelco.tools.migration + +import org.neo4j.driver.v1.Transaction + +fun importFromNeo4j(txn: Transaction, handleCypher: (String) -> Unit) { + + val sb = StringBuilder() + + run { + val stmtResult = txn.run("MATCH (n) RETURN n;") + stmtResult.forEach { record -> + val node = record["n"].asNode() + val labels = node.labels().joinToString(separator = "", prefix = ":") + + val props = node.asMap().map { entry -> + "`${entry.key}`: '${entry.value}'" + }.joinToString(separator = ",\n") + + sb.append("CREATE ($labels {$props});\n\n") + } + } + + run { + val stmtResult = txn.run("MATCH (n)-[r]->(m) RETURN n,r,m;") + stmtResult.forEach { record -> + val fromNode = record["n"].asNode() + val relation = record["r"].asRelationship() + val toNode = record["m"].asNode() + + val type = relation.type() + + var props = relation.asMap().map { entry -> + "`${entry.key}`: '${entry.value}'" + }.joinToString(separator = ",\n") + + props = if (props.isNotBlank()) { + " {$props}" + } else { + props + } + + sb.append( +""" +MATCH (n:${fromNode.labels().first()} {id: '${fromNode.asMap()["id"]}'}) + WITH n +MATCH (m:${toNode.labels().first()} {id: '${toNode.asMap()["id"]}'}) +CREATE (n)-[:$type$props]->(m); +""") + } + } + + handleCypher(sb.toString()) +} \ No newline at end of file diff --git a/tools/neo4j-admin-tools/src/main/resources/.gitignore b/tools/neo4j-admin-tools/src/main/resources/.gitignore index b329cc412..82b1ba983 100644 --- a/tools/neo4j-admin-tools/src/main/resources/.gitignore +++ b/tools/neo4j-admin-tools/src/main/resources/.gitignore @@ -1,2 +1,5 @@ prod.cypher -test.cypher \ No newline at end of file +test.cypher +backup.cypher +backup.dev.cypher +backup.prod.cypher \ No newline at end of file diff --git a/tools/neo4j-admin-tools/src/main/resources/docker-compose.yaml b/tools/neo4j-admin-tools/src/main/resources/docker-compose.yaml index c3b46e20f..e8f1e59fa 100644 --- a/tools/neo4j-admin-tools/src/main/resources/docker-compose.yaml +++ b/tools/neo4j-admin-tools/src/main/resources/docker-compose.yaml @@ -3,7 +3,7 @@ version: "3.7" services: neo4j: container_name: "neo4j" - image: neo4j:3.4.4-enterprise + image: neo4j:3.4.7-enterprise environment: - NEO4J_AUTH=none - NEO4J_ACCEPT_LICENSE_AGREEMENT=yes