From aa9d8fdc9d939b67394b2bff138afea1afa4369d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 24 Aug 2018 08:57:35 +0200 Subject: [PATCH 1/2] Add a bit of coding standard for how to avoid the issue we saw in the init shellscript for starting prime yesterday. (#226) --- CONTRIBUTING.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 18371448a..e4563d25b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -114,6 +114,20 @@ by the github repository. It should all be open. * [Greg's wiki about shell scripting is a very good resource](http://mywiki.wooledge.org/) * For complex output (multiple lines etc.), consider using "printf" instead of the simple "echo". +* Scripts, in particular scripts that are run during startup of containers + either for production or testing, should die cleanly and visibly as soon as + possible if they have to. For instance, if the script depends on + an executable that may be missing, the script should test for the + existance of the executable before any of the script's "payload" is + executed. Example: + + DEPENDENCIES="foo bar baz gazonk" + for dep in $DEPENDENCIES; do + if [[ -z "$(type $dep)" ]] ; then + echo "Cannot locate executable '${dep}', bailing out" + exit 1 + fi + done ## Kotlin From fc8001e45ad9bb7831e93f3ac2297930783741f0 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Fri, 24 Aug 2018 13:13:30 +0200 Subject: [PATCH 2/2] Merged pseudonym-server into prime as prime-module --- acceptance-tests/script/wait.sh | 8 - .../common/{Payment.kt => StripePayment.kt} | 15 +- .../kotlin/org/ostelco/at/jersey/Tests.kt | 6 +- .../kotlin/org/ostelco/at/okhttp/Tests.kt | 6 +- .../client/api/ClientApiConfiguration.kt | 10 - .../prime/client/api/ClientApiModule.kt | 5 +- .../client/api/resources/ProfileResource.kt | 4 - .../api/resources/SubscriptionResource.kt | 12 +- .../prime/client/api/store/SubscriberDAO.kt | 4 +- .../client/api/store/SubscriberDAOImpl.kt | 9 + .../api/resources/SubscriptionResourceTest.kt | 32 +- docker-compose.override.yaml | 22 +- payment-processor/build.gradle | 3 +- .../org.ostelco.prime.module.PrimeModule | 2 +- .../pseudonymizer/PseudonymizerService.kt | 7 + prime/Dockerfile.test | 2 +- prime/build.gradle | 3 +- prime/config/config.yaml | 2 +- prime/config/test.yaml | 3 +- .../integration-tests/resources/config.yaml | 2 +- pseudonym-server/Dockerfile | 19 -- pseudonym-server/build.gradle | 54 +--- pseudonym-server/config/.gitignore | 1 - pseudonym-server/config/config.yaml | 7 - pseudonym-server/script/start.sh | 6 - pseudonym-server/script/wait_for_emulators.sh | 44 --- .../ostelco/pseudonym/PseudonymServerTest.kt | 43 --- .../integration-test/resources/config.yaml | 7 - .../kotlin/org/ostelco/pseudonym/Model.kt | 13 + .../org/ostelco/pseudonym/PseudonymModule.kt | 34 ++ .../pseudonym/PseudonymServerApplication.kt | 100 ------ .../pseudonym/config/PseudonymServerConfig.kt | 19 -- .../pseudonym/managed/MessageProcessor.kt | 175 ---------- .../pseudonym/managed/PseudonymExport.kt | 60 ++-- .../pseudonym/resources/PseudonymResource.kt | 186 +---------- .../service/PseudonymizerServiceSingleton.kt | 227 +++++++++++++ .../org/ostelco/pseudonym/utils/DateUtils.kt | 4 +- .../io.dropwizard.jackson.Discoverable | 1 + .../org.ostelco.prime.module.PrimeModule | 1 + ...o.prime.pseudonymizer.PseudonymizerService | 1 + .../org/ostelco/pseudonym/DateUtilsTest.kt | 4 +- .../pseudonym/PseudonymResourceTest.kt | 299 ++++++++++-------- 42 files changed, 563 insertions(+), 899 deletions(-) rename acceptance-tests/src/main/kotlin/org/ostelco/at/common/{Payment.kt => StripePayment.kt} (56%) create mode 100644 prime-api/src/main/kotlin/org/ostelco/prime/pseudonymizer/PseudonymizerService.kt delete mode 100644 pseudonym-server/Dockerfile delete mode 100644 pseudonym-server/config/.gitignore delete mode 100644 pseudonym-server/config/config.yaml delete mode 100755 pseudonym-server/script/start.sh delete mode 100755 pseudonym-server/script/wait_for_emulators.sh delete mode 100644 pseudonym-server/src/integration-test/kotlin/org/ostelco/pseudonym/PseudonymServerTest.kt delete mode 100644 pseudonym-server/src/integration-test/resources/config.yaml create mode 100644 pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/Model.kt create mode 100644 pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/PseudonymModule.kt delete mode 100644 pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/PseudonymServerApplication.kt delete mode 100644 pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/config/PseudonymServerConfig.kt delete mode 100644 pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/managed/MessageProcessor.kt create mode 100644 pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/service/PseudonymizerServiceSingleton.kt create mode 100644 pseudonym-server/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable create mode 100644 pseudonym-server/src/main/resources/META-INF/services/org.ostelco.prime.module.PrimeModule create mode 100644 pseudonym-server/src/main/resources/META-INF/services/org.ostelco.prime.pseudonymizer.PseudonymizerService diff --git a/acceptance-tests/script/wait.sh b/acceptance-tests/script/wait.sh index 7ace4de7d..6a86cecfb 100755 --- a/acceptance-tests/script/wait.sh +++ b/acceptance-tests/script/wait.sh @@ -18,14 +18,6 @@ done echo "Prime launched" -echo "Waiting for pseudonym-server to launch on pseudonym-server:8080..." - -while ! nc -z pseudonym-server 8080; do - sleep 0.1 # wait for 1/10 of the second before check again -done - -echo "pseudonym-server launched" - java -cp '/acceptance-tests.jar' org.junit.runner.JUnitCore \ org.ostelco.at.okhttp.GetPseudonymsTest \ org.ostelco.at.okhttp.GetProductsTest \ diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/common/Payment.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt similarity index 56% rename from acceptance-tests/src/main/kotlin/org/ostelco/at/common/Payment.kt rename to acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt index cdb27950e..42602b971 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/common/Payment.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt @@ -1,9 +1,10 @@ package org.ostelco.at.common import com.stripe.Stripe +import com.stripe.model.Customer import com.stripe.model.Token -object Payment { +object StripePayment { fun createPaymentSourceId(): String { // https://stripe.com/docs/api/java#create_card_token @@ -19,4 +20,16 @@ object Payment { val token = Token.create(tokenMap) return token.id } + + fun deleteAllCustomers() { + // https://stripe.com/docs/api/java#create_card_token + Stripe.apiKey = System.getenv("STRIPE_API_KEY") + + do { + val customers = Customer.list(emptyMap()).data + customers.forEach { customer -> + customer.delete() + } + } while (customers.isNotEmpty()) + } } \ No newline at end of file diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt index c09cfe832..91186ed16 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt @@ -1,7 +1,7 @@ package org.ostelco.at.jersey import org.junit.Test -import org.ostelco.at.common.Payment.createPaymentSourceId +import org.ostelco.at.common.StripePayment import org.ostelco.at.common.createProfile import org.ostelco.at.common.createSubscription import org.ostelco.at.common.expectedProducts @@ -228,6 +228,8 @@ class PurchaseTest { @Test fun `jersey test - POST products purchase`() { + StripePayment.deleteAllCustomers() + val email = "purchase-${randomInt()}@test.com" createProfile(name = "Test Purchase User", email = email) @@ -238,7 +240,7 @@ class PurchaseTest { val balanceBefore = subscriptionStatusBefore.remaining val productSku = "1GB_249NOK" - val sourceId = createPaymentSourceId() + val sourceId = StripePayment.createPaymentSourceId() post { path = "/products/$productSku/purchase" diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt index d572624e8..673645bad 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt @@ -1,7 +1,7 @@ package org.ostelco.at.okhttp import org.junit.Test -import org.ostelco.at.common.Payment.createPaymentSourceId +import org.ostelco.at.common.StripePayment import org.ostelco.at.common.createProfile import org.ostelco.at.common.createSubscription import org.ostelco.at.common.expectedProducts @@ -168,6 +168,8 @@ class PurchaseTest { @Test fun `okhttp test - POST products purchase`() { + StripePayment.deleteAllCustomers() + val email = "purchase-${randomInt()}@test.com" createProfile(name = "Test Purchase User", email = email) @@ -175,7 +177,7 @@ class PurchaseTest { val balanceBefore = client.subscriptionStatus.remaining - val sourceId = createPaymentSourceId() + val sourceId = StripePayment.createPaymentSourceId() client.purchaseProduct("1GB_249NOK", sourceId, false) diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/ClientApiConfiguration.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/ClientApiConfiguration.kt index 1208e5276..bcfe43ef7 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/ClientApiConfiguration.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/ClientApiConfiguration.kt @@ -23,14 +23,4 @@ class ClientApiConfiguration { fun setAuthenticationCachePolicy(spec: String) { this.authenticationCachePolicy = CacheBuilderSpec.parse(spec) } - - @NotNull - var pseudonymEndpoint: String? = null - // TODO vihang: make @NotBlank or @NotEmpty work again - set(value) { - if (value == null || value.isBlank()) { - throw Error("modules.type['api'].config.pseudonymEndpoint is blank") - } - field = value - } } diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/ClientApiModule.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/ClientApiModule.kt index b85210b36..fc2b3ed6e 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/ClientApiModule.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/ClientApiModule.kt @@ -62,10 +62,7 @@ class ClientApiModule : PrimeModule { jerseyEnv.register(ProfileResource(dao)) jerseyEnv.register(ReferralResource(dao)) jerseyEnv.register(PaymentResource(dao)) - jerseyEnv.register(SubscriptionResource( - dao = dao, - pseudonymEndpoint = config.pseudonymEndpoint ?: "", // this will never be empty - client = client)) + jerseyEnv.register(SubscriptionResource(dao)) jerseyEnv.register(SubscriptionsResource(dao)) jerseyEnv.register(ApplicationTokenResource(dao)) diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/resources/ProfileResource.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/resources/ProfileResource.kt index 4ac525549..2ad00b7ad 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/resources/ProfileResource.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/resources/ProfileResource.kt @@ -4,8 +4,6 @@ import io.dropwizard.auth.Auth import org.ostelco.prime.client.api.auth.AccessTokenPrincipal import org.ostelco.prime.client.api.store.SubscriberDAO import org.ostelco.prime.model.Subscriber -import org.ostelco.prime.module.getResource -import org.ostelco.prime.paymentprocessor.PaymentProcessor import javax.validation.constraints.NotNull import javax.ws.rs.Consumes import javax.ws.rs.GET @@ -23,8 +21,6 @@ import javax.ws.rs.core.Response @Path("/profile") class ProfileResource(private val dao: SubscriberDAO) { - private val paymentProcessor by lazy { getResource() } - @GET @Produces("application/json") fun getProfile(@Auth token: AccessTokenPrincipal?): Response { diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/resources/SubscriptionResource.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/resources/SubscriptionResource.kt index c82935a4c..87e7256f4 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/resources/SubscriptionResource.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/resources/SubscriptionResource.kt @@ -6,7 +6,6 @@ import org.ostelco.prime.client.api.store.SubscriberDAO import javax.ws.rs.GET import javax.ws.rs.Path import javax.ws.rs.Produces -import javax.ws.rs.client.Client import javax.ws.rs.core.Response /** @@ -16,9 +15,7 @@ import javax.ws.rs.core.Response @Path("/subscription") @Deprecated("use SubscriptionsResource", ReplaceWith("SubscriptionsResource", "org.ostelco.prime.client.api.resources.SubscriptionsResource")) -class SubscriptionResource(private val dao: SubscriberDAO, - val client: Client, - private val pseudonymEndpoint: String) { +class SubscriptionResource(private val dao: SubscriberDAO) { @GET @Path("status") @@ -44,9 +41,10 @@ class SubscriptionResource(private val dao: SubscriberDAO, .build() } - return dao.getMsisdn(token.name).fold( - { apiError -> Response.status(apiError.status).entity(asJson(apiError.description)).build() }, - { msisdn -> client.target("$pseudonymEndpoint/pseudonym/active/$msisdn").request().get() }) + return dao.getActivePseudonymOfMsisdnForSubscriber(token.name).fold( + { apiError -> Response.status(apiError.status).entity(asJson(apiError.description)) }, + { pseudonym -> Response.status(Response.Status.OK).entity(pseudonym) }) + .build() } } diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAO.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAO.kt index 0975c490d..45bf42fb5 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAO.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAO.kt @@ -5,6 +5,7 @@ import org.ostelco.prime.client.api.model.Consent import org.ostelco.prime.client.api.model.Person import org.ostelco.prime.client.api.model.SubscriptionStatus import org.ostelco.prime.core.ApiError +import org.ostelco.prime.model.ActivePseudonyms import org.ostelco.prime.model.ApplicationToken import org.ostelco.prime.model.Product import org.ostelco.prime.model.PurchaseRecord @@ -13,7 +14,6 @@ import org.ostelco.prime.model.Subscription import org.ostelco.prime.paymentprocessor.core.ProductInfo import org.ostelco.prime.paymentprocessor.core.ProfileInfo import org.ostelco.prime.paymentprocessor.core.SourceInfo -import javax.ws.rs.core.Response /** * @@ -90,4 +90,6 @@ interface SubscriberDAO { @Deprecated(message = "use purchaseProduct") fun purchaseProductWithoutPayment(subscriberId: String, sku: String): Either + + fun getActivePseudonymOfMsisdnForSubscriber(subscriberId: String): Either } diff --git a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt index 34d933132..7ae1ba76b 100644 --- a/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt +++ b/client-api/src/main/kotlin/org/ostelco/prime/client/api/store/SubscriberDAOImpl.kt @@ -13,6 +13,7 @@ import org.ostelco.prime.core.ForbiddenError import org.ostelco.prime.core.InsuffientStorageError import org.ostelco.prime.core.NotFoundError import org.ostelco.prime.logger +import org.ostelco.prime.model.ActivePseudonyms import org.ostelco.prime.model.ApplicationToken import org.ostelco.prime.model.Product import org.ostelco.prime.model.PurchaseRecord @@ -24,6 +25,7 @@ import org.ostelco.prime.paymentprocessor.PaymentProcessor import org.ostelco.prime.paymentprocessor.core.ProductInfo import org.ostelco.prime.paymentprocessor.core.ProfileInfo import org.ostelco.prime.paymentprocessor.core.SourceInfo +import org.ostelco.prime.pseudonymizer.PseudonymizerService import org.ostelco.prime.storage.ClientDataSource import java.time.Instant import java.util.* @@ -37,6 +39,7 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu private val logger by logger() private val paymentProcessor by lazy { getResource() } + private val pseudonymizer by lazy { getResource() } /* Table for 'profiles'. */ private val consentMap = ConcurrentHashMap>() @@ -135,6 +138,12 @@ class SubscriberDAOImpl(private val storage: ClientDataSource, private val ocsSu } } + override fun getActivePseudonymOfMsisdnForSubscriber(subscriberId: String): Either { + return storage.getMsisdn(subscriberId) + .mapLeft { NotFoundError("Failed to msisdn for user. ${it.message}") } + .map { msisdn -> pseudonymizer.getActivePseudonymsForMsisdn(msisdn) } + } + override fun getPurchaseHistory(subscriberId: String): Either> { return try { return storage.getPurchaseRecords(subscriberId).bimap( diff --git a/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/SubscriptionResourceTest.kt b/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/SubscriptionResourceTest.kt index d1c33978c..9192b1cf7 100644 --- a/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/SubscriptionResourceTest.kt +++ b/client-api/src/test/kotlin/org/ostelco/prime/client/api/resources/SubscriptionResourceTest.kt @@ -21,7 +21,6 @@ import org.ostelco.prime.client.api.auth.OAuthAuthenticator import org.ostelco.prime.client.api.model.SubscriptionStatus import org.ostelco.prime.client.api.store.SubscriberDAO import org.ostelco.prime.client.api.util.AccessToken -import org.ostelco.prime.core.ApiError import org.ostelco.prime.model.ActivePseudonyms import org.ostelco.prime.model.Price import org.ostelco.prime.model.Product @@ -29,9 +28,6 @@ import org.ostelco.prime.model.PseudonymEntity import org.ostelco.prime.model.PurchaseRecord import java.time.Instant import java.util.* -import javax.ws.rs.client.Client -import javax.ws.rs.client.Invocation -import javax.ws.rs.client.WebTarget import javax.ws.rs.core.MediaType import javax.ws.rs.core.Response @@ -49,21 +45,18 @@ class SubscriptionResourceTest { product = Product(sku = "1", price = Price(10, "NOK")), timestamp = Instant.now().toEpochMilli())) - private val subscriptionStatus = SubscriptionStatus(5, purchaseRecords) - @Before - @Throws(Exception::class) fun setUp() { `when`(AUTHENTICATOR.authenticate(ArgumentMatchers.anyString())) .thenReturn(Optional.of(AccessTokenPrincipal(email))) } @Test - @Throws(Exception::class) fun getSubscriptionStatus() { + val subscriptionStatus = SubscriptionStatus(5, purchaseRecords) val arg = argumentCaptor() - `when`>(DAO.getSubscriptionStatus(arg.capture())).thenReturn(Either.right(subscriptionStatus)) + `when`(DAO.getSubscriptionStatus(arg.capture())).thenReturn(Either.right(subscriptionStatus)) val resp = RULE.target("/subscription/status") .request() @@ -79,22 +72,17 @@ class SubscriptionResourceTest { } @Test - @Throws(Exception::class) fun getActivePseudonyms() { val arg = argumentCaptor() + val msisdn = "4790300001" - val url = "${PSEUDONYMENDPOINT}/pseudonym/active/$msisdn" val pseudonym = PseudonymEntity(msisdn, "random", 0, 1) val activePseudonyms = ActivePseudonyms(pseudonym, pseudonym) - val responseJsonString = ObjectMapper().writeValueAsString(activePseudonyms) - val response = Response.status(Response.Status.OK) - .entity(responseJsonString) - .build() - `when`>(DAO.getMsisdn(arg.capture())).thenReturn(Either.right(msisdn)) - `when`(client.target(url)).thenReturn(target) - `when`(target.request()).thenReturn(request) - `when`(request.get()).thenReturn(response) + `when`(DAO.getActivePseudonymOfMsisdnForSubscriber(arg.capture())) + .thenReturn(Either.right(activePseudonyms)) + + val responseJsonString = ObjectMapper().writeValueAsString(activePseudonyms) val resp = RULE.target("/subscription/activePseudonyms") .request() @@ -111,10 +99,6 @@ class SubscriptionResourceTest { val DAO: SubscriberDAO = mock(SubscriberDAO::class.java) val AUTHENTICATOR: OAuthAuthenticator = mock(OAuthAuthenticator::class.java) - val PSEUDONYMENDPOINT = "http://localhost" - val client: Client = mock(Client::class.java) - val target: WebTarget = mock(WebTarget::class.java) - val request: Invocation.Builder = mock(Invocation.Builder::class.java) @JvmField @ClassRule @@ -125,7 +109,7 @@ class SubscriptionResourceTest { .setPrefix("Bearer") .buildAuthFilter())) .addResource(AuthValueFactoryProvider.Binder(AccessTokenPrincipal::class.java)) - .addResource(SubscriptionResource(DAO, client, PSEUDONYMENDPOINT)) + .addResource(SubscriptionResource(DAO)) .setTestContainerFactory(GrizzlyWebTestContainerFactory()) .build() } diff --git a/docker-compose.override.yaml b/docker-compose.override.yaml index 71740f6c3..75b71a471 100644 --- a/docker-compose.override.yaml +++ b/docker-compose.override.yaml @@ -12,9 +12,11 @@ services: - PUBSUB_EMULATOR_HOST=pubsub-emulator:8085 - PUBSUB_PROJECT_ID=pantel-2decb - STRIPE_API_KEY=${STRIPE_API_KEY} + - DATASTORE_EMULATOR_HOST=localhost:9090 + - DATASTORE_PROJECT_ID=pantel-2decb + - LOCAL_TESTING=true ports: - "9090:8080" -# - "7687:7687" depends_on: - "pubsub-emulator" - "neo4j" @@ -73,19 +75,6 @@ services: ipv4_address: 172.16.238.2 default: - pseudonym-server: - container_name: pseudonym-server - build: pseudonym-server - ports: - - "8090:8080" - environment: - - DATASTORE_EMULATOR_HOST=localhost:9090 - - DATASTORE_PROJECT_ID=pantel-2decb - - PUBSUB_EMULATOR_HOST=localhost:9080 - - PUBSUB_PROJECT_ID=pantel-2decb - - LOCAL_TESTING=true - command: ["/bin/bash", "./wait_for_emulators.sh"] - neo4j: container_name: "neo4j" image: neo4j:3.4.4 @@ -100,11 +89,6 @@ services: container_name: pubsub-emulator image: knarz/pubsub-emulator - gpubsub-emulator: - container_name: gpubsub-emulator - image: google/cloud-sdk - command: ["gcloud", "beta", "emulators", "pubsub", "start", "--host-port=0.0.0.0:8085"] - datastore-emulator: container_name: datastore-emulator image: google/cloud-sdk diff --git a/payment-processor/build.gradle b/payment-processor/build.gradle index 799aef9fa..7d322d658 100644 --- a/payment-processor/build.gradle +++ b/payment-processor/build.gradle @@ -8,7 +8,6 @@ sourceSets { test { java.srcDirs = ['src/test/kotlin'] } - integration { java.srcDirs = ['src/test/kotlin', 'src/integration-tests/kotlin'] resources.srcDir 'src/integration-tests/resources' @@ -53,6 +52,8 @@ tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).all { } } +apply from: '../jacoco.gradle' + idea { module { testSourceDirs += file('src/integration-tests/kotlin') diff --git a/payment-processor/src/main/resources/META-INF/services/org.ostelco.prime.module.PrimeModule b/payment-processor/src/main/resources/META-INF/services/org.ostelco.prime.module.PrimeModule index 74ceaf0a7..d45eee5b2 100644 --- a/payment-processor/src/main/resources/META-INF/services/org.ostelco.prime.module.PrimeModule +++ b/payment-processor/src/main/resources/META-INF/services/org.ostelco.prime.module.PrimeModule @@ -1 +1 @@ - org.ostelco.prime.paymentprocessor.PaymentProcessorModule +org.ostelco.prime.paymentprocessor.PaymentProcessorModule diff --git a/prime-api/src/main/kotlin/org/ostelco/prime/pseudonymizer/PseudonymizerService.kt b/prime-api/src/main/kotlin/org/ostelco/prime/pseudonymizer/PseudonymizerService.kt new file mode 100644 index 000000000..721d50c6f --- /dev/null +++ b/prime-api/src/main/kotlin/org/ostelco/prime/pseudonymizer/PseudonymizerService.kt @@ -0,0 +1,7 @@ +package org.ostelco.prime.pseudonymizer + +import org.ostelco.prime.model.ActivePseudonyms + +interface PseudonymizerService { + fun getActivePseudonymsForMsisdn(msisdn: String): ActivePseudonyms +} \ No newline at end of file diff --git a/prime/Dockerfile.test b/prime/Dockerfile.test index 773b23f0c..06e6201a9 100644 --- a/prime/Dockerfile.test +++ b/prime/Dockerfile.test @@ -5,7 +5,7 @@ FROM openjdk:8u171 MAINTAINER CSI "csi@telenordigital.com" RUN apt-get update \ - && apt-get install -y --no-install-recommends netcat \ + && apt-get install -y --no-install-recommends netcat socat=1.7.3.1-2+deb9u1 \ && rm -rf /var/lib/apt/lists/* COPY script/start.sh /start.sh diff --git a/prime/build.gradle b/prime/build.gradle index 6178de08b..b7f89834f 100644 --- a/prime/build.gradle +++ b/prime/build.gradle @@ -18,7 +18,7 @@ sourceSets { } } -version = "1.11.0" +version = "1.12.0" repositories { maven { @@ -34,6 +34,7 @@ dependencies { runtimeOnly project(':ocs') runtimeOnly project(':firebase-store') runtimeOnly project(':neo4j-store') + runtimeOnly project(':pseudonym-server') runtimeOnly project(':client-api') runtimeOnly project(':admin-api') runtimeOnly project(':app-notifier') diff --git a/prime/config/config.yaml b/prime/config/config.yaml index cd52ffb82..3a074cbdc 100644 --- a/prime/config/config.yaml +++ b/prime/config/config.yaml @@ -13,10 +13,10 @@ modules: projectId: pantel-2decb topicId: data-traffic lowBalanceThreshold: 100000000 + - type: pseudonymizer - type: api config: authenticationCachePolicy: maximumSize=10000, expireAfterAccess=10m - pseudonymEndpoint: http://pseudonym-server-service.default.svc.cluster.local jerseyClient: timeout: 2s - type: stripe-payment-processor diff --git a/prime/config/test.yaml b/prime/config/test.yaml index efa68475e..808edcdda 100644 --- a/prime/config/test.yaml +++ b/prime/config/test.yaml @@ -15,12 +15,13 @@ modules: projectId: pantel-2decb topicId: data-traffic lowBalanceThreshold: 0 + - type: pseudonymizer - type: api config: authenticationCachePolicy: maximumSize=10000, expireAfterAccess=10m - pseudonymEndpoint: http://pseudonym-server:8080 jerseyClient: timeout: 3s + connectionRequestTimeout: 1s - type: stripe-payment-processor - type: firebase-app-notifier config: diff --git a/prime/src/integration-tests/resources/config.yaml b/prime/src/integration-tests/resources/config.yaml index 7d593c8d5..a41bb9fdd 100644 --- a/prime/src/integration-tests/resources/config.yaml +++ b/prime/src/integration-tests/resources/config.yaml @@ -13,10 +13,10 @@ modules: projectId: pantel-2decb topicId: data-traffic lowBalanceThreshold: 0 + - type: pseudonymizer - type: api config: authenticationCachePolicy: maximumSize=10000, expireAfterAccess=10m - pseudonymEndpoint: http://pseudonym-server:8080 jerseyClient: timeout: 3s - type: stripe-payment-processor diff --git a/pseudonym-server/Dockerfile b/pseudonym-server/Dockerfile deleted file mode 100644 index c2862c2f6..000000000 --- a/pseudonym-server/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM openjdk:8u171 - -MAINTAINER CSI "csi@telenordigital.com" - -RUN apt-get update && apt-get install -y --no-install-recommends \ - socat=1.7.3.1-2+deb9u1 \ - && rm -rf /var/lib/apt/lists/* - -COPY script/start.sh /start.sh -COPY script/wait_for_emulators.sh /wait_for_emulators.sh - -COPY config /config - -COPY build/libs/pseudonym-server-uber.jar /pseudonym-server.jar - -EXPOSE 8080 -EXPOSE 8081 - -CMD ["/start.sh"] \ No newline at end of file diff --git a/pseudonym-server/build.gradle b/pseudonym-server/build.gradle index 3b17980cd..690557280 100644 --- a/pseudonym-server/build.gradle +++ b/pseudonym-server/build.gradle @@ -1,15 +1,16 @@ plugins { id "org.jetbrains.kotlin.jvm" version "1.2.61" - id "application" - id "com.github.johnrengelman.shadow" version "2.0.4" - id "idea" + id "java-library" } -version = "1.6.0" + dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion" - implementation "io.dropwizard:dropwizard-core:$dropwizardVersion" - implementation "io.dropwizard:dropwizard-client:$dropwizardVersion" + implementation project(':ocs-api') + implementation project(':prime-api') + implementation project(':model') + + implementation "io.dropwizard:dropwizard-client:$dropwizardVersion" implementation 'com.google.guava:guava:25.1-jre' // Match with grpc-netty-shaded via PubSub // removing io.grpc:grpc-netty-shaded:1.13.1 causes ALPN error @@ -19,49 +20,10 @@ dependencies { implementation "com.google.cloud:google-cloud-pubsub:$googleCloudVersion" implementation "com.fasterxml.jackson.module:jackson-module-kotlin:$jacksonVersion" - implementation project(':model') - testImplementation "io.dropwizard:dropwizard-testing:$dropwizardVersion" testImplementation "org.jetbrains.kotlin:kotlin-test-junit:$kotlinVersion" - testCompile group: 'org.mockito', name: 'mockito-all', version: '1.10.19' + testImplementation "org.mockito:mockito-all:1.10.19" testRuntimeOnly 'org.hamcrest:hamcrest-all:1.3' } -shadowJar { - mainClassName = 'org.ostelco.pseudonym.PseudonymServerApplicationKt' - mergeServiceFiles() - classifier = "uber" - version = null -} - -sourceSets { - integrationTest { - kotlin { - compileClasspath += main.output + test.output - runtimeClasspath += main.output + test.output - srcDirs += file('src/integration-test/kotlin') - } - resources.srcDir file('src/integration-test/resources') - } -} - -configurations { - integrationTestImplementation.extendsFrom testImplementation - integrationTestCompile.extendsFrom testCompile - integrationTestRuntime.extendsFrom testRuntime -} - -task integrationTest(type: Test, description: 'Runs the integration tests.', group: 'Verification') { - testClassesDirs = sourceSets.integrationTest.output.classesDirs - classpath = sourceSets.integrationTest.runtimeClasspath -} - -integrationTest.mustRunAfter test - apply from: '../jacoco.gradle' - -idea { - module { - testSourceDirs += file('src/integration-test/kotlin') - } -} \ No newline at end of file diff --git a/pseudonym-server/config/.gitignore b/pseudonym-server/config/.gitignore deleted file mode 100644 index bf045303f..000000000 --- a/pseudonym-server/config/.gitignore +++ /dev/null @@ -1 +0,0 @@ -pantel-prod.json \ No newline at end of file diff --git a/pseudonym-server/config/config.yaml b/pseudonym-server/config/config.yaml deleted file mode 100644 index 36c72f81e..000000000 --- a/pseudonym-server/config/config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -serviceAccountKey: /config/pantel-prod.json -databaseName: pantel-2decb - -logging: - level: INFO - loggers: - org.ostelco: DEBUG diff --git a/pseudonym-server/script/start.sh b/pseudonym-server/script/start.sh deleted file mode 100755 index 68e64bade..000000000 --- a/pseudonym-server/script/start.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# Start app -exec java \ - -Dfile.encoding=UTF-8 \ - -jar /pseudonym-server.jar server /config/config.yaml diff --git a/pseudonym-server/script/wait_for_emulators.sh b/pseudonym-server/script/wait_for_emulators.sh deleted file mode 100755 index 96e62ea43..000000000 --- a/pseudonym-server/script/wait_for_emulators.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -set -e - -echo "pseudonym-server waiting Datastore emulator to launch on datastore-emulator:8081..." - -ds=$(curl --silent http://datastore-emulator:8081 | head -n1) -until [[ $ds == 'Ok' ]] ; do - printf 'pseudonymiser waiting ...' - sleep 5 - ds=$(curl --silent http://datastore-emulator:8081 | head -n1) -done - -echo "Datastore emulator launched" - -echo "pseudonym-server waiting pubsub emulator to launch on gpubsub-emulator:8085..." - -ds=$(curl --silent http://gpubsub-emulator:8085 | head -n1) -until [[ $ds == 'Ok' ]] ; do - printf 'pseudonymiser waiting ...' - sleep 5 - ds=$(curl --silent http://gpubsub-emulator:8085 | head -n1) -done - -echo "Pubsub emulator launched" - -echo "Creating topics and subscriptions...." - -curl -X PUT gpubsub-emulator:8085/v1/projects/pantel-2decb/topics/data-traffic -curl -X PUT gpubsub-emulator:8085/v1/projects/pantel-2decb/topics/pseudo-traffic -curl -X PUT -H "Content-Type: application/json" -d '{"topic":"projects/pantel-2decb/topics/data-traffic","ackDeadlineSeconds":10}' gpubsub-emulator:8085/v1/projects/pantel-2decb/subscriptions/test-pseudo - -echo "Done creating topics and subscriptions" - -# Forward the local port 9090 to datastore-emulator:8081 -# The -socat TCP-LISTEN:9090,fork TCP:datastore-emulator:8081 & -socat TCP-LISTEN:9080,fork TCP:gpubsub-emulator:8085 & - -# Start app -exec java \ - -Dfile.encoding=UTF-8 \ - -jar /pseudonym-server.jar server /config/config.yaml - diff --git a/pseudonym-server/src/integration-test/kotlin/org/ostelco/pseudonym/PseudonymServerTest.kt b/pseudonym-server/src/integration-test/kotlin/org/ostelco/pseudonym/PseudonymServerTest.kt deleted file mode 100644 index cd3f4eadf..000000000 --- a/pseudonym-server/src/integration-test/kotlin/org/ostelco/pseudonym/PseudonymServerTest.kt +++ /dev/null @@ -1,43 +0,0 @@ -package org.ostelco.pseudonym - -import com.google.gson.JsonParser -import io.dropwizard.testing.ConfigOverride -import io.dropwizard.testing.ResourceHelpers -import io.dropwizard.testing.junit.DropwizardAppRule -import org.glassfish.jersey.client.JerseyClientBuilder -import org.junit.ClassRule -import org.junit.Test -import java.util.* -import kotlin.test.assertEquals - -/** - * Class to do integration testing of pseudonymiser. - */ -class PseudonymServerTest { - - private val msisdn = "4790303333" - companion object { - - @JvmField - @ClassRule - val RULE = DropwizardAppRule( - PseudonymServerApplication::class.java, - ResourceHelpers.resourceFilePath("config.yaml"), - ConfigOverride.config("datastoreType", "inmemory-emulator")) - } - - /** - * Test a normal request - */ - @Test - fun testPseudonymServer() { - - println("testPseudonymServer") - val response = JerseyClientBuilder().build() - ?.target("http://0.0.0.0:${RULE.getLocalPort()}/pseudonym/current/${msisdn}") - ?.request() - ?.get() - assertEquals(200, response?.status) - - } -} diff --git a/pseudonym-server/src/integration-test/resources/config.yaml b/pseudonym-server/src/integration-test/resources/config.yaml deleted file mode 100644 index c0c01688c..000000000 --- a/pseudonym-server/src/integration-test/resources/config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -serviceAccountKey: config/pantel-prod.json -databaseName: pantel-2decb - -logging: - level: INFO - loggers: - org.ostelco: DEBUG \ No newline at end of file diff --git a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/Model.kt b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/Model.kt new file mode 100644 index 000000000..81d6334f1 --- /dev/null +++ b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/Model.kt @@ -0,0 +1,13 @@ +package org.ostelco.pseudonym + +const val PseudonymEntityKind = "Pseudonym" +const val msisdnPropertyName = "msisdn" +const val pseudonymPropertyName = "pseudonym" +const val startPropertyName = "start" +const val endPropertyName = "end" + +const val ExportTaskKind = "ExportTask" +const val exportIdPropertyName = "exportId" +const val statusPropertyName = "status" +const val errorPropertyName = "error" + diff --git a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/PseudonymModule.kt b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/PseudonymModule.kt new file mode 100644 index 000000000..762e270c7 --- /dev/null +++ b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/PseudonymModule.kt @@ -0,0 +1,34 @@ +package org.ostelco.pseudonym + +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.annotation.JsonTypeName +import io.dropwizard.Configuration +import io.dropwizard.setup.Environment +import org.ostelco.prime.module.PrimeModule +import org.ostelco.pseudonym.resources.PseudonymResource +import org.ostelco.pseudonym.service.PseudonymizerServiceSingleton + +@JsonTypeName("pseudonymizer") +class PseudonymModule : PrimeModule { + + @JsonProperty + fun setConfig(config: PseudonymServerConfig) { + ConfigRegistry.config = config + } + + override fun init(env: Environment) { + PseudonymizerServiceSingleton.init() + env.jersey().register(PseudonymResource()) + } +} + +object ConfigRegistry { + var config = PseudonymServerConfig() +} + +/** + * The configuration for Pseudonymiser. + */ +class PseudonymServerConfig : Configuration() { + var datastoreType = "default" +} \ No newline at end of file diff --git a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/PseudonymServerApplication.kt b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/PseudonymServerApplication.kt deleted file mode 100644 index bfe91e11f..000000000 --- a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/PseudonymServerApplication.kt +++ /dev/null @@ -1,100 +0,0 @@ -package org.ostelco.pseudonym - -import com.google.cloud.bigquery.BigQuery -import com.google.cloud.bigquery.BigQueryOptions -import com.google.cloud.datastore.Datastore -import com.google.cloud.datastore.DatastoreOptions -import com.google.cloud.datastore.testing.LocalDatastoreHelper -import com.google.pubsub.v1.ProjectSubscriptionName -import com.google.pubsub.v1.ProjectTopicName -import io.dropwizard.Application -import io.dropwizard.client.JerseyClientBuilder -import io.dropwizard.jetty.HttpConnectorFactory -import io.dropwizard.server.DefaultServerFactory -import io.dropwizard.setup.Environment -import org.glassfish.jersey.client.ClientProperties -import org.ostelco.pseudonym.config.PseudonymServerConfig -import org.ostelco.pseudonym.managed.MessageProcessor -import org.ostelco.pseudonym.resources.PseudonymResource -import org.ostelco.pseudonym.utils.WeeklyBounds -import org.slf4j.LoggerFactory -import javax.ws.rs.client.Client - - -/** - * Entry point for running the server - */ -fun main(args: Array) { - PseudonymServerApplication().run(*args) -} - -/** - * Dropwizard application for running pseudonymiser service that - * converts Data-Traffic PubSub message to a pseudonymised version. - */ -class PseudonymServerApplication : Application() { - - private val logger = LoggerFactory.getLogger(PseudonymServerApplication::class.java) - - // Find port for the local REST endpoint - private fun getPseudonymEndpoint(config: PseudonymServerConfig): String { - val endpoint = config.pseudonymEndpoint - if (!endpoint.isEmpty()) { - return endpoint - } - var httpPort: Int? = null - val serverFactory = config.getServerFactory() as? DefaultServerFactory - if (serverFactory != null) { - for (connector in serverFactory.applicationConnectors) { - if (connector.javaClass.isAssignableFrom(HttpConnectorFactory::class.java)) { - httpPort = (connector as? HttpConnectorFactory)?.port - break - } - } - } - return "http://localhost:${httpPort ?: 8080}" - } - - // Integration testing helper for Datastore. - private fun getDatastore(config: PseudonymServerConfig): Datastore { - val datastore: Datastore? - if (config.datastoreType == "inmemory-emulator") { - logger.info("Starting with in-memory datastore emulator...") - val helper: LocalDatastoreHelper = LocalDatastoreHelper.create(1.0) - helper.start() - datastore = helper.options.service - } else { - datastore = DatastoreOptions.getDefaultInstance().service - } - return datastore - } - - // Run the dropwizard application (called by the kotlin [main] wrapper). - override fun run( - config: PseudonymServerConfig, - env: Environment) { - val datastore = getDatastore(config) - val client: Client = JerseyClientBuilder(env).using(config.jerseyClient).build(name) - // Increase HTTP timeout values - client.property(ClientProperties.CONNECT_TIMEOUT, 2000) - client.property(ClientProperties.READ_TIMEOUT, 2000) - val subscriptionName = ProjectSubscriptionName.of(config.projectName, config.subscriptionName) - val publisherTopicName = ProjectTopicName.of(config.projectName, config.publisherTopic) - val endpoint = getPseudonymEndpoint(config) - logger.info("Pseudonym endpoint = $endpoint") - val messageProcessor = MessageProcessor(subscriptionName, - publisherTopicName, - endpoint, - WeeklyBounds(), - client) - env.lifecycle().manage(messageProcessor) - - var bigquery: BigQuery? = null - if(System.getenv("LOCAL_TESTING") != "true") { - bigquery = BigQueryOptions.getDefaultInstance().getService() - } else { - logger.info("Local testing, BigQuery is not available...") - } - env.jersey().register(PseudonymResource(datastore, WeeklyBounds(), bigquery)) - } -} \ No newline at end of file diff --git a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/config/PseudonymServerConfig.kt b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/config/PseudonymServerConfig.kt deleted file mode 100644 index b5dad5d33..000000000 --- a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/config/PseudonymServerConfig.kt +++ /dev/null @@ -1,19 +0,0 @@ -package org.ostelco.pseudonym.config - -import io.dropwizard.Configuration -import io.dropwizard.client.JerseyClientConfiguration - -/** - * The configuration for Pseudonymiser. - */ -class PseudonymServerConfig : Configuration() { - - var serviceAccountKey = "" - var databaseName = "" - var datastoreType = "default" - var projectName = "pantel-2decb" - var subscriptionName = "test-pseudo" - var publisherTopic = "pseudo-traffic" - var pseudonymEndpoint = "" - var jerseyClient = JerseyClientConfiguration() -} diff --git a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/managed/MessageProcessor.kt b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/managed/MessageProcessor.kt deleted file mode 100644 index 92355f055..000000000 --- a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/managed/MessageProcessor.kt +++ /dev/null @@ -1,175 +0,0 @@ -package org.ostelco.pseudonym.managed - -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper -import com.fasterxml.jackson.module.kotlin.readValue -import com.google.api.core.ApiFutureCallback -import com.google.api.core.ApiFutures -import com.google.api.gax.core.NoCredentialsProvider -import com.google.api.gax.grpc.GrpcTransportChannel -import com.google.api.gax.rpc.ApiException -import com.google.api.gax.rpc.FixedTransportChannelProvider -import com.google.cloud.pubsub.v1.AckReplyConsumer -import com.google.cloud.pubsub.v1.MessageReceiver -import com.google.cloud.pubsub.v1.Publisher -import com.google.cloud.pubsub.v1.Subscriber -import com.google.common.cache.Cache -import com.google.common.cache.CacheBuilder -import com.google.protobuf.util.Timestamps -import com.google.pubsub.v1.ProjectSubscriptionName -import com.google.pubsub.v1.ProjectTopicName -import com.google.pubsub.v1.PubsubMessage -import io.dropwizard.lifecycle.Managed -import io.grpc.ManagedChannel -import io.grpc.ManagedChannelBuilder -import org.ostelco.ocs.api.DataTrafficInfo -import org.ostelco.prime.model.PseudonymEntity -import org.ostelco.pseudonym.resources.DateBounds -import org.slf4j.LoggerFactory -import java.util.concurrent.Callable -import java.util.concurrent.ExecutionException -import javax.ws.rs.client.Client - - -/** - * This class converts the Plain DataTrafficInfo message to - * a pseudonymous version. Pushes the new message - * to different PubSub topic. - */ - -class MessageProcessor(private val subscriptionName: ProjectSubscriptionName, - private val publisherTopicName: ProjectTopicName, - private val pseudonymEndpoint: String, - private val dateBounds: DateBounds, - private val client: Client) : Managed { - - private val logger = LoggerFactory.getLogger(MessageProcessor::class.java) - private val receiver: MessageReceiver - private var subscriber: Subscriber? = null - private var publisher: Publisher? = null - val mapper = jacksonObjectMapper() - val pseudonymCache: Cache - - // Testing helpers. - val emulatorHost: String? = System.getenv("PUBSUB_EMULATOR_HOST") - var channel: ManagedChannel? = null - - init { - receiver = MessageReceiver { message, consumer -> - handleMessage(message, consumer) - } - pseudonymCache = CacheBuilder.newBuilder() - .maximumSize(5000) - .build() - } - - @Throws(Exception::class) - override fun start() { - logger.info("Starting MessageProcessor...") - if (emulatorHost != null && !emulatorHost.isEmpty()) { - // Setup for picking up emulator settings - // https://cloud.google.com/pubsub/docs/emulator#pubsub-emulator-java - channel = ManagedChannelBuilder.forTarget(emulatorHost).usePlaintext().build() - val channelProvider = FixedTransportChannelProvider.create(GrpcTransportChannel.create(channel)) - val credentialsProvider = NoCredentialsProvider.create() - publisher = Publisher.newBuilder(publisherTopicName) - .setChannelProvider(channelProvider) - .setCredentialsProvider(credentialsProvider) - .build() - subscriber = Subscriber.newBuilder(subscriptionName, receiver) - .setChannelProvider(channelProvider) - .setCredentialsProvider(credentialsProvider) - .build() - - } else { - // Production, connect to real pubsub host - publisher = Publisher.newBuilder(publisherTopicName).build() - subscriber = Subscriber.newBuilder(subscriptionName, receiver).build() - } - subscriber?.startAsync() - } - - @Throws(Exception::class) - override fun stop() { - logger.info("Stopping MessageProcessor...") - channel?.shutdown() - subscriber?.stopAsync() - publisher?.shutdown() - } - - private fun getPseudonymUrl(msisdn: String, timestamp: Long): String { - return "$pseudonymEndpoint/pseudonym/get/$msisdn/$timestamp" - } - - private fun getPseudonymEntity(msisdn: String, timestamp: Long): PseudonymEntity? { - val (_, keyPrefix) = dateBounds.getBoundsNKeyPrefix(msisdn, timestamp) - try { - // Retrieves the element from cache. - // Incase of cache miss, get the entity via a REST call - return pseudonymCache.get(keyPrefix, Callable { - val url = getPseudonymUrl(msisdn, timestamp) - val target = client.target(url) - val response = target.request().get() - if (response.getStatus() != 200) { - val unexpectedResponse = response.readEntity(String::class.java) - logger.warn("$url returned ${response.getStatus()} Response: $unexpectedResponse") - throw javax.ws.rs.ProcessingException(unexpectedResponse) - } - logger.warn("$url returned ${response.getStatus()}") - val json = response.readEntity(String::class.java) - response.close() - mapper.readValue(json) - }) - } catch (e: ExecutionException) { - logger.warn("getPseudonymEntity failed, ${e.toString()}") - } - return null - } - - private fun handleMessage(message: PubsubMessage, consumer: AckReplyConsumer) { - val trafficInfo = DataTrafficInfo.parseFrom(message.data) - // Retrieve the pseudonym for msisdn - val pseudonymEntity = getPseudonymEntity(trafficInfo.msisdn, Timestamps.toMillis(trafficInfo.timestamp)) - if (pseudonymEntity == null) { - logger.error("Error converting DataTrafficInfo message ${message.messageId}") - consumer.nack() - return - } - // New message with pseudonym msisdn - val data = DataTrafficInfo.newBuilder() - .setMsisdn(pseudonymEntity.pseudonym) - .setBucketBytes(trafficInfo.bucketBytes) - .setBundleBytes(trafficInfo.bundleBytes) - .setTimestamp(trafficInfo.timestamp) - .build() - .toByteString() - logger.info("msisdn {}, bucketBytes {}", trafficInfo.msisdn, trafficInfo.bucketBytes) - val pubsubMessage = PubsubMessage.newBuilder() - .setData(data) - .build() - - //schedule a message to be published, messages are automatically batched - val future = publisher?.publish(pubsubMessage) - // add an asynchronous callback to handle success / failure - ApiFutures.addCallback(future, object : ApiFutureCallback { - - override fun onFailure(throwable: Throwable) { - if (throwable is ApiException) { - // details on the API exception - logger.warn("Status code: {}", throwable.statusCode.code) - logger.warn("Retrying: {}", throwable.isRetryable) - } - logger.warn("Error publishing message for msisdn: {}", trafficInfo.msisdn) - consumer.nack() - } - - override fun onSuccess(messageId: String) { - // Once published, returns server-assigned message ids (unique within the topic) - logger.debug(messageId) - logger.info("Processed message $messageId") - consumer.ack() - } - }) - - } -} - diff --git a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/managed/PseudonymExport.kt b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/managed/PseudonymExport.kt index 1cf312888..856464e80 100644 --- a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/managed/PseudonymExport.kt +++ b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/managed/PseudonymExport.kt @@ -16,27 +16,25 @@ import com.google.cloud.datastore.Query import com.google.cloud.datastore.StructuredQuery import com.google.common.cache.Cache import com.google.common.cache.CacheBuilder -import org.ostelco.pseudonym.resources.ExportTaskKind -import org.ostelco.pseudonym.resources.PseudonymEntityKind -import org.ostelco.pseudonym.resources.errorPropertyName -import org.ostelco.pseudonym.resources.exportIdPropertyName -import org.ostelco.pseudonym.resources.msisdnPropertyName -import org.ostelco.pseudonym.resources.pseudonymPropertyName -import org.ostelco.pseudonym.resources.statusPropertyName +import org.ostelco.pseudonym.ExportTaskKind +import org.ostelco.pseudonym.PseudonymEntityKind +import org.ostelco.pseudonym.errorPropertyName +import org.ostelco.pseudonym.exportIdPropertyName +import org.ostelco.pseudonym.msisdnPropertyName +import org.ostelco.pseudonym.pseudonymPropertyName +import org.ostelco.pseudonym.statusPropertyName import org.slf4j.LoggerFactory import java.util.* -import java.util.concurrent.Callable -const val datasetName = "exported_pseudonyms" -const val msisdnFieldName = "msisdn" -const val pseudonymFiledName = "pseudonym" -const val idFieldName = "msisdnid" +private const val datasetName = "exported_pseudonyms" +private const val msisdnFieldName = "msisdn" +private const val pseudonymFiledName = "pseudonym" +private const val idFieldName = "msisdnid" /** * Exports pseudonym objects to a bigquery Table */ - -class PseudonymExport(val exportId: String, val bigquery: BigQuery, val datastore: Datastore) { +class PseudonymExport(private val exportId: String, private val bigquery: BigQuery, private val datastore: Datastore) { private val logger = LoggerFactory.getLogger(PseudonymExport::class.java) /** @@ -46,16 +44,14 @@ class PseudonymExport(val exportId: String, val bigquery: BigQuery, val datastor INITIAL, RUNNING, FINISHED, ERROR } - private val tableName: String - private val idCache: Cache + private val tableName: String = exportId.replace("-", "") + private val idCache: Cache = CacheBuilder.newBuilder() + .maximumSize(5000) + .build() private var status = Status.INITIAL private var error: String = "" init { - tableName = exportId.replace("-", "") - idCache = CacheBuilder.newBuilder() - .maximumSize(5000) - .build() upsertTaskStatus() } @@ -80,9 +76,7 @@ class PseudonymExport(val exportId: String, val bigquery: BigQuery, val datastor private fun getIdForMsisdn(msisdn: String): String { // Retrieves the element from cache. // Incase of cache miss, generate a new UUID - return idCache.get(msisdn, Callable { - UUID.randomUUID().toString() - }) + return idCache.get(msisdn) { UUID.randomUUID().toString() } } private fun createTablePage(pageSize: Int, cursor: Cursor?, table: Table): Cursor? { @@ -112,13 +106,13 @@ class PseudonymExport(val exportId: String, val bigquery: BigQuery, val datastor val response = table.insert(rows, true, true) if (response.hasErrors()) { logger.error("Failed to insert Records", response.insertErrors) - error = "$error${response.insertErrors.toString()}\n" + error = "$error${response.insertErrors}\n" } } - if (totalPseudonyms < pageSize) { - return null + return if (totalPseudonyms < pageSize) { + null } else { - return pseudonyms.getCursorAfter() + pseudonyms.cursorAfter } } @@ -154,12 +148,12 @@ class PseudonymExport(val exportId: String, val bigquery: BigQuery, val datastor try { // Verify before writing a new value. val currentEntity = transaction.get(exportKey) - val builder: Entity.Builder? - if (currentEntity == null) { - builder = Entity.newBuilder(exportKey) - } else { - builder = Entity.newBuilder(currentEntity) - } + val builder: Entity.Builder = + if (currentEntity == null) { + Entity.newBuilder(exportKey) + } else { + Entity.newBuilder(currentEntity) + } // Prepare the new datastore entity val exportTask = builder .set(exportIdPropertyName, exportId) diff --git a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/resources/PseudonymResource.kt b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/resources/PseudonymResource.kt index aa15c08cb..b32be3f0c 100644 --- a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/resources/PseudonymResource.kt +++ b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/resources/PseudonymResource.kt @@ -1,19 +1,11 @@ package org.ostelco.pseudonym.resources -import com.google.cloud.bigquery.BigQuery -import com.google.cloud.datastore.Datastore -import com.google.cloud.datastore.Entity -import com.google.cloud.datastore.Key -import com.google.cloud.datastore.Query -import com.google.cloud.datastore.StructuredQuery.PropertyFilter import org.hibernate.validator.constraints.NotBlank -import org.ostelco.prime.model.ActivePseudonyms -import org.ostelco.prime.model.PseudonymEntity -import org.ostelco.pseudonym.managed.PseudonymExport +import org.ostelco.pseudonym.service.PseudonymizerServiceSingleton +import org.ostelco.pseudonym.service.PseudonymizerServiceSingleton.getExportTask +import org.ostelco.pseudonym.service.PseudonymizerServiceSingleton.getPseudonymEntityFor import org.slf4j.LoggerFactory import java.time.Instant -import java.util.* -import java.util.concurrent.Executors import javax.ws.rs.DELETE import javax.ws.rs.GET import javax.ws.rs.Path @@ -21,58 +13,21 @@ import javax.ws.rs.PathParam import javax.ws.rs.core.MediaType import javax.ws.rs.core.Response import javax.ws.rs.core.Response.Status -import kotlin.collections.HashMap -const val PseudonymEntityKind = "Pseudonym" -const val msisdnPropertyName = "msisdn" -const val pseudonymPropertyName = "pseudonym" -const val startPropertyName = "start" -const val endPropertyName = "end" - /** * Class representing the Export task entity in Datastore. */ data class ExportTask(val exportId: String, val status: String, val error: String) -const val ExportTaskKind = "ExportTask" -const val exportIdPropertyName = "exportId" -const val statusPropertyName = "status" -const val errorPropertyName = "error" - - -/** - * Class representing the boundary timestamps. - */ -data class Bounds(val start: Long, val end: Long) - -/** - * Interface which provides the method to retrieve the boundary timestamps. - */ -interface DateBounds { - /** - * Returns the boundaries for the period of the given timestamp. - * (start <= timestamp <= end). Timestamps are in UTC - * Also returns the key prefix - */ - fun getBoundsNKeyPrefix(msisdn: String, timestamp: Long): Pair - - /** - * Returns the timestamp for start of the next period for given timestamp. - * (value > timestamp). Timestamps are in UTC - */ - fun getNextPeriodStart(timestamp: Long): Long -} - /** * Resource used to handle the pseudonym related REST calls. The map of pseudonym objects * are store in datastore. The key for the object is made from "-. */ @Path("/pseudonym") -class PseudonymResource(val datastore: Datastore, val dateBounds: DateBounds, val bigquery: BigQuery?) { +class PseudonymResource { private val logger = LoggerFactory.getLogger(PseudonymResource::class.java) - private val executor = Executors.newFixedThreadPool(3) /** * Get the pseudonym which is valid at the timestamp for the given @@ -84,7 +39,7 @@ class PseudonymResource(val datastore: Datastore, val dateBounds: DateBounds, va fun getPseudonym(@NotBlank @PathParam("msisdn") msisdn: String, @NotBlank @PathParam("timestamp") timestamp: String): Response { logger.info("GET pseudonym for Msisdn = $msisdn at timestamp = $timestamp") - val entity = getPseudonymEntityFor(msisdn, timestamp.toLong()) + val entity = PseudonymizerServiceSingleton.getPseudonymEntityFor(msisdn, timestamp.toLong()) return Response.ok(entity, MediaType.APPLICATION_JSON).build() } @@ -110,22 +65,9 @@ class PseudonymResource(val datastore: Datastore, val dateBounds: DateBounds, va @GET @Path("/active/{msisdn}") fun getActivePseudonyms(@NotBlank @PathParam("msisdn") msisdn: String): Response { - val currentTimestamp = Instant.now().toEpochMilli() - val nextTimestamp = dateBounds.getNextPeriodStart(currentTimestamp) - logger.info("GET pseudonym for Msisdn = $msisdn at timestamps = $currentTimestamp & $nextTimestamp") - val current = getPseudonymEntityFor(msisdn, currentTimestamp) - val next = getPseudonymEntityFor(msisdn, nextTimestamp) - val entity = ActivePseudonyms(current, next) - return Response.ok(entity, MediaType.APPLICATION_JSON).build() - } - - private fun getPseudonymEntityFor(@NotBlank msisdn: String, timestamp: Long): PseudonymEntity { - val (bounds, keyPrefix) = dateBounds.getBoundsNKeyPrefix(msisdn, timestamp) - var entity = getPseudonymEntity(keyPrefix) - if (entity == null) { - entity = createPseudonym(msisdn, bounds, keyPrefix) - } - return entity + return Response.ok( + PseudonymizerServiceSingleton.getActivePseudonymsForMsisdn(msisdn = msisdn), + MediaType.APPLICATION_JSON).build() } /** @@ -136,18 +78,9 @@ class PseudonymResource(val datastore: Datastore, val dateBounds: DateBounds, va @Path("/find/{pseudonym}") fun findPseudonym(@NotBlank @PathParam("pseudonym") pseudonym: String): Response { logger.info("Find details for pseudonym = $pseudonym") - val query = Query.newEntityQueryBuilder() - .setKind(PseudonymEntityKind) - .setFilter(PropertyFilter.eq(pseudonymPropertyName, pseudonym)) - .setLimit(1) - .build() - val results = datastore.run(query) - if (results.hasNext()) { - val entity = results.next() - return Response.ok(convertToPseudonymEntity(entity), MediaType.APPLICATION_JSON).build() - } - logger.info("Couldn't find, pseudonym = $pseudonym") - return Response.status(Status.NOT_FOUND).build() + return PseudonymizerServiceSingleton.findPseudonym(pseudonym = pseudonym) + ?.let { Response.ok(it, MediaType.APPLICATION_JSON).build() } + ?: Response.status(Status.NOT_FOUND).build() } /** @@ -159,21 +92,9 @@ class PseudonymResource(val datastore: Datastore, val dateBounds: DateBounds, va @Path("/delete/{msisdn}") fun deleteAllPseudonyms(@NotBlank @PathParam("msisdn") msisdn: String): Response { logger.info("delete all pseudonyms for Msisdn = $msisdn") - val query = Query.newEntityQueryBuilder() - .setKind(PseudonymEntityKind) - .setFilter(PropertyFilter.eq(msisdnPropertyName, msisdn)) - .setLimit(1) - .build() - val results = datastore.run(query) - var count = 0 - while (results.hasNext()) { - val entity = results.next() - datastore.delete(entity.key) - count++ - } + val count = PseudonymizerServiceSingleton.deleteAllPseudonyms(msisdn = msisdn) // Return a Json object with number of records deleted. - val countMap = HashMap() - countMap["count"] = count + val countMap = mapOf("count" to count) logger.info("deleted $count records for Msisdn = $msisdn") return Response.ok(countMap, MediaType.APPLICATION_JSON).build() } @@ -187,12 +108,7 @@ class PseudonymResource(val datastore: Datastore, val dateBounds: DateBounds, va @Path("/export/{exportId}") fun exportPseudonyms(@NotBlank @PathParam("exportId") exportId: String): Response { logger.info("GET export all pseudonyms to the table $exportId") - if (bigquery == null) { - logger.info("BigQuery is not available, ignoring export request $exportId") - return Response.status(Status.NOT_FOUND).build() - } - val exporter = PseudonymExport(exportId, bigquery, datastore) - executor.execute(exporter.getRunnable()) + PseudonymizerServiceSingleton.exportPseudonyms(exportId = exportId) return Response.ok("Started Exporting", MediaType.TEXT_PLAIN).build() } @@ -203,76 +119,8 @@ class PseudonymResource(val datastore: Datastore, val dateBounds: DateBounds, va @Path("/exportstatus/{exportId}") fun getExportStatus(@NotBlank @PathParam("exportId") exportId: String): Response { logger.info("GET status of export $exportId") - val exportTask = getExportTask(exportId) - if (exportTask != null) { - return Response.ok(exportTask, MediaType.APPLICATION_JSON).build() - } - return Response.status(Status.NOT_FOUND).build() - } - - private fun getExportTask(exportId: String): ExportTask? { - val exportKey = datastore.newKeyFactory().setKind(ExportTaskKind).newKey(exportId) - val value = datastore.get(exportKey) - if (value != null) { - // Create the object from datastore entity - return ExportTask( - value.getString(exportIdPropertyName), - value.getString(statusPropertyName), - value.getString(errorPropertyName)) - } - return null - } - - private fun getPseudonymKey(keyPrefix: String): Key { - return datastore.newKeyFactory().setKind(PseudonymEntityKind).newKey(keyPrefix) - } - - private fun getPseudonymEntity(keyPrefix: String): PseudonymEntity? { - val pseudonymKey = getPseudonymKey(keyPrefix) - val value = datastore.get(pseudonymKey) - if (value != null) { - // Create the object from datastore entity - return convertToPseudonymEntity(value) - } - return null - } - - private fun convertToPseudonymEntity(entity: Entity): PseudonymEntity { - return PseudonymEntity( - entity.getString(msisdnPropertyName), - entity.getString(pseudonymPropertyName), - entity.getLong(startPropertyName), - entity.getLong(endPropertyName)) - } - - private fun createPseudonym(msisdn: String, bounds: Bounds, keyPrefix: String): PseudonymEntity { - val uuid = UUID.randomUUID().toString() - var entity = PseudonymEntity(msisdn, uuid, bounds.start, bounds.end) - val pseudonymKey = getPseudonymKey(keyPrefix) - - val transaction = datastore.newTransaction() - try { - // Verify before writing a new value. - val currentEntity = transaction.get(pseudonymKey) - if (currentEntity == null) { - // Prepare the new datastore entity - val pseudonym = Entity.newBuilder(pseudonymKey) - .set(msisdnPropertyName, entity.msisdn) - .set(pseudonymPropertyName, entity.pseudonym) - .set(startPropertyName, entity.start) - .set(endPropertyName, entity.end) - .build() - transaction.put(pseudonym) - transaction.commit() - } else { - // Use the existing one - entity = convertToPseudonymEntity(currentEntity) - } - } finally { - if (transaction.isActive) { - transaction.rollback() - } - } - return entity + return getExportTask(exportId) + ?.let { Response.ok(it, MediaType.APPLICATION_JSON).build() } + ?: Response.status(Status.NOT_FOUND).build() } } diff --git a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/service/PseudonymizerServiceSingleton.kt b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/service/PseudonymizerServiceSingleton.kt new file mode 100644 index 000000000..b8eeb24d6 --- /dev/null +++ b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/service/PseudonymizerServiceSingleton.kt @@ -0,0 +1,227 @@ +package org.ostelco.pseudonym.service + +import com.google.cloud.bigquery.BigQuery +import com.google.cloud.bigquery.BigQueryOptions +import com.google.cloud.datastore.Datastore +import com.google.cloud.datastore.DatastoreOptions +import com.google.cloud.datastore.Entity +import com.google.cloud.datastore.Key +import com.google.cloud.datastore.Query +import com.google.cloud.datastore.StructuredQuery.PropertyFilter +import com.google.cloud.datastore.testing.LocalDatastoreHelper +import org.hibernate.validator.constraints.NotBlank +import org.ostelco.prime.logger +import org.ostelco.prime.model.ActivePseudonyms +import org.ostelco.prime.model.PseudonymEntity +import org.ostelco.prime.pseudonymizer.PseudonymizerService +import org.ostelco.pseudonym.ConfigRegistry +import org.ostelco.pseudonym.ExportTaskKind +import org.ostelco.pseudonym.PseudonymEntityKind +import org.ostelco.pseudonym.PseudonymServerConfig +import org.ostelco.pseudonym.endPropertyName +import org.ostelco.pseudonym.errorPropertyName +import org.ostelco.pseudonym.exportIdPropertyName +import org.ostelco.pseudonym.managed.PseudonymExport +import org.ostelco.pseudonym.msisdnPropertyName +import org.ostelco.pseudonym.pseudonymPropertyName +import org.ostelco.pseudonym.resources.ExportTask +import org.ostelco.pseudonym.startPropertyName +import org.ostelco.pseudonym.statusPropertyName +import org.ostelco.pseudonym.utils.WeeklyBounds +import java.time.Instant +import java.util.* +import java.util.concurrent.Executors + +class PseudonymServiceImpl : PseudonymizerService by PseudonymizerServiceSingleton + +/** + * Class representing the boundary timestamps. + */ +data class Bounds(val start: Long, val end: Long) + +/** + * Interface which provides the method to retrieve the boundary timestamps. + */ +interface DateBounds { + /** + * Returns the boundaries for the period of the given timestamp. + * (start <= timestamp <= end). Timestamps are in UTC + * Also returns the key prefix + */ + fun getBoundsNKeyPrefix(msisdn: String, timestamp: Long): Pair + + /** + * Returns the timestamp for start of the next period for given timestamp. + * (value > timestamp). Timestamps are in UTC + */ + fun getNextPeriodStart(timestamp: Long): Long +} + +object PseudonymizerServiceSingleton : PseudonymizerService { + + private val logger by logger() + + private lateinit var datastore: Datastore + private lateinit var bigquery: BigQuery + private val dateBounds: DateBounds = WeeklyBounds() + + private val executor = Executors.newFixedThreadPool(3) + + fun init() { + datastore = getDatastore(ConfigRegistry.config) + if(System.getenv("LOCAL_TESTING") != "true") { + bigquery = BigQueryOptions.getDefaultInstance().service + } else { + logger.info("Local testing, BigQuery is not available...") + } + } + + override fun getActivePseudonymsForMsisdn(msisdn: String): ActivePseudonyms { + val currentTimestamp = Instant.now().toEpochMilli() + val nextTimestamp = dateBounds.getNextPeriodStart(currentTimestamp) + logger.info("GET pseudonym for Msisdn = $msisdn at timestamps = $currentTimestamp & $nextTimestamp") + val current = getPseudonymEntityFor(msisdn, currentTimestamp) + val next = getPseudonymEntityFor(msisdn, nextTimestamp) + return ActivePseudonyms(current, next) + } + + fun findPseudonym(pseudonym: String): PseudonymEntity? { + val query = Query.newEntityQueryBuilder() + .setKind(PseudonymEntityKind) + .setFilter(PropertyFilter.eq(pseudonymPropertyName, pseudonym)) + .setLimit(1) + .build() + val results = datastore.run(query) + if (results.hasNext()) { + val entity = results.next() + return convertToPseudonymEntity(entity) + } + logger.info("Couldn't find, pseudonym = $pseudonym") + return null + } + + fun deleteAllPseudonyms(msisdn: String): Int { + val query = Query.newEntityQueryBuilder() + .setKind(PseudonymEntityKind) + .setFilter(PropertyFilter.eq(msisdnPropertyName, msisdn)) + .setLimit(1) + .build() + val results = datastore.run(query) + var count = 0 + while (results.hasNext()) { + val entity = results.next() + datastore.delete(entity.key) + count++ + } + return count + } + + fun exportPseudonyms(exportId: String) { + logger.info("GET export all pseudonyms to the table $exportId") + val exporter = PseudonymExport(exportId, bigquery, datastore) + executor.execute(exporter.getRunnable()) + } + + // Integration testing helper for Datastore. + private fun getDatastore(config: PseudonymServerConfig): Datastore { + val datastore: Datastore? + if (config.datastoreType == "inmemory-emulator") { + logger.info("Starting with in-memory datastore emulator...") + val helper: LocalDatastoreHelper = LocalDatastoreHelper.create(1.0) + helper.start() + datastore = helper.options.service + } else { + datastore = DatastoreOptions.getDefaultInstance().service + logger.info("Created default instance of datastore client") + + // TODO vihang: make this part of health-check + val testKey = datastore.newKeyFactory().setKind("TestKind").newKey("testKey") + val testPropertyKey = "testPropertyKey" + val testPropertyValue = "testPropertyValue" + val testEntity = Entity.newBuilder(testKey).set(testPropertyKey, testPropertyValue).build() + datastore.put(testEntity) + val value = datastore.get(testKey).getString(testPropertyKey) + if (testPropertyValue != value) { + logger.warn("Unable to fetch test property value from datastore") + } + datastore.delete(testKey) + // END + } + return datastore + } + + fun getExportTask(exportId: String): ExportTask? { + val exportKey = datastore.newKeyFactory().setKind(ExportTaskKind).newKey(exportId) + val value = datastore.get(exportKey) + if (value != null) { + // Create the object from datastore entity + return ExportTask( + value.getString(exportIdPropertyName), + value.getString(statusPropertyName), + value.getString(errorPropertyName)) + } + return null + } + + private fun getPseudonymKey(keyPrefix: String): Key { + return datastore.newKeyFactory().setKind(PseudonymEntityKind).newKey(keyPrefix) + } + + private fun getPseudonymEntity(keyPrefix: String): PseudonymEntity? { + val pseudonymKey = getPseudonymKey(keyPrefix) + val value = datastore.get(pseudonymKey) + if (value != null) { + // Create the object from datastore entity + return convertToPseudonymEntity(value) + } + return null + } + + fun getPseudonymEntityFor(@NotBlank msisdn: String, timestamp: Long): PseudonymEntity { + val (bounds, keyPrefix) = dateBounds.getBoundsNKeyPrefix(msisdn, timestamp) + var entity = getPseudonymEntity(keyPrefix) + if (entity == null) { + entity = createPseudonym(msisdn, bounds, keyPrefix) + } + return entity + } + + private fun createPseudonym(msisdn: String, bounds: Bounds, keyPrefix: String): PseudonymEntity { + val uuid = UUID.randomUUID().toString() + var entity = PseudonymEntity(msisdn, uuid, bounds.start, bounds.end) + val pseudonymKey = getPseudonymKey(keyPrefix) + + val transaction = datastore.newTransaction() + try { + // Verify before writing a new value. + val currentEntity = transaction.get(pseudonymKey) + if (currentEntity == null) { + // Prepare the new datastore entity + val pseudonym = Entity.newBuilder(pseudonymKey) + .set(msisdnPropertyName, entity.msisdn) + .set(pseudonymPropertyName, entity.pseudonym) + .set(startPropertyName, entity.start) + .set(endPropertyName, entity.end) + .build() + transaction.put(pseudonym) + transaction.commit() + } else { + // Use the existing one + entity = convertToPseudonymEntity(currentEntity) + } + } finally { + if (transaction.isActive) { + transaction.rollback() + } + } + return entity + } + + private fun convertToPseudonymEntity(entity: Entity): PseudonymEntity { + return PseudonymEntity( + entity.getString(msisdnPropertyName), + entity.getString(pseudonymPropertyName), + entity.getLong(startPropertyName), + entity.getLong(endPropertyName)) + } +} \ No newline at end of file diff --git a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/utils/DateUtils.kt b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/utils/DateUtils.kt index 2d50f5bba..0c201f843 100644 --- a/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/utils/DateUtils.kt +++ b/pseudonym-server/src/main/kotlin/org/ostelco/pseudonym/utils/DateUtils.kt @@ -1,7 +1,7 @@ package org.ostelco.pseudonym.utils -import org.ostelco.pseudonym.resources.Bounds -import org.ostelco.pseudonym.resources.DateBounds +import org.ostelco.pseudonym.service.Bounds +import org.ostelco.pseudonym.service.DateBounds import java.util.* /** diff --git a/pseudonym-server/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable b/pseudonym-server/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable new file mode 100644 index 000000000..8056fe23b --- /dev/null +++ b/pseudonym-server/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable @@ -0,0 +1 @@ +org.ostelco.prime.module.PrimeModule \ No newline at end of file diff --git a/pseudonym-server/src/main/resources/META-INF/services/org.ostelco.prime.module.PrimeModule b/pseudonym-server/src/main/resources/META-INF/services/org.ostelco.prime.module.PrimeModule new file mode 100644 index 000000000..9e15e0fdf --- /dev/null +++ b/pseudonym-server/src/main/resources/META-INF/services/org.ostelco.prime.module.PrimeModule @@ -0,0 +1 @@ +org.ostelco.pseudonym.PseudonymModule \ No newline at end of file diff --git a/pseudonym-server/src/main/resources/META-INF/services/org.ostelco.prime.pseudonymizer.PseudonymizerService b/pseudonym-server/src/main/resources/META-INF/services/org.ostelco.prime.pseudonymizer.PseudonymizerService new file mode 100644 index 000000000..e54e12f9b --- /dev/null +++ b/pseudonym-server/src/main/resources/META-INF/services/org.ostelco.prime.pseudonymizer.PseudonymizerService @@ -0,0 +1 @@ +org.ostelco.pseudonym.service.PseudonymServiceImpl \ No newline at end of file diff --git a/pseudonym-server/src/test/kotlin/org/ostelco/pseudonym/DateUtilsTest.kt b/pseudonym-server/src/test/kotlin/org/ostelco/pseudonym/DateUtilsTest.kt index 194841478..e0711998c 100644 --- a/pseudonym-server/src/test/kotlin/org/ostelco/pseudonym/DateUtilsTest.kt +++ b/pseudonym-server/src/test/kotlin/org/ostelco/pseudonym/DateUtilsTest.kt @@ -8,7 +8,7 @@ import kotlin.test.assertEquals * Class for unit testing DateUtils. */ class DateUtilsTest { - val dateBounds = WeeklyBounds() + private val dateBounds = WeeklyBounds() /** * Test the most common use case, find next start period */ @@ -19,7 +19,7 @@ class DateUtilsTest { // GMT: Monday, May 14, 2018 12:00:00 AM val expectedNextTimestamp = 1526256000000 val nextTimestamp = dateBounds.getNextPeriodStart(timestamp) - print("Expected Timestamp ${expectedNextTimestamp} Next timestamp ${nextTimestamp}"); + print("Expected Timestamp $expectedNextTimestamp Next timestamp $nextTimestamp"); assertEquals(expectedNextTimestamp, nextTimestamp) } /** diff --git a/pseudonym-server/src/test/kotlin/org/ostelco/pseudonym/PseudonymResourceTest.kt b/pseudonym-server/src/test/kotlin/org/ostelco/pseudonym/PseudonymResourceTest.kt index 9814b32cc..cd8d5e973 100644 --- a/pseudonym-server/src/test/kotlin/org/ostelco/pseudonym/PseudonymResourceTest.kt +++ b/pseudonym-server/src/test/kotlin/org/ostelco/pseudonym/PseudonymResourceTest.kt @@ -2,17 +2,13 @@ package org.ostelco.pseudonym import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue -import com.google.cloud.bigquery.BigQuery -import com.google.cloud.datastore.Datastore -import com.google.cloud.datastore.testing.LocalDatastoreHelper import io.dropwizard.testing.junit.ResourceTestRule import org.junit.ClassRule import org.junit.Test -import org.mockito.Mockito.mock import org.ostelco.prime.model.ActivePseudonyms import org.ostelco.prime.model.PseudonymEntity import org.ostelco.pseudonym.resources.PseudonymResource -import org.ostelco.pseudonym.utils.WeeklyBounds +import org.ostelco.pseudonym.service.PseudonymizerServiceSingleton import javax.ws.rs.core.Response.Status import kotlin.test.assertEquals import kotlin.test.assertNotNull @@ -22,31 +18,32 @@ import kotlin.test.assertTrue * Class for unit testing PseudonymResource. */ class PseudonymResourceTest { + private val pathForGet = "/pseudonym/get" private val pathForCurrent = "/pseudonym/current" private val pathForActive = "/pseudonym/active" - private val pathForFind= "/pseudonym/find" - private val pathForDelete= "/pseudonym/delete" + private val pathForFind = "/pseudonym/find" + private val pathForDelete = "/pseudonym/delete" private val testMsisdn1 = "4790303333" private val testMsisdn2 = "4790309999" companion object { - private var datastore: Datastore - init { - val helper: LocalDatastoreHelper = LocalDatastoreHelper.create(1.0) - helper.start() - datastore = helper.options.service + ConfigRegistry.config = PseudonymServerConfig() + .apply { this.datastoreType = "inmemory-emulator" } + PseudonymizerServiceSingleton.init() } @ClassRule @JvmField - val resources = ResourceTestRule.builder() - .addResource(PseudonymResource(datastore, WeeklyBounds(), mock(BigQuery::class.java))) + val resources: ResourceTestRule? = ResourceTestRule.builder() + .addResource(PseudonymResource()) .build() } - val mapper = jacksonObjectMapper() + + private val mapper = jacksonObjectMapper() + /** * Test what happens when parameter is not given */ @@ -61,6 +58,7 @@ class PseudonymResourceTest { assertEquals(Status.NOT_FOUND.statusCode, statusCode) } + /** * Test a normal request will all parameters */ @@ -81,27 +79,32 @@ class PseudonymResourceTest { @Test fun testGetPseudonym() { - var result = resources - ?.target("$pathForCurrent/$testMsisdn1") - ?.request() - ?.get() - assertNotNull(result) - if (result == null) return - assertEquals(Status.OK.statusCode, result.status) - var json = result.readEntity(String::class.java) - var pseudonymEntity = mapper.readValue(json) - assertEquals(pseudonymEntity.msisdn, testMsisdn1) - - result = resources - ?.target("$pathForGet/$testMsisdn1/${pseudonymEntity.start}") - ?.request() - ?.get() - assertNotNull(result) - if (result == null) return - assertEquals(Status.OK.statusCode, result.status) - json = result.readEntity(String::class.java) - val pseudonymEntity2 = mapper.readValue(json) - assertEquals(pseudonymEntity2.pseudonym, pseudonymEntity.pseudonym) + lateinit var pseudonymEntity:PseudonymEntity + run { + val result = resources + ?.target("$pathForCurrent/$testMsisdn1") + ?.request() + ?.get() + assertNotNull(result) + if (result == null) return + assertEquals(Status.OK.statusCode, result.status) + val json = result.readEntity(String::class.java) + pseudonymEntity = mapper.readValue(json) + assertEquals(testMsisdn1, pseudonymEntity.msisdn) + } + + run { + val result = resources + ?.target("$pathForGet/$testMsisdn1/${pseudonymEntity.start}") + ?.request() + ?.get() + assertNotNull(result) + if (result == null) return + assertEquals(Status.OK.statusCode, result.status) + val json = result.readEntity(String::class.java) + val pseudonymEntity2 = mapper.readValue(json) + assertEquals(pseudonymEntity.pseudonym, pseudonymEntity2.pseudonym) + } } /** @@ -110,34 +113,39 @@ class PseudonymResourceTest { @Test fun testActivePseudonyms() { - var result = resources - ?.target("$pathForCurrent/$testMsisdn1") - ?.request() - ?.get() - assertNotNull(result) - if (result == null) return - assertEquals(Status.OK.statusCode, result.status) - var json = result.readEntity(String::class.java) - var pseudonymEntity = mapper.readValue(json) - assertEquals(pseudonymEntity.msisdn, testMsisdn1) - - result = resources - ?.target("$pathForActive/$testMsisdn1") - ?.request() - ?.get() - assertNotNull(result) - if (result == null) return - assertEquals(Status.OK.statusCode, result.status) - json = result.readEntity(String::class.java) - // This is how the client will recieve the output. - val mapOfPseudonyms:Map = mapper.readValue>(json) - val current = mapOfPseudonyms["current"] - val next = mapOfPseudonyms["next"] - assertNotNull(current) - assertNotNull(next) - if (current != null && next != null) { - assertEquals(current.pseudonym, pseudonymEntity.pseudonym) - assertEquals(current.end+1, next.start) + lateinit var pseudonymEntity:PseudonymEntity + run { + val result = resources + ?.target("$pathForCurrent/$testMsisdn1") + ?.request() + ?.get() + assertNotNull(result) + if (result == null) return + assertEquals(Status.OK.statusCode, result.status) + val json = result.readEntity(String::class.java) + pseudonymEntity = mapper.readValue(json) + assertEquals(testMsisdn1, pseudonymEntity.msisdn) + } + + run { + val result = resources + ?.target("$pathForActive/$testMsisdn1") + ?.request() + ?.get() + assertNotNull(result) + if (result == null) return + assertEquals(Status.OK.statusCode, result.status) + val json = result.readEntity(String::class.java) + // This is how the client will recieve the output. + val mapOfPseudonyms: Map = mapper.readValue(json) + val current = mapOfPseudonyms["current"] + val next = mapOfPseudonyms["next"] + assertNotNull(current) + assertNotNull(next) + if (current != null && next != null) { + assertEquals(current.pseudonym, pseudonymEntity.pseudonym) + assertEquals(current.end + 1, next.start) + } } } @@ -147,28 +155,33 @@ class PseudonymResourceTest { @Test fun testActivePseudonymUsingModel() { - var result = resources - ?.target("$pathForCurrent/$testMsisdn1") - ?.request() - ?.get() - assertNotNull(result) - if (result == null) return - assertEquals(Status.OK.statusCode, result.status) - var json = result.readEntity(String::class.java) - var pseudonymEntity = mapper.readValue(json) - assertEquals(pseudonymEntity.msisdn, testMsisdn1) - - result = resources - ?.target("$pathForActive/$testMsisdn1") - ?.request() - ?.get() - assertNotNull(result) - if (result == null) return - assertEquals(Status.OK.statusCode, result.status) - json = result.readEntity(String::class.java) - val active = mapper.readValue(json) - assertEquals(active.current.pseudonym, pseudonymEntity.pseudonym) - assertEquals(active.current.end+1, active.next.start) + lateinit var pseudonymEntity:PseudonymEntity + run { + val result = resources + ?.target("$pathForCurrent/$testMsisdn1") + ?.request() + ?.get() + assertNotNull(result) + if (result == null) return + assertEquals(Status.OK.statusCode, result.status) + val json = result.readEntity(String::class.java) + pseudonymEntity = mapper.readValue(json) + assertEquals(testMsisdn1, pseudonymEntity.msisdn) + } + + run { + val result = resources + ?.target("$pathForActive/$testMsisdn1") + ?.request() + ?.get() + assertNotNull(result) + if (result == null) return + assertEquals(Status.OK.statusCode, result.status) + val json = result.readEntity(String::class.java) + val active = mapper.readValue(json) + assertEquals(active.current.pseudonym, pseudonymEntity.pseudonym) + assertEquals(active.current.end + 1, active.next.start) + } } /** @@ -177,27 +190,32 @@ class PseudonymResourceTest { @Test fun testFindPseudonym() { - var result = resources - ?.target("$pathForCurrent/$testMsisdn1") - ?.request() - ?.get() - assertNotNull(result) - if (result == null) return - assertEquals(Status.OK.statusCode, result.status) - var json = result.readEntity(String::class.java) - var pseudonymEntity = mapper.readValue(json) - assertEquals(pseudonymEntity.msisdn, testMsisdn1) - - result = resources - ?.target("$pathForFind/${pseudonymEntity.pseudonym}") - ?.request() - ?.get() - assertNotNull(result) - if (result == null) return - assertEquals(Status.OK.statusCode, result.status) - json = result.readEntity(String::class.java) - pseudonymEntity = mapper.readValue(json) - assertEquals(pseudonymEntity.msisdn, testMsisdn1) + lateinit var pseudonymEntity:PseudonymEntity + run { + val result = resources + ?.target("$pathForCurrent/$testMsisdn1") + ?.request() + ?.get() + assertNotNull(result) + if (result == null) return + assertEquals(Status.OK.statusCode, result.status) + val json = result.readEntity(String::class.java) + pseudonymEntity = mapper.readValue(json) + assertEquals(testMsisdn1, pseudonymEntity.msisdn) + } + + run { + val result = resources + ?.target("$pathForFind/${pseudonymEntity.pseudonym}") + ?.request() + ?.get() + assertNotNull(result) + if (result == null) return + assertEquals(Status.OK.statusCode, result.status) + val json = result.readEntity(String::class.java) + val pseudonymEntity2 = mapper.readValue(json) + assertEquals(testMsisdn1, pseudonymEntity.msisdn) + } } /** @@ -205,35 +223,42 @@ class PseudonymResourceTest { */ @Test fun testDeletePseudonym() { - var result = resources - ?.target("$pathForCurrent/$testMsisdn2") - ?.request() - ?.get() - assertNotNull(result) - if (result == null) return - assertEquals(Status.OK.statusCode, result.status) - var json = result.readEntity(String::class.java) - var pseudonymEntity = mapper.readValue(json) - assertEquals(pseudonymEntity.msisdn, testMsisdn2) - - result = resources - ?.target("$pathForDelete/$testMsisdn2") - ?.request() - ?.delete() - assertNotNull(result) - if (result == null) return - assertEquals(Status.OK.statusCode, result.status) - json = result.readEntity(String::class.java) - val countMap = mapper.readValue>(json) - val count = countMap["count"] ?: -1 - assertTrue(count >= 1) - - result = resources - ?.target("$pathForFind/${pseudonymEntity.pseudonym}") - ?.request() - ?.get() - assertNotNull(result) - if (result == null) return - assertEquals(Status.NOT_FOUND.statusCode, result.status) + lateinit var pseudonymEntity:PseudonymEntity + run { + val result = resources + ?.target("$pathForCurrent/$testMsisdn2") + ?.request() + ?.get() + assertNotNull(result) + if (result == null) return + assertEquals(Status.OK.statusCode, result.status) + val json = result.readEntity(String::class.java) + pseudonymEntity = mapper.readValue(json) + assertEquals(testMsisdn2, pseudonymEntity.msisdn) + } + + run { + val result = resources + ?.target("$pathForDelete/$testMsisdn2") + ?.request() + ?.delete() + assertNotNull(result) + if (result == null) return + assertEquals(Status.OK.statusCode, result.status) + val json = result.readEntity(String::class.java) + val countMap = mapper.readValue>(json) + val count = countMap["count"] ?: -1 + assertTrue(count >= 1) + } + + run { + val result = resources + ?.target("$pathForFind/${pseudonymEntity.pseudonym}") + ?.request() + ?.get() + assertNotNull(result) + if (result == null) return + assertEquals(Status.NOT_FOUND.statusCode, result.status) + } } } \ No newline at end of file