Transfer comit

.gradle/buildOutputCleanup/cache.properties

@@ -1,2 +1,2 @@
-#Thu Apr 14 22:24:00 CEST 2022
-gradle.version=7.1
+#Sat Apr 16 22:52:48 CEST 2022
+gradle.version=6.8.3

build.gradle

@@ -1,17 +1,19 @@
-plugins {
-    id 'groovy'
-    id 'java'
-}
+apply plugin: 'groovy'
+apply plugin: 'java'
 
 group 'org.example'
-version '1.0-SNAPSHOT'
+version '1.0'
+
+sourceCompatibility = '11'
+targetCompatibility = '11'
 
 repositories {
     mavenCentral()
 }
 
 dependencies {
-    implementation 'org.codehaus.groovy:groovy-all:3.0.5'
+    //implementation 'org.codehaus.groovy:groovy-all:3.0.5'
+    implementation "org.codehaus.groovy:groovy-all:3.0.5"
     testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1'
     testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.1'
 
@@ -20,6 +22,12 @@ dependencies {
 
     //Synthea
     implementation group: 'org.mitre.synthea', name: 'synthea', version: '3.0.0'
+
+    //Jackson
+    implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.12.3'
+    implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.12.3'
+    implementation group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: '2.13.2'
+
 }
 
 test {

build/resources/main/config/log4j2.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<Configuration status="info">
+    <Properties>
+        <Property name="filename">log/logging.log</Property>
+    </Properties>
+
+    <Appenders>
+        <Console name="Console" target="SYSTEM_OUT">
+            <PatternLayout pattern="%style{%d{yyyy-MM-dd HH:mm:ss.SSS}}{yellow} %highlight{[%-5level] [%t] %c{1} - %msg}{STYLE=Logback}%n"
+                           disableAnsi="false "/>
+        </Console>
+        <File name="File" fileName="log/logging.log" append="false">
+            <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%-5level] [%t] %c{1} - %msg%n"/>
+        </File>
+    </Appenders>
+    <Loggers>
+        <Root level="trace">
+            <AppenderRef ref="Console" level="info"/>
+            <AppenderRef ref="File" level="debug" />
+        </Root>
+    </Loggers>
+</Configuration>

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.3-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

src/main/groovy/org/example/syntheakds/Main.groovy

@@ -2,19 +2,26 @@ package org.example.syntheakds
 
 import org.apache.logging.log4j.LogManager
 import org.apache.logging.log4j.Logger
+import org.apache.logging.log4j.core.LoggerContext
 import org.example.syntheakds.config.SyntheaKDSConfig
+import org.example.syntheakds.read.DataProvider
 import org.mitre.synthea.engine.Generator
 import org.mitre.synthea.engine.Generator.GeneratorOptions
 import org.mitre.synthea.helpers.Config
 
 import java.nio.file.Path
+import java.nio.file.Paths
+import java.util.concurrent.Executors
+import java.util.concurrent.ThreadPoolExecutor
 
 class Main {
 
     private static final Logger logger = LogManager.getLogger(Main.class)
 
     static void main(String[] args){
 
+        configureLog4j2()
+
         logger.info("[#]Starting application ...")
 
         logger.info("[#]Running Synthea ...")
@@ -23,6 +30,10 @@ class Main {
         generator.run()
         logger.info("[#]Finished running Synthea.")
 
+        logger.info("[#]starting conversion ...")
+        def ioPool = Executors.newFixedThreadPool(50)
+        def provider = new DataProvider(SyntheaKDSConfig.patDir, 100, 10000, ioPool as ThreadPoolExecutor)
+
     }
 
     private static GeneratorOptions configureGeneratorOptions(){
@@ -38,4 +49,14 @@ class Main {
         return options
     }
 
+    /**
+     * Method for setting Log4j2 config file location. Should work in jar as well
+     */
+    private static void configureLog4j2(){
+        def configFilePath = Paths.get('config', 'log4j2.xml').toString()
+        def configFileUrl = Thread.currentThread().getContextClassLoader().getResource(configFilePath)
+        def context = LogManager.getContext(false) as LoggerContext
+        context.setConfigLocation(configFileUrl.toURI())
+    }
+
 }

src/main/groovy/org/example/syntheakds/config/SyntheaKDSConfig.groovy

@@ -6,5 +6,6 @@ class SyntheaKDSConfig {
 
     static int patientCount = 100
     static Path tmpOutputDir = Path.of('src', 'main', 'resources', 'tmp_output')
+    static Path patDir = tmpOutputDir.resolve('fhir')
 
 }

src/main/groovy/org/example/syntheakds/read/DataProvider.groovy

@@ -0,0 +1,60 @@
+package org.example.syntheakds.read
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.ObjectMapper
+import groovy.io.FileType
+import org.apache.logging.log4j.LogManager
+import org.apache.logging.log4j.Logger
+
+import java.nio.file.Path
+import java.util.concurrent.ArrayBlockingQueue
+import java.util.concurrent.ThreadPoolExecutor
+
+class DataProvider {
+
+    private static final Logger logger = LogManager.getLogger(DataProvider.class)
+
+    private final Path toDir
+    private final int patientCount
+    private final ObjectMapper objectMapper
+    private final Queue<JsonNode> readQueue
+    private final ThreadPoolExecutor ioPool
+
+    DataProvider(Path toDir, int patientCount, int qCapacity, ThreadPoolExecutor ioPool){
+        logger.debug("Initializing DataProvider instance ...")
+        this.toDir = toDir
+        this.patientCount = patientCount
+        this.objectMapper = new ObjectMapper()
+        this.readQueue = new ArrayBlockingQueue<>(qCapacity)
+        this.ioPool = ioPool
+        run()
+    }
+
+    private void run(){
+        def files = this.findFiles()
+        logger.debug("Reading files ...")
+        files.each {file ->
+            logger.debug("File @ ${file.getPath()}")
+            this.ioPool.submit({this.readQueue.add(objectMapper.readTree(file))})
+            logger.debug("--Submitted to I/O pool")
+            logger.debug("--Task count: ${this.ioPool.taskCount}")
+        }
+    }
+
+    Queue<JsonNode> getQ(){
+        return this.readQueue
+    }
+
+    private List<File> findFiles(){
+        logger.debug("Searching for files ...")
+        def files = []
+        this.toDir.eachFileMatch(type: FileType.FILES, nameFilter: ~/\*.json/){ file ->
+            logger.debug("Found file @ ${file.toString()}")
+            files << file.toFile()
+        }
+        return files
+    }
+
+
+
+}