Update NodeJS generators to use .filter for subsampling. Issue unicod…

…e-org#283

schema/message_fmt2/test_schema.json

@@ -27,6 +27,10 @@
           "type": "string",
           "description": "Identifier for the test."
         },
+        "description": {
+          "type": "string",
+          "description": "Tell about this test"
+        },
         "hexhash": {
           "description": "A hexadecimal hash code for the test without the label",
           "type": "string"

schema/schema_validator.py

@@ -105,7 +105,7 @@ def validate_json_file(self, schema_and_data_paths):
             validate(data_to_check, schema)
             # Everything worked!
             result_data['result'] = True
-        except ValidationError as err:
+        except exceptions.ValidationError as err:
             result_data['result'] = False
             result_data['error'] = err
             logging.error('ValidationError for test output %s and schema %s',

testgen/generators/common.js

@@ -1,3 +1,11 @@
+// Common functions for generation of test data from NodeJS
+
+// Globals to set increment and keep track of total generated.
+let label_increment = 1;
+let max_count = -1;
+let added_count = 0;
+
+const debug = false;
 
 function sample_tests(all_tests, run_limit) {
   // Gets a sampling of the data based on total and the expected number.
@@ -7,10 +15,32 @@ function sample_tests(all_tests, run_limit) {
   }
 
   let size_all = all_tests.length;
-  let increment = Math.floor(size_all / run_limit);
-  let samples = [];
-  for (let index = 0; index < size_all; index += increment) {
-    samples.push(all_tests[index]);
+  label_increment = Math.floor(size_all / run_limit);
+  max_count = run_limit;
+  added_count = 0;
+
+  if (debug) {
+    console.log('COMMON: ', size_all, ' increment: ', label_increment,
+                ' max_count: ', max_count);
+  }
+
+  let samples = all_tests.filter(filter_every_nth);
+  if (debug) {
+    console.debug('  COMMON results: ', samples.length);
+    console.debug('  COMMON results: ', samples);
   }
   return samples;
 }
+
+function filter_every_nth(test, index) {
+  if (added_count >= max_count) {
+    return false;
+  }
+  const add_this_one = ((index % label_increment) == 0);
+  if (add_this_one) {
+    added_count += 1;
+  }
+  return add_this_one
+}
+
+module.exports = {sample_tests};

testgen/generators/datetime_fmt.py

@@ -46,3 +46,4 @@ def process_test_data(self):
         result = subprocess.run(mv_command, shell=True)
 
         return result
+

testgen/generators/generate_test_hash.js

@@ -22,7 +22,7 @@ function generate_hash_for_test(test_case) {
   // Computes a 32 byte hex hash code for the test case
   // Note that the test case should not include 'label'.
 
-  obj = remove_none(test_case)
+  obj = remove_none(test_case);
   json_str = JSON.stringify(obj);
 
   const hasher = crypto.createHash("sha1");

testgen/generators/list_fmt.py

@@ -22,9 +22,14 @@ def process_test_data(self):
             'icu71': '18.7.0',
         }
 
+        exec_list = ['node generators/list_fmt_gen.js']
+        if self.run_limit > 0:
+            exec_list.append('-run_limit')
+            exec_list.append(str(self.run_limit))
+
         run_list = [
             ['source ~/.nvm/nvm.sh; nvm install 21.6.0; nvm use 21.6.0 --silent'],
-            ['node generators/list_fmt_gen.js'],
+            exec_list,
             ['mv list_fmt*.json icu74']
         ]
 
@@ -34,7 +39,8 @@ def process_test_data(self):
 
         # Set up Node version and call the generator
         nvm_version = icu_nvm_versions[self.icu_version]
-        generate_command = 'source ~/.nvm/nvm.sh; nvm install %s; nvm use %s --silent; node generators/list_fmt_gen.js' % (nvm_version, nvm_version)
+        generate_command = 'source ~/.nvm/nvm.sh; nvm install %s; nvm use %s --silent; %s' %\
+                           (nvm_version, nvm_version, ' '.join(exec_list))
 
         logging.debug('Running this command: %s', generate_command)
         result = result = subprocess.run(generate_command, shell=True)

testgen/generators/list_fmt_gen.js

@@ -8,6 +8,7 @@
 // Set up Node version to generate data specific to ICU/CLDR version
 // e.g., `nvm install 21.6.0;nvm use 21.6.0` (ICU 74)
 
+const common_fns = require("./common.js");
 const gen_hash = require("./generate_test_hash.js");
 
 const fs = require('node:fs');
@@ -70,7 +71,8 @@ function generateAll() {
   const expected_count = locales.length * types.length * styles.length *
         lists.length;
 
-  console.log("Generating ", expected_count, " list_fmt tests for ", process.versions.icu);
+  console.log("Generating up to ", expected_count, " list_fmt tests for ",
+              process.versions.icu);
 
   for (const locale of locales) {
 
@@ -156,25 +158,39 @@ function generateAll() {
   }
 
 
-  console.log('Number of list format tests generated for ',
-              process.versions.icu, ': ', label_num);
+  if (debug) {
+    console.log('Number of list format tests generated for ',
+                process.versions.icu, ': ', label_num);
+    console.log(' RUN LIMIT = ', run_limit);
+  }
 
-  test_obj['tests'] = test_cases;
+  test_obj['tests'] = common_fns.sample_tests(test_cases, run_limit);
   try {
-    fs.writeFileSync('list_fmt_test.json', JSON.stringify(test_obj, null));
+    fs.writeFileSync('list_fmt_test.json', JSON.stringify(test_obj, null, 2));
     // file written successfully
   } catch (err) {
     console.error(err);
   }
 
-  verify_obj['verifications'] = verify_cases;
+  verify_obj['verifications'] = common_fns.sample_tests(verify_cases, run_limit);
   try {
-    fs.writeFileSync('list_fmt_verify.json', JSON.stringify(verify_obj, null));
+    fs.writeFileSync('list_fmt_verify.json', JSON.stringify(verify_obj, null, 2));
     // file written successfully
   } catch (err) {
     console.error(err);
   }
 }
 
+if (debug) {
+  console.log('LIST FORMAT argv: ', process.argv);
+}
+
+let run_limit = -1;
+if (process.argv.length >= 4) {
+  if (process.argv[2] == '-run_limit') {
+    run_limit = Number(process.argv[3]);
+  }
+}
+
 /* Call the generator */
-generateAll();
+generateAll(run_limit);

testgen/generators/rdt_fmt_gen.js

@@ -12,6 +12,7 @@
 // Set up Node version to generate data specific to ICU/CLDR version
 // e.g., `nvm install 21.6.0;nvm use 21.6.0` (ICU 74)
 
+const common_fns = require("./common.js");
 const gen_hash = require("./generate_test_hash.js");
 
 const fs = require('node:fs');
@@ -48,22 +49,6 @@ const numeric = ['auto', 'always'];
 
 const counts = [-100, -4, -2, -1, 0, 1, 1.3, 2, 3, 4, 10];
 
-function sample_tests(all_tests, run_limit) {
-  // Gets a sampling of the data based on total and the expected number.
-
-  if (run_limit < 0 || all_tests.length <= run_limit) {
-    return all_tests;
-  }
-
-  let size_all = all_tests.length;
-  let increment = Math.floor(size_all / run_limit);
-  let samples = [];
-  for (let index = 0; index < size_all; index += increment) {
-    samples.push(all_tests[index]);
-  }
-  return samples;
-}
-
 // Create the test and verify JSON data for this case.
 function save_test(unit, count, locale, all_options, result, label_num,
                    test_cases, verify_cases) {
@@ -99,13 +84,13 @@ function save_test(unit, count, locale, all_options, result, label_num,
       console.log('   expected = ', result);
     }
   } catch (error) {
-    console.log('!!! error ', error, ' in label ', label_num);
+    console.log('!!! Problem pushing verify case. Error: ',
+                error, ' in label ', label_num);
   }
 
 }
 
 
-
 function generateAll() {
 
   let test_obj = {
@@ -166,7 +151,8 @@ function generateAll() {
           formatter_numeric_always =
               new Intl.RelativeTimeFormat(locale, all_options_numeric_always);
         } catch (error) {
-          console.log(error, ' with locale ',
+          console.log('Error creating RelativeTimeFormat: ',
+                      error, ' with locale ',
                       locale, ' and options: ', all_options_numeric_always);
           continue;
         }
@@ -182,7 +168,8 @@ function generateAll() {
         }
 
         if (debug) {
-          console.log("resolved options: ", formatter.resolvedOptions());
+          console.log("resolved options: ",
+                      formatter_numeric_auto.resolvedOptions());
         }
 
         for (const unit of units) {
@@ -222,19 +209,26 @@ function generateAll() {
   }
 
 
-  console.log('Number of relative date/time tests generated for ',
-              process.versions.icu, ': ', label_num);
-  console.log('  %d tests are different between numeric auto and always', diff_count);
+  if (debug) {
+    console.log('Number of relative date/time tests generated for ',
+                process.versions.icu, ': ', label_num);
+    console.log('  %d tests are different between numeric auto and always', diff_count);
+  }
 
-  test_obj['tests'] = sample_tests(test_cases, run_limit);
+  test_obj['tests'] = common_fns.sample_tests(test_cases, run_limit);
   try {
     fs.writeFileSync('rdt_fmt_test.json', JSON.stringify(test_obj, null, 2));
     // file written successfully
   } catch (err) {
     console.error(err);
   }
 
-  verify_obj['verifications'] = sample_tests(verify_cases, run_limit);
+  verify_obj['verifications'] = common_fns.sample_tests(verify_cases, run_limit);
+  if (debug) {
+    console.log('VERIFICATION COUNT: ', verify_obj['verifications'].length,
+                ' run_limit: ', run_limit);
+  }
+
   try {
     fs.writeFileSync('rdt_fmt_verify.json', JSON.stringify(verify_obj, null, 2));
     // file written successfully
@@ -243,11 +237,14 @@ function generateAll() {
   }
 }
 
-/* Call the generator */
+  if (debug) {
+    console.log('RDT_FMT argv: ', process.argv);
+  }
+
 let run_limit = -1;
-if (process.argv.length >= 5) {
-  if (process.argv[3] == '-run_limit') {
-    run_limit = Number(process.argv[4]);
+if (process.argv.length >= 4) {
+  if (process.argv[2] == '-run_limit') {
+    run_limit = Number(process.argv[3]);
   }
 }
 

testgen/generators/relativedatetime_fmt.py

@@ -22,9 +22,15 @@ def process_test_data(self):
             'icu71': '18.7.0',
         }
 
+        exec_list = ['node generators/rdt_fmt_gen.js']
+        if self.run_limit:
+            exec_list.append('-run_limit')
+            exec_list.append(str(self.run_limit))
+            print("RDTF generator: ", exec_list)
+
         run_list = [
             ['source ~/.nvm/nvm.sh; nvm install 21.6.0; nvm use 21.6.0 --silent'],
-            ['node generators/rdt_gen.js'],
+            exec_list,
             ['mv rdt_fmt*.json icu74']
         ]
 
@@ -34,7 +40,8 @@ def process_test_data(self):
 
         # Set up Node version and call the generator
         nvm_version = icu_nvm_versions[self.icu_version]
-        generate_command = 'source ~/.nvm/nvm.sh; nvm install %s; nvm use %s --silent; node generators/rdt_fmt_gen.js' % (nvm_version, nvm_version)
+        generate_command = 'source ~/.nvm/nvm.sh; nvm install %s; nvm use %s --silent; %s' %\
+                           (nvm_version, nvm_version, ' '.join(exec_list))
 
         logging.debug('Running this command: %s', generate_command)
         result = subprocess.run(generate_command, shell=True)