check word sets against lexemes file

ci

@@ -13,6 +13,7 @@ case "${1:?}" in
         ci.d/lexemes_file_with_only_pysblgnt_lemmas "${RootDir:?}"/lexemes.yaml || S=1
         ci.d/lexemes_file_unicode_normalized "${RootDir:?}"/lexemes.yaml || S=1
         ci.d/word_sets_unicode_normalized "${RootDir:?}" || S=1
+        ci.d/word_sets_with_only_lexemes_from_lexemes_files "${RootDir:?}" "${RootDir:?}"/lexemes.yaml || S=1
         exit ${S:?}
         ;;
 esac

ci.d/word_set_with_only_lexemes_from_lexemes_files

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+import argparse
+import sys
+
+import yaml
+
+from morphgnt.utils import load_wordset
+
+
+def is_word_a_lexeme_in_lexemes_file(f, word, lexemes):
+    if word in lexemes:
+        return 0
+    else:
+        print("Word {} shall be removed from the set because not among lexemes in lexemes file.".format(word), file=f)
+        return 1
+
+
+f = sys.stderr
+e = 0 # exit status
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument("wordset", help="word set file")
+argparser.add_argument("lexemes", type=argparse.FileType('r'), help="lexemes file")
+
+args = argparser.parse_args()
+wordset = load_wordset(args.wordset)
+lexemes = yaml.load(args.lexemes)
+
+for word in wordset:
+    e = is_word_a_lexeme_in_lexemes_file(f, word, lexemes) or e
+
+sys.exit(e)

ci.d/word_sets_with_only_lexemes_from_lexemes_files

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+RootDir="${1:?}"
+LexemesFile="${2:?}"
+
+S=0
+for F in $(find "${RootDir:?}" -name "missing_*.txt"); do
+    echo Checking that word set in "${F:?}" contains only lexemes from the lexemes file "${LexemesFile:?}"...
+    "${RootDir}"/ci.d/word_set_with_only_lexemes_from_lexemes_files "${F:?}" "${LexemesFile:?}" || S=1
+done
+exit ${S:?}