module index script

Author: pybites

module_index.py

@@ -0,0 +1,60 @@
+from collections import defaultdict, Counter
+import glob
+import os
+import re
+
+from stdlib import is_std_lib
+
+index = defaultdict(set)
+
+import_regex = re.compile('^(?:from|import)\s(?P<module>\w+).*')
+
+dirname = os.getcwd()
+
+
+def get_dirs():
+    for path in glob.glob('{}/[0-9]*'.format(dirname)):
+        yield path
+
+
+def get_files(path):
+    for fi in os.listdir(path):
+        if fi.endswith('.py'):
+            yield os.path.join(path, fi)
+
+
+def get_lines(src):
+    with open(scr) as f:
+        for line in f:
+            yield line
+
+
+if __name__ == '__main__':
+    for path in get_dirs():
+        day = os.path.basename(path)
+        for scr in get_files(path):
+            for line in get_lines(scr):
+                m = import_regex.match(line)
+                if m:
+                    mod = m.groupdict()['module']
+                    index[mod].add(day)
+
+    cnt = Counter()
+
+    for mod, scripts in sorted(index.items()):
+        if mod == 'common' or \
+            any(glob.glob(os.path.join(dirname, day, mod + '.py'))
+                for day in scripts):
+            source = 'own'
+        else:
+            source = 'stdlib' if is_std_lib(mod) else 'pypi'
+        cnt[source] += 1
+        appeared_in = ', '.join(sorted(scripts))
+        print(f'{mod:<12} | {source:<6} | {appeared_in}')
+
+    total = sum(cnt.values())
+    print()
+    for source, count in cnt.most_common():
+        print(f'{source:<10}: {count:>3} ({count/total*100:.1f}%)')
+    print('-' * 30)
+    print(f'Total: {total}')

stdlib.py

@@ -0,0 +1,35 @@
+'''http://stackoverflow.com/questions/22195382/how-to-check-if-a-module-library-package-is-part-of-the-python-standard-library'''
+from __future__ import unicode_literals, print_function
+import sys
+from contextlib import contextmanager
+from importlib import import_module
+
+
+@contextmanager
+def ignore_site_packages_paths():
+    paths = sys.path
+    # remove all third-party paths
+    # so that only stdlib imports will succeed
+    sys.path = list(filter(
+        None,
+        filter(lambda i: 'site-packages' not in i, sys.path)
+    ))
+    yield
+    sys.path = paths
+
+
+def is_std_lib(module):
+    if module in sys.builtin_module_names:
+        return True
+
+    with ignore_site_packages_paths():
+        imported_module = sys.modules.pop(module, None)
+        try:
+            import_module(module)
+        except ImportError:
+            return False
+        else:
+            return True
+        finally:
+            if imported_module:
+                sys.modules[module] = imported_module