Updated the scripts

synster · synster · commit 83f7b2d6815f · 2018-01-02T12:53:02.000+05:30
diff --git a/censys_subdomain_enum.py b/censys_subdomain_enum.py
@@ -0,0 +1,72 @@
+# A script to extract domain names from related SSL/TLS certificates using Censys
+# You'll need Censys API ID and API Secret to be able to extract SSL/TLS certificates
+# Needs censys module to run. pip install censys.
+
+from __future__ import print_function
+
+import logging
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(message)s"
+    )
+
+__author__  = "Bharath(github.com/yamakira)"
+__version__ = "0.1"
+__purpose__ = "Extract subdomains for a domain from censys certificate dataset"
+
+CENSYS_API_ID = ""
+CENSYS_API_SECRET = ""
+
+import argparse
+import re
+import sys
+
+try:
+    import censys.certificates
+except ImportError:
+    logging.info("\033[1;31m[!] Failed to import censys module. Run 'pip install censys'\033[1;m")
+    sys.exit()
+
+def get_certificates(domain):
+    if not CENSYS_API_ID or not CENSYS_API_SECRET:
+        logging.info("\033[1;31m[!] API KEY or Secret for Censys not provided.\033[1;m" \
+                     "\nYou'll have to provide them in the script") 
+        sys.exit()
+    logging.info("[+] Extracting certificates for {} using Censys".format(domain))
+    c = censys.certificates.CensysCertificates(CENSYS_API_ID, CENSYS_API_SECRET)
+    search_results = c.paged_search(domain)
+    certificates = search_results['results']
+    if len(certificates) == 0:
+        print("\033[1;31m[!] No matching certificates found!\033[1;m")
+        sys.exit()
+    return certificates
+
+def get_subdomains(domain, certificates):
+    logging.info("[+] Extracting sub-domains for {} from certificates".format(domain))
+    subdomains = []
+    for certificate in certificates:
+        parsed_result = re.findall(r'(?<=CN=).*', certificate[u'parsed.subject_dn'])
+        if len(parsed_result) > 0 and domain in parsed_result[0]: subdomains.append(parsed_result[0])
+    return subdomains
+
+def print_subdomains(subdomains):
+    unique_subdomains = list(set(subdomains))
+    print("\033[1;32m[+] Total unique subdomains found: {}\033[1;m".format(len(unique_subdomains)))
+    print("[+] List of subdomains extracted:\n")
+    for sub_domain in unique_subdomains:
+        print(sub_domain)
+
+def get_domain():
+    if len(sys.argv) < 2:
+        print("\n[!] Usage: python subdomain_enum_censys.py <target_domain>\n")
+        sys.exit()
+    else:
+        domain = sys.argv[1]
+        return domain
+
+if __name__ == '__main__':
+    domain = get_domain()
+    certificates = get_certificates(domain)
+    subdomains = get_subdomains(domain, certificates)
+    print_subdomains(subdomains)
diff --git a/cheatsheet.pdf b/cheatsheet.pdf
diff --git a/cloudflare_subdomain_enum.py b/cloudflare_subdomain_enum.py
@@ -0,0 +1,223 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Created using Metafidv2 by Matthew Bryant (mandatory)
+# Unauthorized use is stricly prohibited, please contact mandatory@gmail.com with questions/comments.
+from __future__ import print_function
+import requests
+import getpass
+import json
+import time
+import csv
+import sys
+import os
+from bs4 import BeautifulSoup
+
+class cloudflare_enum:
+    def __init__( self ):
+        # Master list of headers to be used in each connection
+        self.global_headers = {
+        }
+        self.verbose = True
+
+        self.s = requests.Session()
+        self.s.headers.update( self.global_headers )
+        self.atok = ''
+
+    def log_in( self, username, password ):
+        parse_dict = {}
+
+        r = self.s.get('https://www.cloudflare.com/', )
+
+        new_headers = {
+            'Referer': 'https://www.cloudflare.com/',
+        }
+        self.s.headers.update( dict( new_headers.items() + self.global_headers.items() ) )
+        r = self.s.get('https://www.cloudflare.com/a/login', )
+        parse_dict[ 'security_token_0' ] = self.find_between_r( r.text, '"security_token":"', '"}};</script>' ) # http://xkcd.com/292/
+
+        post_data = {
+            'email': username,
+            'password': password,
+            'security_token': parse_dict[ 'security_token_0' ],
+        }
+        new_headers = {
+            'Referer': 'https://www.cloudflare.com/a/login',
+            'Content-Type': 'application/x-www-form-urlencoded',
+        }
+        self.s.headers.update( dict( new_headers.items() + self.global_headers.items() ) )
+        r = self.s.post('https://www.cloudflare.com/a/login', data=post_data)
+        self.atok = self.find_between_r( r.text, 'window.bootstrap = {"atok":"', '","locale":"' ) # http://xkcd.com/292/
+
+    def get_domain_dns( self, domain ):
+        parse_dict = {}
+        post_data = {
+            "betas": [],
+            "created_on": "2015-08-24T00:27:16.048Z",
+            "development_mode": False,
+            "jump_start": True,
+            "meta": {},
+            "modified_on": 'null',
+            "name": domain,
+            "owner": {},
+            "paused": False,
+            "status": "initializing",
+            "type": "full"
+        }
+
+        new_headers = {
+            'Content-Type': 'application/json; charset=UTF-8',
+            'X-Requested-With': 'XMLHttpRequest',
+            'Referer': 'https://www.cloudflare.com/a/add-site',
+            'Pragma': 'no-cache',
+            'Cache-Control': 'no-cache',
+            'X-ATOK': self.atok,
+        }
+        self.s.headers.update( dict( new_headers.items() + self.global_headers.items() ) )
+        r = self.s.post('https://www.cloudflare.com/api/v4/zones', data=json.dumps( post_data ))
+        data = json.loads( r.text )
+        success = data['success']
+        if not success:
+            print( r.text )
+            return False
+
+        request_id = data['result']['id']
+        time.sleep( 60 )
+
+        get_data = {
+            'per_page': '100',
+            'direction': 'asc',
+            'page': '1',
+            'order': 'type',
+        }
+        new_headers = {
+            'X-Requested-With': 'XMLHttpRequest',
+            'Referer': 'https://www.cloudflare.com/a/setup/' + domain + '/step/2',
+            'X-ATOK': self.atok,
+        }
+        self.s.headers.update( dict( new_headers.items() + self.global_headers.items() ) )
+        r = self.s.get('https://www.cloudflare.com/api/v4/zones/' + request_id + '/dns_records', params=get_data)
+        return_data = json.loads( r.text )
+
+        new_headers = {
+            'X-Requested-With': 'XMLHttpRequest',
+            'Referer': 'https://www.cloudflare.com/a/setup/' + domain + '/step/2',
+            'X-ATOK': self.atok,
+        }
+        self.s.headers.update( dict( new_headers.items() + self.global_headers.items() ) )
+        r = self.s.delete('https://www.cloudflare.com/api/v4/zones/' + request_id, )
+
+        get_data = {
+            'status': 'initializing,pending',
+            'per_page': '50',
+            'page': '1',
+        }
+        new_headers = {
+            'X-Requested-With': 'XMLHttpRequest',
+            'Referer': 'https://www.cloudflare.com/a/add-site',
+            'X-ATOK': self.atok,
+        }
+        self.s.headers.update( dict( new_headers.items() + self.global_headers.items() ) )
+        r = self.s.get('https://www.cloudflare.com/api/v4/zones', params=get_data)
+
+        return return_data['result']
+
+    def get_spreadsheet( self, domain ):
+        dns_data = self.get_domain_dns( domain )
+        if dns_data:
+            filename = domain.replace( ".", "_" ) + ".csv"
+
+            with open( filename, 'wb' ) as csvfile:
+                dns_writer = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
+                dns_writer.writerow( [ "name", "type", "content" ] )
+                for record in dns_data:
+                    dns_writer.writerow( [ record["name"], record["type"], record["content"] ] )
+                
+            self.statusmsg( "Spreadsheet created at " + os.getcwd() + "/" + filename )
+
+    def print_banner( self ):
+        if self.verbose:
+            print("""
+            
+                                                     `..--------..`                               
+                                                 .-:///::------::///:.`                           
+                                              `-//:-.`````````````.-://:.`    `   `               
+                                            .://-.```````````````````.-://-`  :  `-   .           
+                                          `-//:.........................-://. /. -: `:`  ``       
+                                         `://--------:::://////:::--------://-::.::`:- .:.        
+                              ``.---..` `///::::::///////////////////:::::::///::::::--:.`.-.     
+                            .://::::///::///::///////////////////////////:::///:-----::--:-`  `    
+                          `:/:-...--:://////////////////////////////////////////----------.--.`    
+                         `:/:..-:://////////////////////////////////////////////-----------.````    
+                         .//-::////////////////////////////////////:::::////////-...--------...`    
+                         -/////////////////////////////////////////////::::----:. `.-::::::-..``    
+                    ``.--:////////////////////////////////////////////////::-..```-///::::///:-`    
+                 `.:///::::://////////////////////////////////////:::::::::::::::-----......-:/:.    
+               `-//:-----::::://///////////////////////////////:///////////////////:-::::---..-//:`    
+              `:/:---://+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//+++//::--//:    
+             `//:-/+oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo+++oooo+//://.    
+             :///ossssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssosssssso+//:    
+            `//+sssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss+/-    
+            `//+ooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo+++++/.    
+             ``````````````````````````````````````````````````````````````````````````````````````     
+                                                             Cloudflare DNS Enumeration Tool v1.3
+                                                                             Created by mandatory
+                                                                             Modified by yamakira
+        """ )
+
+    def pprint( self, input_dict ):
+        print( json.dumps(input_dict, sort_keys=True, indent=4, separators=(',', ': ')) )
+
+    def statusmsg( self, msg ):
+        if self.verbose:
+            print( "[ STATUS ] " + msg )
+
+    def errormsg( self, msg ):
+        if self.verbose:
+            print( "[ ERROR ] " + msg )
+
+    def successmsg( self, msg ):
+        if self.verbose:
+            print( "[ SUCCESS ] " + msg )
+
+    def find_between_r( self, s, first, last ):
+        try:
+            start = s.rindex( first ) + len( first )
+            end = s.rindex( last, start )
+            return s[start:end]
+        except ValueError:
+            return ""
+
+    def find_between( s, first, last ):
+        try:
+            start = s.index( first ) + len( first )
+            end = s.index( last, start )
+            return s[start:end]
+        except ValueError:
+            return ""
+
+    def get_cookie_from_file( self, cookie_file ):
+        return_dict = {}
+        with open( cookie_file ) as tmp:
+            data = tmp.readlines()
+            tmp_data = []
+            for i, item in enumerate(data):
+                if "	" in data[i]:
+                    pew = data[i].split( "	" )
+                    return_dict[ pew[5] ] = pew[6] 
+
+        return return_dict
+
+    def get_creds(self):
+        username = sys.argv[1]
+        password = getpass.getpass('Provide your cloudflare password:')
+        return username,password 
+
+if __name__ == "__main__":
+    if len( sys.argv ) < 2:
+        print( "Usage: " + sys.argv[0] + " username@email.com domain.com" )
+    else:
+        cloud = cloudflare_enum()
+        username,password = cloud.get_creds()
+        cloud.print_banner()
+        cloud.log_in(username,password)
+        cloud.get_spreadsheet(sys.argv[2])
diff --git a/crtsh_enum_psql.py b/crtsh_enum_psql.py
@@ -0,0 +1,52 @@
+from __future__ import print_function
+
+__author__ = 'Bharath'
+__version__ = "0.1.0"
+
+try:
+    import psycopg2
+except ImportError:
+    raise ImportError('\n\033[33mpsycopg2 library missing. pip install psycopg2\033[1;m\n')
+    sys.exit(1)
+import re
+import sys
+
+DB_HOST = 'crt.sh'
+DB_NAME = 'certwatch'
+DB_USER = 'guest'
+
+def connect_to_db(domain_name):
+    try:
+        conn = psycopg2.connect("dbname={0} user={1} host={2}".format(DB_NAME, DB_USER, DB_HOST))
+        cursor = conn.cursor()
+        cursor.execute("SELECT ci.NAME_VALUE NAME_VALUE FROM certificate_identity ci WHERE ci.NAME_TYPE = 'dNSName' AND reverse(lower(ci.NAME_VALUE)) LIKE reverse(lower('%{}'));".format(domain_name))
+    except:
+        print("\n\033[1;31m[!] Unable to connect to the database\n\033[1;m")
+    return cursor
+
+def get_unique_domains(cursor, domain_name):
+    unique_domains = []
+    for result in cursor.fetchall():
+        matches=re.findall(r"\'(.+?)\'",str(result))
+        for subdomain in matches:
+            if subdomain not in unique_domains:
+                if ".{}".format(domain_name) in subdomain:
+                    unique_domains.append(subdomain)
+    return unique_domains
+
+def print_unique_domains(unique_domains):
+    for unique_domain in sorted(unique_domains):
+        print(unique_domain)
+
+def get_domain_name():
+    if len(sys.argv) <= 1:
+        print("\n\033[33mUsage: python crtsh_enum_psql.py <target_domain>\033[1;m\n")
+        sys.exit(1)
+    else:
+        return sys.argv[1]
+
+if __name__ == '__main__':
+    domain_name = get_domain_name()
+    cursor = connect_to_db(domain_name)
+    unique_domains = get_unique_domains(cursor, domain_name)
+    print_unique_domains(unique_domains)
diff --git a/crtsh_enum_psql.sh b/crtsh_enum_psql.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+query="SELECT ci.NAME_VALUE NAME_VALUE FROM certificate_identity ci WHERE ci.NAME_TYPE = 'dNSName' AND reverse(lower(ci.NAME_VALUE)) LIKE reverse(lower('%.$1'));"
+
+echo $query | \
+    psql -t -h crt.sh -p 5432 -U guest certwatch | \
+    sed -e 's:^ *::g' -e 's:^*\.::g' -e '/^$/d' | \
+    sort -u | sed -e 's:*.::g'
diff --git a/crtsh_enum_web.py b/crtsh_enum_web.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+
+import sys
+import urllib.request
+import urllib.parse
+import re
+
+
+if len(sys.argv) == 1:
+	print("Usage: " + sys.argv[0] + " [domain] ...")
+	sys.exit(1)
+
+for i, arg in enumerate(sys.argv, 1):
+	domains = set()
+	with urllib.request.urlopen('https://crt.sh/?q=' + urllib.parse.quote('%.' + arg)) as f:
+		code = f.read().decode('utf-8')
+		for cert, domain in re.findall('<tr>(?:\s|\S)*?href="\?id=([0-9]+?)"(?:\s|\S)*?<td>([*_a-zA-Z0-9.-]+?\.' + re.escape(arg) + ')</td>(?:\s|\S)*?</tr>', code, re.IGNORECASE):
+			domain = domain.split('@')[-1]
+			if not domain in domains:
+				domains.add(domain)
+				print(domain)
diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1,13 @@
+requests==2.18.4
+asn1crypto==0.24.0
+cffi==1.11.2
+cryptography==2.1.4
+enum34==1.1.6
+idna==2.6
+ipaddress==1.0.19
+pycparser==2.18
+pyOpenSSL==17.5.0
+six==1.11.0
+censys==0.0.8
+psycopg2==2.7.3.2
+beautifulsoup4
diff --git a/san_subdomain_enum.py b/san_subdomain_enum.py
diff --git a/virustotal_subdomain_enum.py b/virustotal_subdomain_enum.py
