添加sqlite3

lyy289065406 · lyy289065406 · commit 22080b0ef5c4 · 2020-04-29T22:51:16.000+08:00
diff --git a/.idea/sqldialects.xml b/.idea/sqldialects.xml
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
diff --git a/cache/.empty b/cache/.empty
diff --git a/cache/360.dat b/cache/360.dat
@@ -1,6 +1,6 @@
-4ccd2f7a40228689ffa2e53035851b06
-30770b38066bd2dbceade9d6d8ba0dab
-57413b74c5b4624e1622fdfa9aec9da5
-a4ad6639ce81b49f8ccd41b2aff66e5e
-30611411a0eaad9f43a789588b85e311
-013b94c71aa9b1c5ed302cfe4426b8ca
+027b922fc3cae6e7ca0a8f436b2b86e4
+ee3dce97c903a8f3301eeb3a36c6da95
+898b5097042343c9f13f1849e8357954
+d745dfd09efb36e3f5aa25dc0d21819a
+b4f7c75bde57c131d627d028f313a138
+87435d97a0feffa097f176bb4ece709f
diff --git a/cache/AnQuanKe.dat b/cache/AnQuanKe.dat
@@ -1,40 +1,40 @@
-b084ccbd439974403825d18c63e3fae3
-8dcb656dcea4ec66f08dc347bee44bf5
-073892da5946de068315b2724c2ed283
-c7a405e57fee82beb686a7f1a364b429
-120664e642bb633ce35e80c5476af698
-8a29afa24fbc9f59d3a77fbb67aa2734
-8a351166e098f54e24fa96b2ad65f53f
-d32ccb1a7cab23af55b1abb6f397064c
-14f6e137054325f82209bb2147bcb7d9
-e31a7e1a598caedf5a67b6072b301120
-30af3c790212245f56b6db69c9496467
-c13d8deb81f8bf22fd4161923549ab36
-c5c1454b9e08844d78d39e65099855a0
-9dcc877d239e772317507ceed25cd237
-8f5c143438cec05f0508bd066b1866d0
-bad4bf2c02ae29fefca48b8e4bd6f02d
-325d3ca07f8085654d011c07b0a24284
-ba56ba9d840015ac5df74ceffef51508
-3566d5b8cf11860bd17ec3cf338bacda
-c937babd5350684c08fccb9a497bc7d8
-233e979b0265647f7ee70310f50d33e9
-2386e98272fd68923526e92069010d93
-f3957fca65e210631854065c6293ff42
-cbd6743f889954b70d94d9220e28d03f
-b221cbb1f195abab906a99b48509565b
-a3cc1b1c177341e1cc0ef1d9c0d3675d
-434d23a99a816e20357eb7fc547247f8
-401ba59bbd29f6791549cb42e7aded53
-8a8ed4834d1da33a49710c3ce6b0c789
-95176090c6fefd88a79089d046e1d504
-9357105eab64e6ab84881dcf91b1c6d6
-160781ebaad034f6abd6217c77a90026
-7c8f8b82fa11089201addca65489fc79
-1d54ae73b2cedc16859e086de39f39ae
-c58319ae8c82cd319844ff2cc58d9088
-46a675b4cbc19821374c0f7b49ca502e
-801e78aa22358f92a4ae882220452f1f
-71b213d87ce3edc79116890ea205c4f2
-35aee40182eb7a989c4073a997dd31bc
-efd98fa539fe9003b6241fcb2d9d9030
+29bca901bad8eca0b4b4c815b2942b56
+539cd05def97df85d730dd3a2463f6cf
+33415330cf5956d4d18befa78eae760f
+dc11dd0c613a5961199a54639fba53dc
+2a2755eefc47df5357c41a5b54dd67bf
+d2b1b8c3ccf1e0ffb3270b0b7b21998c
+7b999a39331f1bbeb330da5b8be79842
+dccb882984075e8ae693498b8d33373c
+0d85245523b6646833692af57dd1daa9
+d0f03f90358904908ac07a397ce0ad19
+c3b1595f953bbaf987205c725311515d
+818f596c36829f92e95dad79d91ea421
+3a93ec68e70def4c66d1abb4a40834c6
+c77677bd58cb715055ddca164987ce73
+3fa2df6d799d957980b3a1031da55c35
+222cb3f21218d6696a6a6292d1e184eb
+80f737863fe2e089cc9b111bf793ab99
+8365c0f058368b9401f49587a0caacc3
+ea2a851918a05a26b84ca68a4cac9ec7
+1ee83e405398d407cee708cd8a38f283
+a06545c2c654f33d9d258ce7fc2d2281
+f44af5f92c3858c60725f6d7d144c9e4
+a8fb31c48accd24267c84332a90a5f40
+b2b94426ce14f482042c031c45453983
+bc9241f17f76fb06bafbea7c02b1c558
+a6066f25e255deddaddb2434ada150ea
+b0a333126f61d2c5d843215dd6c8b958
+27d73ef57d41324faf3e857eb5a95739
+3dba915c4ff432b3e899e226d3687c9d
+d04df9fbce7c3021d3ec734d8aefd4f7
+f1fb89bec43e0c444a3570cab759e8bd
+1edf6b5faf88c95e12682afebe3313be
+84df6a6fc403d81658b92d2c97c4fa5f
+2d7e41d776d0058883ad129184a3adba
+0ea15592705b49799593b837912421bc
+bb25027d4d700f932d71f381753c4301
+713902d222bf9bc48b1b09f4276bfc59
+2d7a6ed87d953792f49711f6c9eef0b1
+13dd04d90771316a8e0ebed08c451fa6
+5faedc6e8386d63e7dd0091a518d074b
diff --git a/cache/Nsfocus.dat b/cache/Nsfocus.dat
@@ -1,15 +1,15 @@
-fa8583e7ae5a08302ab70ce90689ba58
-73e6c42b6ea4388547d14fd2d1163380
-302398a5754637d0cdc6acc34b5d3edf
-3676f481649a95d2cd7031edac2bd80b
-ecb1ead92ef563ca25f4372932664e5b
-6b35b527f6751cc0bbc1cf3082fd3067
-792a52f437fc90b406d3cfb269ca336f
-e16013573edcad2f8d909ab5d0d9d9bc
-56ea3c64d3a1518d7c3a717576e4bd4e
-11601eedbafa99cf863ec4c2d4ef839e
-390e46bdf3b47bfedd7515385a62320a
-9e3498f9f2c022c5bb0a9f02ae15fb4e
-c8de1f8ad0d5da5ac7016ee2f0d291db
-59d1cb2aac335119e63588567b17592f
-f5b3de3a245859b62cff9c3b181ea460
+a9f728e62fc2b23012533d909da0126f
+24119c5854ad964706613944d1511592
+416a6f68c779a9e1360fe7ee1c1a7ffd
+0ecbbfa5c39c3596cb325edbf473fd1b
+5c8932d357b5c4df7b2183e6286ec55d
+94753240a9f7cdcb6a0c5b31703ce13e
+030f70c142bff76dc9f10711f86c3f59
+0ce59b5a0aac23cea9dcb195099128f6
+df6d29f11e4c311a92ff4ed0661ec9f1
+f4f0701b017670b1d004ee8f5533c6d6
+531e46762b7f305584f32e6c741340c4
+8324b54fb45223aac86405c4c933cc39
+e4ca9772e6ef5b0a8ef8d9e495ee5740
+62f57cb8731199801080424db40eb70c
+b011978355c3aa91754a127663db1399
diff --git a/cache/QiAnXin.dat b/cache/QiAnXin.dat
@@ -1,10 +1,10 @@
-90b93cb7073fe73b17746ac166a09637
-e318a5efa4803b50cdef480b90b1784d
-cffc3035f7899495cfeae521451f91b2
-3e6175d47d17c6f94bd9ba10d81c3717
-d99d073afb7d248a8a62fb068921997f
-b7b45b14a3af1225ef6eec72d74964df
-504fc79f0123db109a11b149c334b75c
-5b727692d583d4a6e7cdb0f670eac12a
-54b48d765fccbc8dcfa3de0920459f8d
-5b4d5fea09fbc2dca45be53f162d39de
+ec11de324a43e3f6ce1984cd4a4f0540
+bb0c0ee209d91a9547ba1382663b675d
+34cca1697a489fd01f475cde95e5b79d
+e275a418260736a4b42c39b8a93680a4
+02ee42adbb25c0a556d787b5d0ce2207
+a904e7c4b3701901693ed23f17c3233e
+f9238620633659d1338b46aab96d29ca
+95dbb7aca0383535115e125ac5cc5ddf
+55f52656d300ae00d3f9f581b8cf5898
+143db827d8aed61603c8d96f79a3744c
diff --git a/cache/RedQueen.dat b/cache/RedQueen.dat
@@ -1,10 +1,10 @@
-27116b86fcb0ba678cc0b792f6b2e99d
-fb4f61b831968a765ff68bde1f4c373a
-6e59bc98e0626e2ce7a88d8432d51111
-3888601684d7f26f41e3fe54f9082864
-616638911973132683942339ab7b1c0d
-42b40b2496baa5ad4a0bc54e694c77a5
-4ee90b66222a18a5b0a927c87b2abec8
-cffc9a5293606d33e221e27e7b1a8932
-f0de0fb4f82e14ac52d71bafa71a5543
-2cfdcfdf5b3d04a08396a5ce4807288f
+eefacf81ef9f632df900a7ef6c0444b0
+fbe158e119496d55f85694b73b535aaa
+5bdebad01518b7c9fcac8e88fb320ce1
+b473db5e054009888c5e612dd38598ca
+56656a3fbb040cf9dc09f4a49cd29346
+d7d611fbf6528627b4c143e104cb0842
+b82edc0cd73b844f0df5395bd6489708
+6fdc5025e0d39ed3d25bd670ea30f4d0
+82317c027376d876ce577d714c41c9ed
+f75b85d4fe32395f4ea77a27f33c36d0
diff --git a/cache/vas.dat b/cache/vas.dat
@@ -1,5 +1,5 @@
-faa5de1bb89669fb071e637486ce1226
-262557113cf7c8f230d66347b7c13095
-62445307db0fa2eaea15de0fe87244f1
-49ed8eb720e2789ac5628199620acfc5
-66cb99b686cd672c59a987932426f2a2
+b00f087b3c4b01bb74b99436d2357fcf
+63bb5a251524834418d4bbc032978e37
+c74b79538fc02baf184fa5fca614f824
+3b890e303e116e80ab58be97f9e946ce
+22458f5182f4713d9fdf5b327e05487b
diff --git a/data/cves.db b/data/cves.db
diff --git a/log/.empty b/log/.empty
@@ -1,4 +0,0 @@
-# Copyright (C)
-# Author: EXP
-# Site  : http://exp-blog.com
-# Mail  : 272629724@qq.com
diff --git a/script/cves-create.sql b/script/cves-create.sql
@@ -0,0 +1,9 @@
+CREATE TABLE IF NOT EXISTS `t_cves` (
+  `s_md5`  TEXT(64),
+  `s_src`  TEXT(32),
+  `s_cves`  TEXT(128),
+  `s_title`  TEXT(256),
+  `s_time`  TEXT(20),
+  `s_info`  TEXT(2048),
+  `s_url`  TEXT(256)
+);
diff --git a/script/cves-rollback.sql b/script/cves-rollback.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS `t_cves`;
diff --git a/src/bean/cve_info.py b/src/bean/cve_info.py
@@ -27,7 +27,7 @@ def is_vaild(self):
 
     def MD5(self):
         if not self.md5:
-            data = '%s%s%s' % (self.id, self.title, self.url)
+            data = '%s%s%s%s' % (self.id, self.title, self.time, self.url)
             self.md5 = hashlib.md5(data.encode(encoding='UTF-8')).hexdigest()
         return self.md5
 
diff --git a/src/cfg/env.py b/src/cfg/env.py
@@ -7,4 +7,9 @@
 
 import os
 
+CHARSET = 'utf-8' \
+          ''
 PRJ_DIR = os.path.dirname(os.path.abspath(__file__)).replace(r'/src/cfg', '').replace(r'\src\cfg', '')
+
+SQL_PATH = '%s/script/cves-create.sql' % PRJ_DIR
+DB_PATH =  '%s/data/cves.db' % PRJ_DIR
diff --git a/src/main.py b/src/main.py
@@ -6,7 +6,9 @@
 # -----------------------------------------------
 
 import sys
+from src.cfg import env
 from src.utils import log
+from src.utils.sdbc import SqliteSDBC
 
 from src.crawler.cert360 import Cert360
 from src.crawler.nsfocus import Nsfocus
@@ -18,9 +20,14 @@
 
 def init():
     reload(sys)
-    sys.setdefaultencoding('utf-8')
+    sys.setdefaultencoding(env.CHARSET)
+
     log.init()
 
+    sdbc = SqliteSDBC(env.DB_PATH)
+    sdbc.init(env.SQL_PATH)
+
+
 
 def main(a, b, c):
     srcs = [ Cert360(), Nsfocus(), QiAnXin(), RedQueen(), AnQuanKe(), Vas() ]
@@ -60,9 +67,5 @@ def get_sys_args(sys_args) :
     init()
     main(*get_sys_args(sys.argv))
 
-    # import sqlite3
-    #
-    # conn = sqlite3.connect('test.db')
-    #
-    # print "Opened database successfully"
+
 
diff --git a/src/utils/sdbc.py b/src/utils/sdbc.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Author : EXP
+# @Time   : 2020/4/29 22:14
+# @File   : sdbc.py
+# -----------------------------------------------
+# Sqlite 数据库接口
+# -----------------------------------------------
+
+from src.utils import log
+import sqlite3
+
+
+class SqliteSDBC:
+    """
+    Sqlite 数据库封装类
+    """
+
+    def __init__(self, dbpath='test.db'):
+        """
+        构造函数
+        :param dbname: 数据库路径
+        """
+        self.dbpath = dbpath
+        self._conn = None
+
+
+    def conn(self):
+        """
+        连接到数据库
+        :return: 数据库连接（失败返回 None）
+        """
+        if not self._conn:
+            try:
+                self._conn = sqlite3.connect(
+                    database = self.dbpath
+                )
+            except:
+                log.error("连接数据库 [%s] 失败" % self.dbpath)
+        return self._conn
+
+
+    def close(self):
+        """
+        断开数据库连接
+        :return: 是否断开成功
+        """
+        is_ok = False
+        if self._conn:
+            try:
+                self._conn.close()
+                self._conn = None
+                is_ok = True
+            except:
+                log.error("断开数据库 [%s] 失败" % self.dbpath)
+        return is_ok
+
+
+    def reconn(self):
+        """
+        重连数据库
+        :return: 数据库连接（失败返回 None）
+        """
+        self.close()
+        return self.conn()
+
+
+    def commit(self):
+        """
+        提交事务
+        :return: 是否提交成功
+        """
+        is_ok = False
+        if self._conn:
+            try:
+                self._conn.commit()
+                is_ok = True
+            except:
+                log.error("提交事务到数据库 [%s] 失败" % self.dbpath)
+        return is_ok
+
+
+    def init(self, sql_script):
+        """
+        初始化数据库
+        :param sql_script: 建库脚本（注意控制好数据表是否存在）
+        :return: 是否初始化成功
+        """
+        if self.conn():
+            try:
+                data = ""
+                with open(sql_script, "r") as file:
+                    data = file.read()
+
+                if data:
+                    cursor = self._conn.cursor()
+                    sqls = data.split(";")
+                    for sql in sqls:
+                        sql = sql.strip()
+                        if sql:
+                            cursor.execute(sql)
+                    cursor.close()
+            except:
+                log.error("初始化数据库 [%s] 失败" % self.dbpath)
+            self.close()
+
