'fixed' the dataparser so it will connect to mssql from a unix machin…

…e. Also updated a bunch of html stuff for #102

c:\inetpub\wwwroot\mdf\log__main__.log

@@ -0,0 +1,10 @@
+2018-03-07 16:26:00,136 - INFO - __main__.parse_data() (126): 
+========================================================
+
+2018-03-07 16:26:00,137 - INFO - __main__.parse_data() (127): Creating data files.
+2018-03-07 16:26:00,137 - INFO - __main__.load_watershed_data() (81): Getting sites for iUTAH_Logan_OD
+2018-03-08 10:45:04,285 - INFO - __main__.parse_data() (126): 
+========================================================
+
+2018-03-08 10:45:04,285 - INFO - __main__.parse_data() (127): Creating data files.
+2018-03-08 10:45:04,285 - INFO - __main__.load_watershed_data() (81): Getting sites for iUTAH_Logan_OD

src/DataParser/DataParser.py

@@ -13,6 +13,7 @@
 from odmservices import ServiceManager
 from logger import LoggerTool
 
+
 tool = LoggerTool()
 logger = tool.setupLogger(__name__, __name__ + '.log', 'a', logging.DEBUG)
 
@@ -74,14 +75,25 @@ def load_watershed_data(watershed_database):
         'user': 'webapplication',
         'password': 'W3bAppl1c4t10n!',
         'address': 'iutahdbs.uwrl.usu.edu',
+        'port': 1443,
         'db': watershed_database
     }
 
+    # service_manager._current_connection = {
+    #     'engine': 'mysql',
+    #     'user': 'root',
+    #     'password': 'password',
+    #     'address': 'localhost',
+    #     'port': 1443,
+    #     'db': watershed_database
+    # }
+
     watershed = {}
     logger.info("Getting sites for %s" % watershed_database)
     series_service = service_manager.get_series_service()
 
     sites = series_service.get_all_sites()
+
     raw_qc_level_id = series_service.get_raw_qc_level_id()
 
     for site in sites:
@@ -143,8 +155,53 @@ def parse_data():
 
 
 def write_json(watershed_dictionary, watershed_name):
-    with open(os.path.join('%s', '%sSite.json') % (static_folder, watershed_name), "w") as out_file:
+    if sys.platform != 'win32':
+        path = os.path.join(os.getcwd(), os.pardir, 'mdfserver', 'static', 'mdfserver', 'json', '%sSite.json' % watershed_name)
+        path = os.path.realpath(path)
+    else:
+        path = os.path.join('%s', '%sSite.json') % (static_folder, watershed_name)
+    with open(path, "w") as out_file:
         json.dump(watershed_dictionary, out_file, indent=4)
 
 
+def initialize_database():
+    """
+    Run this method for first time setup of database. Creates database(s) based on the models in the odmdata folder.
+
+    For some reason, which we may never know, you need to add connection information in a file named
+    'connection.config', located in a directory determined by 'odmservices.utilities.util.resource_path()'.
+
+    To add a database connection info, add a line in 'connection.config' in the format of:
+        <engine> <user_name> <password> <address> <database_name>
+
+    Example of 'connection.config' with three connections:
+    ================================================
+    mysql root password localhost iUTAH_Logan_OD
+    mysql root password localhost iUTAH_RedButte_OD
+    mssql admin_1 soopersekretpaswurd database.somewhere.else.com iUTAH_Provo_OD
+    ================================================
+
+    Yup, pretty genius.
+
+    My sincerest hope is this project will burn to the ground before anyone else might find this comment useful.
+    """
+    for conn in service_manager.get_connections():
+        if service_manager.database_exists(conn_dict=conn):
+            continue
+
+        try:
+            service_manager.create_database(conn_dict=conn)
+
+            try:
+                service_manager.create_tables(conn_dict=conn)
+            except Exception:
+                raise Exception("Failed to create tables for database '{0}'".format(conn['db']))
+
+        except Exception:
+            raise Exception("Failed to create database '{0}'".format(conn.get('db', 'N/A')))
+
+
+# initialize_database()
+
 parse_data()
+

src/DataParser/odmdata/__init__.py

@@ -63,4 +63,4 @@
     'MemoryDatabase',
     'copy_series',
     'copy_data_value'
-]
+]

src/DataParser/odmdata/base.py

@@ -1,2 +1,13 @@
 from sqlalchemy.ext.declarative import declarative_base
-Base = declarative_base()
+Base = declarative_base()
+
+
+def create_database(engine, db_name):
+    engine.execute("CREATE DATABASE %s" % db_name)
+
+
+def create_tables(engine):
+    try:
+        Base.metadata.create_all(engine)
+    except Exception as e:
+        print(e)

src/DataParser/odmdata/censor_code_cv.py

@@ -6,8 +6,8 @@
 class CensorCodeCV(Base):
     __tablename__ = 'CensorCodeCV'
 
-    term = Column('Term', String, primary_key=True)
-    definition = Column('Definition', String)
+    term = Column('Term', String(255), primary_key=True)
+    definition = Column('Definition', String(255))
 
     def __repr__(self):
         return "<CensorCodeCV('%s', '%s')>" % (self.term, self.definition)

src/DataParser/odmdata/data_type_cv.py

@@ -6,8 +6,8 @@
 class DataTypeCV(Base):
     __tablename__ = 'DataTypeCV'
 
-    term = Column('Term', String, primary_key=True)
-    definition = Column('Definition', String)
+    term = Column('Term', String(255), primary_key=True)
+    definition = Column('Definition', String(255))
 
     def __repr__(self):
         return "<DataTypeCV('%s', '%s')>" % (self.term, self.definition)

src/DataParser/odmdata/data_value.py

@@ -47,7 +47,7 @@ class DataValue(Base):
     variable_id = Column('VariableID', Integer, ForeignKey('Variables.VariableID'), nullable=False)
     offset_value = Column('OffsetValue', Float)
     offset_type_id = Column('OffsetTypeID', Integer, ForeignKey('OffsetTypes.OffsetTypeID'))
-    censor_code = Column('CensorCode', String)
+    censor_code = Column('CensorCode', String(255))
     qualifier_id = Column('QualifierID', Integer, ForeignKey('Qualifiers.QualifierID'))
     method_id = Column('MethodID', Integer, ForeignKey('Methods.MethodID'), nullable=False)
     source_id = Column('SourceID', Integer, ForeignKey('Sources.SourceID'), nullable=False)

src/DataParser/odmdata/general_category_cv.py

@@ -6,8 +6,8 @@
 class GeneralCategoryCV(Base):
     __tablename__ = 'GeneralCategoryCV'
 
-    term = Column('Term', String, primary_key=True)
-    definition = Column('Definition', String)
+    term = Column('Term', String(255), primary_key=True)
+    definition = Column('Definition', String(255))
 
     def __repr__(self):
         return "<GeneralCategoryCV('%s', '%s')>" % (self.term, self.definition)

src/DataParser/odmdata/iso_metadata.py

@@ -7,11 +7,11 @@ class ISOMetadata(Base):
     __tablename__ = 'ISOMetadata'
 
     id = Column('MetadataID', Integer, primary_key=True)
-    topic_category = Column('TopicCategory', String, nullable=False)
-    title = Column('Title', String, nullable=False)
-    abstract = Column('Abstract', String, nullable=False)
-    profile_version = Column('ProfileVersion', String, nullable=False)
-    metadata_link = Column('MetadataLink', String)
+    topic_category = Column('TopicCategory', String(255), nullable=False)
+    title = Column('Title', String(255), nullable=False)
+    abstract = Column('Abstract', String(255), nullable=False)
+    profile_version = Column('ProfileVersion', String(255), nullable=False)
+    metadata_link = Column('MetadataLink', String(255))
 
     def __repr__(self):
         return "<ISOMetadata('%s', '%s', '%s')>" % (self.id, self.topic_category, self.title)

src/DataParser/odmdata/lab_method.py

@@ -7,11 +7,11 @@ class LabMethod(Base):
     __tablename__ = 'LabMethods'
 
     id = Column('LabMethodID', Integer, primary_key=True)
-    name = Column('LabName', String, nullable=False)
-    organization = Column('LabOrganization', String, nullable=False)
-    method_name = Column('LabMethodName', String, nullable=False)
-    method_description = Column('LabMethodDescription', String, nullable=False)
-    method_link = Column('LabMethodLink', String)
+    name = Column('LabName', String(255), nullable=False)
+    organization = Column('LabOrganization', String(255), nullable=False)
+    method_name = Column('LabMethodName', String(255), nullable=False)
+    method_description = Column('LabMethodDescription', String(255), nullable=False)
+    method_link = Column('LabMethodLink', String(255))
 
     def __repr__(self):
         return "<LabMethod('%s', '%s', '%s', '%s')>" % (self.id, self.name, self.organization, self.method_name)

src/DataParser/odmdata/method.py

@@ -7,8 +7,8 @@ class Method(Base):
     __tablename__ = 'Methods'
 
     id = Column('MethodID', Integer, primary_key=True)
-    description = Column('MethodDescription', String, nullable=False)
-    link = Column('MethodLink', String)
+    description = Column('MethodDescription', String(255), nullable=False)
+    link = Column('MethodLink', String(255))
 
     def __repr__(self):
         return "<Method('%s', '%s', '%s')>" % (self.id, self.description, self.link)

src/DataParser/odmdata/odm_version.py

@@ -6,7 +6,7 @@
 class ODMVersion(Base):
     __tablename__ = 'ODMVersion'
 
-    version_number = Column('VersionNumber', String, primary_key=True)
+    version_number = Column('VersionNumber', String(255), primary_key=True)
 
     def __repr__(self):
         return "<ODMVersion('%s')>" % (self.version_number)

src/DataParser/odmdata/offset_type.py

@@ -10,7 +10,7 @@ class OffsetType(Base):
 
     id = Column('OffsetTypeID', Integer, primary_key=True)
     unit_id = Column('OffsetUnitsID', Integer, ForeignKey('Units.UnitsID'), nullable=False)
-    description = Column('OffsetDescription', String)
+    description = Column('OffsetDescription', String(255))
 
     # relationships
     unit = relationship(Unit)

src/DataParser/odmdata/qualifier.py

@@ -7,8 +7,8 @@ class Qualifier(Base):
     __tablename__ = 'Qualifiers'
 
     id = Column('QualifierID', Integer, primary_key=True)
-    code = Column('QualifierCode', String, nullable=False)
-    description = Column('QualifierDescription', String, nullable=False)
+    code = Column('QualifierCode', String(255), nullable=False)
+    description = Column('QualifierDescription', String(255), nullable=False)
 
     def __repr__(self):
         return "<Qualifier('%s', '%s', '%s')>" % (self.id, self.code, self.description)

src/DataParser/odmdata/quality_control_level.py

@@ -7,9 +7,9 @@ class QualityControlLevel(Base):
     __tablename__ = 'QualityControlLevels'
 
     id = Column('QualityControlLevelID', Integer, primary_key=True)
-    code = Column('QualityControlLevelCode', String, nullable=False)
-    definition = Column('Definition', String, nullable=False)
-    explanation = Column('Explanation', String, nullable=False)
+    code = Column('QualityControlLevelCode', String(255), nullable=False)
+    definition = Column('Definition', String(255), nullable=False)
+    explanation = Column('Explanation', String(255), nullable=False)
 
     def __repr__(self):
         return "<QualityControlLevel('%s', '%s', '%s', '%s')>" % (self.id, self.code, self.definition, self.explanation)

src/DataParser/odmdata/sample.py

@@ -9,8 +9,8 @@ class Sample(Base):
     __tablename__ = 'Samples'
 
     id = Column('SampleID', Integer, primary_key=True)
-    type = Column('SampleType', String, nullable=False)
-    lab_sample_code = Column('LabSampleCode', String, nullable=False)
+    type = Column('SampleType', String(255), nullable=False)
+    lab_sample_code = Column('LabSampleCode', String(255), nullable=False)
     lab_method_id = Column('LabMethodID', Integer, ForeignKey('LabMethods.LabMethodID'), nullable=False)
 
     # relationships

src/DataParser/odmdata/sample_medium_cv.py

@@ -6,8 +6,8 @@
 class SampleMediumCV(Base):
     __tablename__ = 'SampleMediumCV'
 
-    term = Column('Term', String, primary_key=True)
-    definition = Column('Definition', String)
+    term = Column('Term', String(255), primary_key=True)
+    definition = Column('Definition', String(255))
 
     def __repr__(self):
         return "<SampleMedium('%s', '%s')>" % (self.term, self.definition)

src/DataParser/odmdata/sample_type_cv.py

@@ -6,8 +6,8 @@
 class SampleTypeCV(Base):
     __tablename__ = 'SampleTypeCV'
 
-    term = Column('Term', String, primary_key=True)
-    definition = Column('Definition', String)
+    term = Column('Term', String(255), primary_key=True)
+    definition = Column('Definition', String(255))
 
     def __repr__(self):
         return "<SampleTypeCV('%s', '%s')>" % (self.term, self.definition)

src/DataParser/odmdata/series.py

@@ -50,30 +50,30 @@ class Series(Base):
 
     id = Column('SeriesID', Integer, primary_key=True)
     site_id = Column('SiteID', Integer, ForeignKey('Sites.SiteID'), nullable=False)
-    site_code = Column('SiteCode', String)
-    site_name = Column('SiteName', String)
+    site_code = Column('SiteCode', String(255))
+    site_name = Column('SiteName', String(255))
     variable_id = Column('VariableID', Integer, ForeignKey('Variables.VariableID'), nullable=False)
-    variable_code = Column('VariableCode', String)
-    variable_name = Column('VariableName', String)
-    speciation = Column('Speciation', String)
+    variable_code = Column('VariableCode', String(255))
+    variable_name = Column('VariableName', String(255))
+    speciation = Column('Speciation', String(255))
     variable_units_id = Column('VariableUnitsID', Integer)
-    variable_units_name = Column('VariableUnitsName', String)
-    sample_medium = Column('SampleMedium', String)
-    value_type = Column('ValueType', String)
+    variable_units_name = Column('VariableUnitsName', String(255))
+    sample_medium = Column('SampleMedium', String(255))
+    value_type = Column('ValueType', String(255))
     time_support = Column('TimeSupport', Float)
     time_units_id = Column('TimeUnitsID', Integer)
-    time_units_name = Column('TimeUnitsName', String)
-    data_type = Column('DataType', String)
-    general_category = Column('GeneralCategory', String)
+    time_units_name = Column('TimeUnitsName', String(255))
+    data_type = Column('DataType', String(255))
+    general_category = Column('GeneralCategory', String(255))
     method_id = Column('MethodID', Integer, ForeignKey('Methods.MethodID'), nullable=False)
-    method_description = Column('MethodDescription', String)
+    method_description = Column('MethodDescription', String(255))
     source_id = Column('SourceID', Integer, ForeignKey('Sources.SourceID'), nullable=False)
-    source_description = Column('SourceDescription', String)
-    organization = Column('Organization', String)
-    citation = Column('Citation', String)
+    source_description = Column('SourceDescription', String(255))
+    organization = Column('Organization', String(255))
+    citation = Column('Citation', String(255))
     quality_control_level_id = Column('QualityControlLevelID', Integer,
                                       ForeignKey('QualityControlLevels.QualityControlLevelID'), nullable=False)
-    quality_control_level_code = Column('QualityControlLevelCode', String)
+    quality_control_level_code = Column('QualityControlLevelCode', String(255))
     begin_date_time = Column('BeginDateTime', DateTime)
     end_date_time = Column('EndDateTime', DateTime)
     begin_date_time_utc = Column('BeginDateTimeUTC', DateTime)

src/DataParser/odmdata/site.py

@@ -10,22 +10,22 @@ class Site(Base):
     __tablename__ = 'Sites'
 
     id = Column('SiteID', Integer, primary_key=True)
-    code = Column('SiteCode', String)
-    name = Column('SiteName', String)
+    code = Column('SiteCode', String(255))
+    name = Column('SiteName', String(255))
     latitude = Column('Latitude', Float)
     longitude = Column('Longitude', Float)
     lat_long_datum_id = Column('LatLongDatumID', Integer, ForeignKey('SpatialReferences.SpatialReferenceID'))
     elevation_m = Column('Elevation_m', Float)
-    vertical_datum = Column('VerticalDatum', String)
+    vertical_datum = Column('VerticalDatum', String(255))
     local_x = Column('LocalX', Float)
     local_y = Column('LocalY', Float)
     local_projection_id = Column('LocalProjectionID', Integer, ForeignKey('SpatialReferences.SpatialReferenceID'))
     pos_accuracy_m = Column('PosAccuracy_m', Float)
-    state = Column('State', String)
-    county = Column('County', String)
-    comments = Column('Comments', String)
+    state = Column('State', String(255))
+    county = Column('County', String(255))
+    comments = Column('Comments', String(255))
 
-    type = Column('SiteType', String)
+    type = Column('SiteType', String(255))
 
     # relationships
     spatial_ref = relationship(SpatialReference, primaryjoin=("SpatialReference.id==Site.lat_long_datum_id"), lazy='subquery')

src/DataParser/odmdata/site_type_cv.py

@@ -6,8 +6,8 @@
 class SiteTypeCV(Base):
     __tablename__ = 'SiteTypeCV'
 
-    term = Column('Term', String, primary_key=True)
-    definition = Column('Definition', String)
+    term = Column('Term', String(255), primary_key=True)
+    definition = Column('Definition', String(255))
 
     def __repr__(self):
         return "<SiteTypeCV('%s', '%s')>" % (self.term, self.definition)