CINERGI-SiteMap.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://datadiscoverystudio.org/geoportal/rest/metadata/item/\n",
      "c:\\tmp\\\n",
      "request1:  http://datadiscoverystudio.org/geoportal/elastic/metadata/item/_search?_source=sys_modified_dt&scroll=1m&size=10000\n",
      "total records:  1663930\n",
      "count:  20000\n",
      "count:  30000\n",
      "count:  40000\n",
      "count:  50000\n",
      "count:  60000\n",
      "count:  70000\n",
      "count:  80000\n",
      "count:  90000\n",
      "count:  100000\n",
      "count:  110000\n",
      "count:  120000\n",
      "count:  130000\n",
      "count:  140000\n",
      "count:  150000\n",
      "count:  160000\n",
      "count:  170000\n",
      "count:  180000\n",
      "count:  190000\n",
      "count:  200000\n",
      "count:  210000\n",
      "count:  220000\n",
      "count:  230000\n",
      "count:  240000\n",
      "count:  250000\n",
      "count:  260000\n",
      "count:  270000\n",
      "count:  280000\n",
      "count:  290000\n",
      "count:  300000\n",
      "count:  310000\n",
      "count:  320000\n",
      "count:  330000\n",
      "count:  340000\n",
      "count:  350000\n",
      "count:  360000\n",
      "count:  370000\n",
      "count:  380000\n",
      "count:  390000\n",
      "count:  400000\n",
      "count:  410000\n",
      "count:  420000\n",
      "count:  430000\n",
      "count:  440000\n",
      "count:  450000\n",
      "count:  460000\n",
      "count:  470000\n",
      "count:  480000\n",
      "count:  490000\n",
      "count:  500000\n",
      "count:  510000\n",
      "count:  520000\n",
      "count:  530000\n",
      "count:  540000\n",
      "count:  550000\n",
      "count:  560000\n",
      "count:  570000\n",
      "count:  580000\n",
      "count:  590000\n",
      "count:  600000\n",
      "count:  610000\n",
      "count:  620000\n",
      "count:  630000\n",
      "count:  640000\n",
      "count:  650000\n",
      "count:  660000\n",
      "ERROR writing sitemap url for _id= _settings\n",
      "'sys_modified_dt'\n",
      "count:  670000\n",
      "count:  680000\n",
      "count:  690000\n",
      "count:  700000\n",
      "count:  710000\n",
      "count:  720000\n",
      "count:  730000\n",
      "count:  740000\n",
      "count:  750000\n",
      "count:  760000\n",
      "count:  770000\n",
      "count:  780000\n",
      "count:  790000\n",
      "count:  800000\n",
      "count:  810000\n",
      "count:  820000\n",
      "count:  830000\n",
      "count:  840000\n",
      "count:  850000\n",
      "count:  860000\n",
      "count:  870000\n",
      "count:  880000\n",
      "count:  890000\n",
      "count:  900000\n",
      "count:  910000\n",
      "ERROR writing sitemap url for _id= AWAUKpyhvn2fSNpqyGsu\n",
      "'sys_modified_dt'\n",
      "count:  920000\n",
      "count:  930000\n",
      "count:  940000\n",
      "count:  950000\n",
      "count:  960000\n",
      "count:  970000\n",
      "count:  980000\n",
      "count:  990000\n",
      "count:  1000000\n",
      "count:  1010000\n",
      "count:  1020000\n",
      "count:  1030000\n",
      "count:  1040000\n",
      "count:  1050000\n",
      "count:  1060000\n",
      "count:  1070000\n",
      "count:  1080000\n",
      "count:  1090000\n",
      "count:  1100000\n",
      "count:  1110000\n",
      "count:  1120000\n",
      "count:  1130000\n",
      "count:  1140000\n",
      "count:  1150000\n",
      "count:  1160000\n",
      "count:  1170000\n",
      "count:  1180000\n",
      "count:  1190000\n",
      "count:  1200000\n",
      "count:  1210000\n",
      "count:  1220000\n",
      "count:  1230000\n",
      "count:  1240000\n",
      "count:  1250000\n",
      "count:  1260000\n",
      "count:  1270000\n",
      "ERROR writing sitemap url for _id= AWUbQH9QbXIlqySHpFre\n",
      "'sys_modified_dt'\n",
      "count:  1280000\n",
      "count:  1290000\n",
      "count:  1300000\n",
      "count:  1310000\n",
      "count:  1320000\n",
      "count:  1330000\n",
      "count:  1340000\n",
      "count:  1350000\n",
      "count:  1360000\n",
      "count:  1370000\n",
      "count:  1380000\n",
      "count:  1390000\n",
      "count:  1400000\n",
      "count:  1410000\n",
      "count:  1420000\n",
      "count:  1430000\n",
      "count:  1440000\n",
      "count:  1450000\n",
      "ERROR writing sitemap url for _id= AWUbOYMRbXIlqySHpFrb\n",
      "'sys_modified_dt'\n",
      "ERROR writing sitemap url for _id= AWUbOhvXbXIlqySHpFrc\n",
      "'sys_modified_dt'\n",
      "ERROR writing sitemap url for _id= AWUky5lDbXIlqySHpFrr\n",
      "'sys_modified_dt'\n",
      "count:  1460000\n",
      "count:  1470000\n",
      "count:  1480000\n",
      "count:  1490000\n",
      "count:  1500000\n",
      "count:  1510000\n",
      "count:  1520000\n",
      "count:  1530000\n",
      "count:  1540000\n",
      "count:  1550000\n",
      "count:  1560000\n",
      "count:  1570000\n",
      "count:  1580000\n",
      "count:  1590000\n",
      "count:  1600000\n",
      "count:  1610000\n",
      "count:  1620000\n",
      "count:  1630000\n",
      "count:  1640000\n",
      "count:  1650000\n",
      "count:  1660000\n",
      "count:  1670000\n",
      "done, counter =  1670000\n"
     ]
    }
   ],
   "source": [
    "from datetime import datetime\n",
    "import pytz\n",
    "import requests\n",
    "import sys\n",
    "# see http://docs.python-requests.org/en/master/user/quickstart/ for package documentation\n",
    "\n",
    "ddsbaseURL = 'http://datadiscoverystudio.org/'\n",
    "geoportalBaseURL = ddsbaseURL + 'geoportal/'\n",
    "catalogISOmetadataBase = geoportalBaseURL + 'rest/metadata/item/'\n",
    "\n",
    "print catalogISOmetadataBase\n",
    "XML_HEADER = '<?xml version=\"1.0\" encoding=\"UTF-8\"?>'\n",
    "\n",
    "fileLocationBase = 'c:\\\\tmp\\\\'\n",
    "print fileLocationBase\n",
    "\n",
    "#sitemaptohtml = 'https://raw.githubusercontent.com/CINERGI/xmlsitemap/master/xml-sitemap.xsl'\n",
    "#suggest copying the xslt file into the same directory with the sitemaps, in which case, use this\n",
    "# value for sitemaptohtml:\n",
    "sitemaptohtml = 'xml-sitemap.xsl'\n",
    "\n",
    "# first some utility functions for file generation and writing\n",
    "def writeLinks( response, mfile ):\n",
    "#   writes entries in sitemap file, with URL for metadata record as html; the record\n",
    "# is expected to include a schema.org JSON-LD script for use by the search indexers\n",
    "    for hit in response[\"hits\"][\"hits\"]:\n",
    "#        hittitle = hit[\"_source\"][\"title\"]\n",
    "        try:\n",
    "            hitid = hit[\"_id\"]\n",
    "            hitmodified =  hit[\"_source\"][\"sys_modified_dt\"]\n",
    "#        print \"title: \", hittitle, \" id: \", hitid, \" date: \", hitmodified  \n",
    "\n",
    "            mfile.write('<url>')\n",
    "            mfile.write(\"\\n\")\n",
    "# original CINERGI catalog location\n",
    "#mfile.write('<loc>http://cinergi.sdsc.edu/geoportal/rest/metadata/item/' \n",
    "#                       + hitid + '/html</loc>')\n",
    "            mfile.write('<loc>' + catalogISOmetadataBase + hitid + '/html</loc>')\n",
    "            mfile.write(\"\\n\")\n",
    "            mfile.write('<lastmod>' + hitmodified + '</lastmod>')\n",
    "            mfile.write(\"\\n\")\n",
    "            mfile.write('<changefreq>monthly</changefreq>')\n",
    "            mfile.write(\"\\n\")\n",
    "#        mfile.write('<priority>0.8</priority>')\n",
    "#        mfile.write(\"\\n\")\n",
    "            mfile.write('</url>')\n",
    "            mfile.write(\"\\n\")\n",
    "        except:\n",
    "            print(\"ERROR writing sitemap url for _id= \" + hitid)\n",
    "            print(sys.exc_info()[1])\n",
    "    return\n",
    "\n",
    "def indexFile():\n",
    "# set up the sitemap index. This file has a link to each sitemap file. \n",
    "# sitemaps are limited to 10000 entries, so if there is a bigger catalog, have\n",
    "# to generate multiple sitemaps and point to them from the index.\n",
    "    try:\n",
    "        file_object  = open(fileLocationBase + \"DDSSiteIndex.xml\", \"w\")\n",
    "    except:\n",
    "        print(\"ERROR: Can't open the index file, bailing out\")\n",
    "        print(sys.exc_info()[1])\n",
    "        sys.exit(0)\n",
    "    # put in the header stuff\n",
    "    file_object.write(XML_HEADER)\n",
    "    file_object.write(\"\\n\")\n",
    "    file_object.write('<?xml-stylesheet type=\"text/xsl\" href=\"' + sitemaptohtml + '\"?>')\n",
    "    file_object.write('\\n')\n",
    "    file_object.write('<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">')\n",
    "    file_object.write(\"\\n\")\n",
    "\n",
    "    return(file_object)\n",
    "\n",
    "def siteMapFile(name):\n",
    "# opens a new empty sitemap file and returns the file_object for writing to it.\n",
    "    try:\n",
    "        file_object  = open(fileLocationBase + name, \"w\")\n",
    "    except:\n",
    "        print(\"ERROR: Can't open the new sitemap file: \" + name + \", bailing out\")\n",
    "        print(sys.exc_info()[1])\n",
    "        sys.exit(0)\n",
    "        \n",
    "    #put in the header stuff\n",
    "    file_object.write(XML_HEADER)\n",
    "    file_object.write('\\n')\n",
    "    file_object.write('<?xml-stylesheet type=\"text/xsl\" href=\"' + sitemaptohtml + '\"?>')\n",
    "    file_object.write('\\n')\n",
    "    file_object.write('<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">')\n",
    "    file_object.write('\\n')\n",
    "    return(file_object)\n",
    "\n",
    "# construct Elasticsearch URL with  search request\n",
    "# espath=\"http://cinergi.sdsc.edu/geoportal/elastic/\"\n",
    "espath= geoportalBaseURL + \"elastic/\"\n",
    "esindex=\"metadata\"\n",
    "esresource=\"/item/_search\"\n",
    "baseURL = espath+esindex+esresource\n",
    "# need to use scrolling because there are >10000 records\n",
    "# this is the time to live for the scroll index; renewed on each search call\n",
    "p_scroll=\"1m\"\n",
    "#number of records to return in each batch. \n",
    "# This will be the number of links in each sitemap file\n",
    "p_size=\"10000\"\n",
    "#use this for testing\n",
    "#p_size=\"10\"\n",
    "# the only field we need for the sitemap is the modified date\n",
    "# comma delimited list of index fields to return from the _source section of the hits object\n",
    "#p_source=\"sys_modified_dt,title\"\n",
    "p_source=\"sys_modified_dt\"\n",
    "\n",
    "# first get the scroll index to start scrolling loop, and the total number of records\n",
    "\n",
    "counter = 0\n",
    "filecount = 0\n",
    "#print counter\n",
    "\n",
    "#first request to get scrolling set up\n",
    "p = {'scroll':p_scroll, \n",
    "    'size' : p_size, \n",
    "    '_source' : p_source}\n",
    "r = requests.get(baseURL, params=p)\n",
    "print \"request1: \", r.url\n",
    "\n",
    "if r.status_code == requests.codes.ok:\n",
    "    response = r.json()\n",
    "    totalRecords = response[\"hits\"][\"total\"]\n",
    "    scrollID = response[\"_scroll_id\"]\n",
    "\n",
    "    #    set up the index file\n",
    "    indexhandle = indexFile()\n",
    "    print \"total records: \", totalRecords\n",
    "    sitemapfilename = \"ddssitemap\" + str(filecount)+ \".xml\"\n",
    "    sitemaphandle = siteMapFile(sitemapfilename)\n",
    "    writeLinks(response, sitemaphandle)\n",
    "    sitemaphandle.write('</urlset>')\n",
    "    sitemaphandle.close() \n",
    "        \n",
    "\n",
    "    # first bit is just to get a correctly formatted xsd:datetime with timezone\n",
    "    tz = pytz.timezone(\"America/Phoenix\")\n",
    "    aware_dt = tz.localize(datetime.now())\n",
    "    \n",
    "        #write first index entry\n",
    "    indexhandle.write('<sitemap>')\n",
    "    indexhandle.write('\\n')\n",
    "#    indexhandle.write('<loc>http://cinergi.sdsc.edu/geoportal/' + sitemapfilename + '</loc>')\n",
    "#  providing a full URL to put links in the sitemap index:\n",
    "    indexhandle.write('<loc>' + ddsbaseURL + 'sitemap/' + sitemapfilename + '</loc>')\n",
    "# using local file paths doesn't work for Google, don't use:\n",
    "#    indexhandle.write('<loc>' + sitemapfilename + '</loc>')\n",
    "    indexhandle.write('\\n')\n",
    "    indexhandle.write('<lastmod>' + str(aware_dt.isoformat())+ '</lastmod>')\n",
    "    indexhandle.write('\\n')\n",
    "    indexhandle.write('</sitemap>')\n",
    "    indexhandle.write('\\n')\n",
    "        \n",
    "    filecount = filecount + 1\n",
    "    counter = counter + int(p_size)\n",
    "else:\n",
    "    r.raise_for_status()\n",
    "    sys.exit(0)\n",
    "            \n",
    "        \n",
    "while counter < totalRecords:\n",
    "# use this for testing:\n",
    "#while counter < 50:\n",
    "    #have to hit the scroll resource for Elasticsearch\n",
    "    esresource=\"_search/scroll\"\n",
    "    #Geoportal Elasticsearch pass through requires publisher role to run the scroll resource\n",
    "    espath=\"http://admin:admin@datadiscoverystudio.org/geoportal/elastic/\"\n",
    "    baseURL = espath+esresource\n",
    "    p = { 'scroll':p_scroll, \n",
    "    'scroll_id' : scrollID}\n",
    "    r = requests.get(baseURL, params=p)\n",
    "#    print \"request: \", r.url, r.status_code\n",
    "#        print \"raw response2: \", r, \" status: \", r.status_code\n",
    "#        print r.headers['content-type']\n",
    "    if r.status_code == requests.codes.ok:\n",
    "        response = r.json()\n",
    "        scrollID = response[\"_scroll_id\"]\n",
    "        sitemapfilename = \"ddssitemap\" + str(filecount)+ \".xml\"\n",
    "# ******* these lines comment out to just write the sitemap index\n",
    "        sitemaphandle = siteMapFile(sitemapfilename)\n",
    " # This call actually writes the individual sitemap entries\n",
    "        writeLinks(response, sitemaphandle)\n",
    "        sitemaphandle.write('</urlset>')\n",
    "        sitemaphandle.close() \n",
    "# ********\n",
    "        \n",
    "\n",
    "        tz = pytz.timezone(\"America/Phoenix\")\n",
    "        aware_dt = tz.localize(datetime.now())\n",
    "    \n",
    "        #new index entry\n",
    "        indexhandle.write('<sitemap>')\n",
    "        indexhandle.write('\\n')\n",
    "        indexhandle.write('<loc>' + ddsbaseURL + 'sitemap/' + sitemapfilename + '</loc>')\n",
    "        indexhandle.write('\\n')\n",
    "        indexhandle.write('<lastmod>' + str(aware_dt.isoformat())+ '</lastmod>')\n",
    "        indexhandle.write('\\n')\n",
    "        indexhandle.write('</sitemap>')\n",
    "        indexhandle.write('\\n')\n",
    "        \n",
    "        filecount = filecount + 1\n",
    "        counter = counter + int(p_size)\n",
    "        print \"count: \", counter\n",
    "    else:\n",
    "        r.raise_for_status()\n",
    "        break\n",
    "\n",
    "indexhandle.write('</sitemapindex>')        \n",
    "indexhandle.close()\n",
    "       \n",
    "#\n",
    "print \"done, counter = \",counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2019-05-17T12:00:21.645000-07:00'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# figuring out how to get an ISO8601 datetime with time zone for Google sitemap!\n",
    "\n",
    "from datetime import datetime\n",
    "import pytz\n",
    "\n",
    "tz = pytz.timezone(\"America/Phoenix\")\n",
    "aware_dt = tz.localize(datetime.now())\n",
    "aware_dt.isoformat()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}