{"id":789,"date":"2022-07-01T21:43:18","date_gmt":"2022-07-01T19:43:18","guid":{"rendered":"https:\/\/olkn.myvnc.com\/?p=789"},"modified":"2022-07-02T18:43:56","modified_gmt":"2022-07-02T16:43:56","slug":"web-scraper","status":"publish","type":"post","link":"https:\/\/olkn.myvnc.com\/?p=789","title":{"rendered":"web scraper"},"content":{"rendered":"<div id=\"ez-toc-container\" class=\"ez-toc-v2_0_82_2 counter-hierarchy ez-toc-counter ez-toc-grey ez-toc-container-direction\">\n<div class=\"ez-toc-title-container\">\n<p class=\"ez-toc-title\" style=\"cursor:inherit\">Table of Contents<\/p>\n<span class=\"ez-toc-title-toggle\"><a href=\"#\" class=\"ez-toc-pull-right ez-toc-btn ez-toc-btn-xs ez-toc-btn-default ez-toc-toggle\" aria-label=\"Toggle Table of Content\"><span class=\"ez-toc-js-icon-con\"><span class=\"\"><span class=\"eztoc-hide\" style=\"display:none;\">Toggle<\/span><span class=\"ez-toc-icon-toggle-span\"><svg style=\"fill: #999;color:#999\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" class=\"list-377408\" width=\"20px\" height=\"20px\" viewBox=\"0 0 24 24\" fill=\"none\"><path d=\"M6 6H4v2h2V6zm14 0H8v2h12V6zM4 11h2v2H4v-2zm16 0H8v2h12v-2zM4 16h2v2H4v-2zm16 0H8v2h12v-2z\" fill=\"currentColor\"><\/path><\/svg><svg style=\"fill: #999;color:#999\" class=\"arrow-unsorted-368013\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"10px\" height=\"10px\" viewBox=\"0 0 24 24\" version=\"1.2\" baseProfile=\"tiny\"><path d=\"M18.2 9.3l-6.2-6.3-6.2 6.3c-.2.2-.3.4-.3.7s.1.5.3.7c.2.2.4.3.7.3h11c.3 0 .5-.1.7-.3.2-.2.3-.5.3-.7s-.1-.5-.3-.7zM5.8 14.7l6.2 6.3 6.2-6.3c.2-.2.3-.5.3-.7s-.1-.5-.3-.7c-.2-.2-.4-.3-.7-.3h-11c-.3 0-.5.1-.7.3-.2.2-.3.5-.3.7s.1.5.3.7z\"\/><\/svg><\/span><\/span><\/span><\/a><\/span><\/div>\n<nav><ul class='ez-toc-list ez-toc-list-level-1 eztoc-toggle-hide-by-default' ><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-1\" href=\"https:\/\/olkn.myvnc.com\/?p=789\/#contents\" >contents<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-2\" href=\"https:\/\/olkn.myvnc.com\/?p=789\/#logging\" >logging<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-3\" href=\"https:\/\/olkn.myvnc.com\/?p=789\/#Data_base_access\" >Data base access<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-4\" href=\"https:\/\/olkn.myvnc.com\/?p=789\/#solr_indexing\" >solr indexing<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-5\" href=\"https:\/\/olkn.myvnc.com\/?p=789\/#filesystem_access\" >filesystem access<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-6\" href=\"https:\/\/olkn.myvnc.com\/?p=789\/#web_scraping\" >web scraping<\/a><\/li><\/ul><\/nav><\/div>\n<h1><span class=\"ez-toc-section\" id=\"contents\"><\/span>contents<span class=\"ez-toc-section-end\"><\/span><\/h1>\n<ul>\n<li>logging<\/li>\n<li>data base access<\/li>\n<li>solr indexing<\/li>\n<li>filesystem access<\/li>\n<li>web scraping<\/li>\n<\/ul>\n<h1><span class=\"ez-toc-section\" id=\"logging\"><\/span>logging<span class=\"ez-toc-section-end\"><\/span><\/h1>\n<h1><span class=\"ez-toc-section\" id=\"Data_base_access\"><\/span>Data base access<span class=\"ez-toc-section-end\"><\/span><\/h1>\n<p>&#8211; mysql in python<\/p>\n<pre><code>\r\nimport mysql.connector\r\n# from mysql.connector import Error\r\n\r\n# pip3 install mysql-connector\r\n# https:\/\/dev.mysql.com\/doc\/connector-python\/en\/connector-python-reference.html\r\n\r\nclass DB():\r\n    def __init__(self, config):\r\n        self.connection = None\r\n        self.connection = mysql.connector.connect(**config)\r\n        \r\n    def query(self, sql, args):\r\n        cursor = self.connection.cursor()\r\n        cursor.execute(sql, args)\r\n        return cursor\r\n\r\n    def insert(self,sql,args):\r\n        cursor = self.query(sql, args)\r\n        id = cursor.lastrowid\r\n        self.connection.commit()\r\n        cursor.close()\r\n        return id\r\n\r\n    # https:\/\/dev.mysql.com\/doc\/connector-python\/en\/connector-python-api-mysqlcursor-executemany.html\r\n    def insertmany(self,sql,args):\r\n        cursor = self.connection.cursor()\r\n        cursor.executemany(sql, args)\r\n        rowcount = cursor.rowcount\r\n        self.connection.commit()\r\n        cursor.close()\r\n        return rowcount\r\n\r\n    def update(self,sql,args):\r\n        cursor = self.query(sql, args)\r\n        rowcount = cursor.rowcount\r\n        self.connection.commit()\r\n        cursor.close()\r\n        return rowcount\r\n\r\n    def fetch(self, sql, args):\r\n        rows = []\r\n        cursor = self.query(sql, args)\r\n        if cursor.with_rows:\r\n            rows = cursor.fetchall()\r\n        cursor.close()\r\n        return rows\r\n\r\n    def fetchone(self, sql, args):\r\n        row = None\r\n        cursor = self.query(sql, args)\r\n        if cursor.with_rows:\r\n            row = cursor.fetchone()\r\n        cursor.close()\r\n        return row\r\n\r\n    def __del__(self):\r\n        if self.connection != None:\r\n            self.connection.close()\r\n\r\n  # write your function here for CRUD operations\r\n\r\n<\/code><\/pre>\n<h1><span class=\"ez-toc-section\" id=\"solr_indexing\"><\/span>solr indexing<span class=\"ez-toc-section-end\"><\/span><\/h1>\n<h1><span class=\"ez-toc-section\" id=\"filesystem_access\"><\/span> filesystem access<span class=\"ez-toc-section-end\"><\/span><\/h1>\n<h1><span class=\"ez-toc-section\" id=\"web_scraping\"><\/span>web scraping<span class=\"ez-toc-section-end\"><\/span><\/h1>\n","protected":false},"excerpt":{"rendered":"<p>contents logging data base access solr indexing filesystem access web scraping logging Data base access &#8211; mysql in python import mysql.connector # from mysql.connector import Error # pip3 install mysql-connector # https:\/\/dev.mysql.com\/doc\/connector-python\/en\/connector-python-reference.html class DB(): def __init__(self, config): self.connection = None self.connection = mysql.connector.connect(**config) def query(self, sql, args): cursor = self.connection.cursor() cursor.execute(sql, args) return cursor def &hellip; <a href=\"https:\/\/olkn.myvnc.com\/?p=789\" class=\"more-link\">Continue reading <span class=\"screen-reader-text\">web scraper<\/span><\/a><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[5,24,7,199,11],"tags":[79,196,116,195,131],"class_list":["post-789","post","type-post","status-publish","format-standard","hentry","category-knowledgebase","category-programming","category-projects","category-serachserver","category-software","tag-linux","tag-search","tag-server","tag-solr","tag-webfrontend"],"_links":{"self":[{"href":"https:\/\/olkn.myvnc.com\/index.php?rest_route=\/wp\/v2\/posts\/789","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/olkn.myvnc.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/olkn.myvnc.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/olkn.myvnc.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/olkn.myvnc.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=789"}],"version-history":[{"count":5,"href":"https:\/\/olkn.myvnc.com\/index.php?rest_route=\/wp\/v2\/posts\/789\/revisions"}],"predecessor-version":[{"id":794,"href":"https:\/\/olkn.myvnc.com\/index.php?rest_route=\/wp\/v2\/posts\/789\/revisions\/794"}],"wp:attachment":[{"href":"https:\/\/olkn.myvnc.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=789"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/olkn.myvnc.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=789"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/olkn.myvnc.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=789"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}