Code

Description

Goals:

  • bla
  • blabla
  • bla
  • blabla
#!pip install toolforge
Collecting toolforge
  Using cached https://files.pythonhosted.org/packages/c0/74/b18d4301b56e7decb1c361af8d5b49fa0c6f2da17e102f19d4539f3737b4/toolforge-4.3.0-py3-none-any.whl
Requirement already satisfied: pymysql in /srv/paws/lib/python3.6/site-packages (from toolforge)
Requirement already satisfied: requests in /srv/paws/lib/python3.6/site-packages (from toolforge)
Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /srv/paws/lib/python3.6/site-packages (from requests->toolforge)
Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /srv/paws/lib/python3.6/site-packages (from requests->toolforge)
Requirement already satisfied: certifi>=2017.4.17 in /srv/paws/lib/python3.6/site-packages (from requests->toolforge)
Requirement already satisfied: idna<2.9,>=2.5 in /srv/paws/lib/python3.6/site-packages (from requests->toolforge)
Installing collected packages: toolforge
Successfully installed toolforge-4.3.0
import pandas as pd
import pymysql
from sqlalchemy import create_engine
import requests
import toolforge
#import oursql
---------------------------------------------------------------------------
ModuleNotFoundError                       Traceback (most recent call last)
<ipython-input-3-e7b4e8dea4a1> in <module>
      3 from sqlalchemy import create_engine
      4 import requests
----> 5 import toolforge
      6 #import oursql

ModuleNotFoundError: No module named 'toolforge'
## Retrieve the data from the edit history database (ToolForge query)
 
 
 
## Store the data locally ---this will be execute in HER
# How many tools can I track that are relevant
# General tools
data_hay_tools = pd.read_json(path_or_buf="https://tools.wmflabs.org/hay/directory/api.php", lines=False,encoding='utf-8', orient="records")
data_hay_tools.head()
# header: added	author	deleted	description	fulltext	id	jsonurl	keywords	name	redirects	repository	title	unavailable	url
# index of columns that are lists: 1, 7
added author deleted description fulltext id jsonurl keywords name redirects repository title unavailable url
0 2016-01-02T23:36:43+00:00 [Magnus Manske] 0 Generate an image RSS feed (for your screensav... mm_wdq_image_feed wdq image feed generate an i... 73 http://tools.wmflabs.org/wikidata-todo/toolinf... [wikidata, rss, feed, images] mm_wdq_image_feed 25 None WDQ image feed 0 https://tools.wmflabs.org/wikidata-todo/wdq_im...
1 2016-01-02T23:36:47+00:00 [Danilo.mac] 0 Lists of pages or users, based on many criteri... ptwikis-lists page and user lists lists of pag... 255 http://tools.wmflabs.org/ptwikis/static/toolin... [list, pages, category, template, wikiproject,... ptwikis-lists 76 https://github.com/ptwikis/ptwikis Page and User Lists 0 https://tools.wmflabs.org/ptwikis/Listas
2 2016-01-02T23:36:43+00:00 [Magnus Manske] 0 Generates a pseudo-random list of Wikidata ite... mm_no_statements items without statements gene... 70 http://tools.wmflabs.org/wikidata-todo/toolinf... [wikidata, statements, items] mm_no_statements 63 https://bitbucket.org/magnusmanske/wikidata-todo Items without statements 0 http://tools.wmflabs.org/wikidata-todo/no_stat...
3 2016-01-02T23:36:43+00:00 [Magnus Manske] 0 A list of files not used on Wikipedia, from a ... mm_unused_files unused files a list of files n... 95 http://tools.wmflabs.org/magnustools/toolinfo.... [glam, files, images, views, commons, list, un... mm_unused_files 42 https://bitbucket.org/magnusmanske/magnustools Unused files 0 http://tools.wmflabs.org/magnustools/unused_fi...
4 2016-01-02T23:36:44+00:00 [Magnus Manske] 0 Wikidata automated description API mm_autodesc autodesc wikidata automated descri... 157 http://tools.wmflabs.org/magnustools/toolinfo.... [wikidata, api, description] mm_autodesc 53 None AutoDesc 0 http://tools.wmflabs.org/autodesc
print(len(data_hay_tools['name'].unique()))
598
data_hay_tools['name'].unique()
array(['mm_wdq_image_feed', 'ptwikis-lists', 'mm_no_statements',
       'mm_unused_files', 'mm_autodesc', 'hay-sgdtransform', 'wmcounter',
       'toolforge.mapycz', 'mw-tool-snapshots', 'lonelylinks', 'checker',
       'wikitrends', 'mm_glamorous', 'textcatdemo', 'ia-upload',
       'hartman', 'toolforge.apt-browser', 'mm_wd_missing_images',
       'OAuth Hello World', 'BsAut', 'wikipagestats',
       'ptwikis-abusefilter-graphs', 'mw-tool-orphantalk',
       'toolforge.charttableconverter', 'hay-chantek', 'sal',
       'wikidata-descriptioner', 'mm_prep_bio', 'iw-category-redirects',
       'glamify', 'mm_flickr2commons', 'cite-o-meter', 'wikiviewstats',
       'citation_hunter', 'stylize', 'mm_wikidata_useful',
       'ptwikis-mobile', 'hay-nadownload', 'wikiComment', 'worklist-demo',
       'remarkup2wikitext', 'gmt', 'toolforge.whodunnit',
       'mw-tool-expose-data', 'hay-wdtranslate', 'videoconvert',
       'toolforge.maplayers-demo', 'mm_artwork_images', 'mm_terminator',
       'mm_templator', 'xtools-articleinfo', 'bbctvcite',
       'toolforge.portal', 'CSV2Wikitable', 'toolforge.wikiportretdev',
       'ptools-uploadcounter', 'mm_makeref', 'contribsize',
       'mm_commons_api', 'seth-url-converter', 'InceptionBot',
       'mm_random_page_in_cat', 'programs-and-events-dashboard',
       'interaction-timeline', 'grid-jobs', 'stewardbots',
       'mw-tool-stylize', 'translatemplate', 'code-lookup',
       'toolforge.fscbot', 'stimmberechtigung',
       'toolforge.osm-gadget-leaflet', 'mm_wikisoba', 'mm_joanjoc',
       'jury', 'openstack-browser', 'mm_autolist', 'rm-stats',
       'xtools-echo', 'mm_quick_intersection', 'pywikibot', 'mw2sparql',
       'icalendar', 'wikidata-osm-link', 'mm_quick_statements',
       'mm_file_reuse', 'wm-metrics', 'mm_mixnmatch',
       'meta-crossactivity', 'whois-gateway', 'mm_pages_in_categories',
       'ReplicationLag', 'hay-gwtcook', 'commons-mass-description',
       'toolforge.wdvaliditycheck', 'mm_misstake', 'churches',
       'iw-contributors', 'sqid', 'mm_baglama2',
       'templatetransclusioncheck', 'mormegil-natbirths', 'section-links',
       'mm_most_wanted', 'ninthcircuit', 'ordia', 'mirador', 'mm_wdq',
       'ws-cat-browser', 'yadkard', 'xtools-pagehistory', 'redirectviews',
       'toolforge.volby2wiki', 'WikidataClassBrowser',
       'mapillary_simpleUpload', 'dna', 'mw-tool-usage', 'integraality',
       'CleanupWorklistBot', 'ldap', 'shortnames', 'toolforge.maria',
       'toolforge.quickpreset-migrate', 'PageTools',
       'toolforge.addscholartopics', 'toolforge.event-live-session',
       'hay-nasearch', 'disambig-monitor',
       'toolforge.google-drive-photos-to-commons', 'vadalead',
       'toolforge.kmlexport-cswiki', 'mormegil-catwithoutcoords',
       'mm_redirector', 'wlm-wikidata', 'toolforge.global-search',
       'ipcheck', 'giftbot', 'wlm-italy', 'cluebot3', 'mw-tool-validator',
       'hay-directory', 'mm_pagepile', 'mm_file_siblings', 'heritage-api',
       'mm_rcvis', 'mormegil-catsuggest', 'mm_url2commons',
       'facebook-messenger-chatbot', 'hay-propbrowse', 'mm_beacon',
       'excel2wiki', 'mm_wikidata_todo', 'mm_fist', 'ws-google-ocr',
       'mm_wd_species_commons', 'oauth-hello-world', 'locator-tool',
       'nonautomated-namespace-edits', 'mm_item_names', 'ascalendar',
       'meta-accounteligibility', 'toolforge.blubberd',
       'wikidata-timeline', 'copyvios', 'wiktioutils_planches',
       'mm_html2wiki', 'crosswatch', 'asurabot', 'toolforge.borracho',
       'versions', 'languagetool-wikicheck', 'ptwikis-matrix',
       'mm_persondata', 'cats-php', 'heritage-daily-uploads',
       'directory-directory-ng', 'hay-wdskim',
       'toolforge.messenger-chatbot', 'category_maintenance',
       'superyetkin', 'wikidata-recent-changes', 'meta-magicredirect',
       'mm_multidesc', 'wdprop-', 'ircredirect', 'admin', 'topveiws',
       'toolforge.ttt', 'ptwikis-timeline', 'mm_cas', 'mm_wikidata_game',
       'policy-edits', 'assamesetool', 'wdq2sparql', 'recently-active',
       'mm_wiki_todo', 'wikidata-recent-deaths', 'wsexport', 'interactoa',
       'tools-info-optimizer', 'mm_catnap', 'wikidipendenza',
       'mm_property_trees', 'wscontest', 'mm_pagepile2kml',
       'Authority control validator', 'translate', 'zppixbot',
       'toolforge.wikintu', 'mm_random_out_of_sight',
       'toolforge.phabulous', 'gridengine-status', 'mwpackages-',
       'mm_tooldir', 'ptwikis-wle-2014', 'ruprecht', 'intersect-contribs',
       'mm_wdfist', 'citationhunt', 'admin-beta', 'mm_label_no_instance',
       'taxon-info', 'mm_get_distinct_authors',
       'mm_missing_wp_animal_audio', 'Gerrit Patch Uploader',
       'toolforge.scrotbot', 'toolforge.gerrit-avatar-uploader',
       'mm_wdvenn', 'toolforge.csp-report', 'mm_wikidata_stats',
       'mm_widar', 'newusers', 'wikilaeum', 'wikiatlas2014',
       'mw-tool-octodata', 'toolforge.lincolnbot', 'meta-userpages',
       'pronuncify', 'TemplateParam', 'toolforge.commtech-commons',
       'toolforge.toolviews', 'checkwiki', 'video2commons',
       'filtered_contributions', 'toolforge.reviewer', 'wembedder',
       'mm_resolver', 'meta-gusersearch', 'ws-search', 'wikicontrib-',
       'bibleversefinder', 'hay-vizquery', 'commons-delinquent',
       'top_visited_pages', 'toolforge.extjsonuploader', 'mm_relator',
       'toolforge.jitrixis-test', 'mm_project_link_stats',
       'xtools-adminstats', 'NOA rand-upload tool', 'Jubilarac', 'meta',
       'raun', 'mm_wikisource_dnb', 'mm_tooltranslate', 'scholia',
       'MediaWiki-Benchmarker', 'wmf-sitematrix', 'toolforge.ascal',
       'lp_WLM', 'toolforge.clpo13-flask', 'ptwikis-short-articles',
       'meetbot', 'meta-synchbot', 'mm_tabernacle',
       'toolforge.sge-status', 'mm_tempo_spatial_display',
       'seth-searchsbl', 'random-featured', 'lp_KMB', 'ksamsok-rest',
       'mobo', 'toolforge.otrs-helper', 'tsreports',
       'toolforge.awmd-stats', 'mm_important_blank_items',
       'mm_dnb_ratios', 'upload-stats-bot', 'mm_geograph2commons',
       'maps-userscript-boilerplate', 'toolforge.fvcbot', 'pub', 'relgen',
       'mm_related_places', 'toolforge.bing-maps',
       'mm_find_duplicate_items', 'mm_deep_out_of_sight',
       'toolforge.wikibase-databridge-storybook', 'mormegil-catcompleter',
       'mm_concept_cloud', 'WikiProjectRecentChanges', 'ptable',
       'toolforge.lexeme-forms', 'mm_icommons', 'ptools-uploadersincat',
       'ptools-filesincat', 'labs-tools-maintgraph', 'mediaviews',
       'mm_data_url', 'toolforge.bldrwnsch', 'svgtranslate-test-',
       'hay-langviews', 'mm_multibeacon', 'depicts', 'amineeded',
       'mm_add_information', 'toolforge.texbot',
       'toolforge.stockholm-mania', 'lp_BBox', 'mm_listeria', 'alphatest',
       'commonsarchive', 'ptools-intertwined', 'extreg-wos', 'wikilist',
       'wikidata-addlingue', 'mm_wd_reconcile', 'pending-subs',
       'wikidata-harvesttemplates', 'mm_creator_from_wikidata',
       'wlm-maps', 'mm_property_string_matcher', 'wikilaeum-tool',
       'toolforge.ooui-debug', 'toolforge.rmstats', 'mm_nitol',
       'mm_wp_no_image', 'lp_credit_my_cc', 'mm_wiki2playlist',
       'mm_random_items_no_instance', 'svgtranslate', 'toolforge.igl',
       'mm_tab2wiki', 'amdb', 'mm_metamine', 'mm_catfood',
       'WC3(WC-triple):Wikipedia Category Consistency Checker',
       'commons-warper-viewer', 'dewkin', 'livestats', 'mm_book2scroll',
       'replag', 'yichengtry', 'mm_reasonator', 'ppp-sparql',
       'mm_sparql_rc', 'mm_commons_image_feed', 'wlmuk',
       'mormegil_randomlist', 'afdstats', 'bub2-',
       'toolforge.lexeme-senses', 'ptwikis-ip-patrol', 'mm_linked_items',
       'toolforge.iepcbm-bot', 'pbbot', 'hay-reccat', 'poiimport',
       'wikidata-taxonomy', 'mm_autolist2', 'toolforge.olympics',
       'mm_user_edits', 'toolforge.svgworkaroundbot',
       'mm_translate_items', 'mm_isbn2wiki', 'prop-explorer',
       'mapillary-commons', 'outreachy-useredits',
       'wikidata-primary-sources', 'category-edits', 'mm_bong',
       'toolforge.hub', 'userviews', 'dump-torrents', 'kyrksok-se',
       'cluebot', 'hay-picapermalink', 'mm_recent_deaths',
       'toolforge.rhinosf1-afdclose', 'wikidata-externalid-url',
       'hay-streetwiki', 'contentcontributor', 'nonautomated-edit',
       'universalviewer', 'toolforge.dibyadutta', 'jouncebot',
       'mapillary_widget', 'xtools-editcounter', 'bash', 'ipinfo',
       'WikidataNavelGazer', 'primerpedia', 'toolforge.massmailer',
       'warped-to-iframe', 'iabot', 'toolforge.archivesearch',
       'toolforge.cdnjs-beta', 'mw-tool-blankpages', 'meta-globalgroups',
       'toolforge.wdprop', 'wikimap', 'sparqlblocks', 'mm_item_creator',
       'mm_reverse_category_tree', 'toolforge.ru_monuments',
       'human-readable-kulturarvsdata', 'grlc', 'labs-tools-lists',
       'cvrminer', 'mm_get_article_intro', 'paws', 'rhinosf1-afdclose-',
       'RedirFinder', 'mm_wd_edit_stats', 'toolforge.mathqa',
       'wikiwhatsappbot', 'mm_authority_control', 'mm_wikishootme',
       'contributor_locations', 'mm_map2wp', 'toolforge.basyounybot',
       'mm_glamorous2', 'toolforge.xtools-autoedits',
       'articles-by-lat-lon-without-images',
       'toolforge.wikicite-dashboard', 'wikiloves', 'ptools-useractions',
       'mm_awarder', 'mm_baglama', 'toolforge.interact-oa', 'grafa',
       'toolforge.sge-jobs', 'mm_petscan', 'mm_file_siblings_interface',
       'toolforge.fpcstats', 'toolforge.flickrdash', 'ci-last-run',
       'toolforge.facebook-messenger-chatbot', 'mm_missing_topics',
       'stashbot', 'toolforge.corhist', 'cil2',
       'toolforge.weapon-of-mass-description', 'WikiProvenance',
       'ptwikis-visual-editor', 'mediawiki-feeds', 'meta-stewardry',
       'toolforge.phab-ban', 'toolforge.enhourly', 'ramp',
       'pronuncify.net', 'mm_commonshelper', 'zhnotofu', 'wordpile-',
       'mm_isin', 'abcgames', 'erex-yomi', 'mm_wd_commons',
       'irc_logs_browser', 'toolforge.outreachy-userrank',
       'outreachy-recent-edits-tool-', 'toolforge.wm-commons-emoji-bot',
       'mm_wikidata_locations', 'toolforge.timerelengteam',
       'mm_subclass_dates', 'toolforge.editor-retention-dashboard',
       'parliamentdiagram', 'WikiLinkBot', 'toolforge.srish',
       'wikipedia-externallinks-fast-extraction', 'crossactivity',
       'wikiradio', 'toolforge.freebase', 'mm_geohack', 'intuition',
       'mm_get_item_names', 'dawiki-tools', 'toolforge.omarghridabot',
       'MainAuthors', 'mm_usual_suspects', 'WPCleaner', 'lst-guard',
       'iw-uselessprotections', 'persondata', 'mm_dnb_link_finder',
       'WDProp', 'mm_update_article_counter', 'itwikinews-rss',
       'toolforge.ipwatcher', 'mm_missing_props', 'xtools-pagescreated',
       'mapillary_WLM', 'lrtools_pages', 'mm_new_items',
       'toolforge.tts-comparison', 'blankpages', 'Gerrit Reviewer Bot',
       'toolforge.userrank', 'wikinity', 'toolforge.wpcleaner', 'twl17',
       'meta-catanalysis', 'hay-gtaa', 'dataviz', 'douglasbot',
       'mm_duplicity', 'wlm-stats', 'harvesting-data-refinery',
       'plaintexteditcounter', 'mm_wdq2graph', 'wikistats',
       'bracketbotfind', 'vandalstat', 'toolforge.swviewer',
       'toolforge.parliament-diagram-generator', 'wikiprovenance-',
       'toolforge.rfc-analyzer', 'iw-bot-statistics', 'pileviews',
       'hay-wdview', 'hay-exturl', 'analytics-quarry-web', 'listpages',
       'coverage', 'mm_toolscript', 'mm_edit_counter', 'xfd-stats',
       'toolforge.spamforbreakfast', 'wikibugs',
       'toolforge.tool-db-usage', 'wd-rank', 'yadfa', 'vada', 'xtools-ec',
       'blp-edits', 'ytcleaner', 'outreachy-wikicv',
       'schema-app-wikidata-lookup', 'pageviews', 'ajapaik2commons',
       'wikitools', 'ascal', 'siteviews', 'xtools-xagent',
       'mm_file_dupes', 'Jembot', 'mm_deep_insight', 'croptool',
       'hay-multisearch', 'mw-tool-convert', 'my-first-flask-oauth-tool',
       'NOA upload tool', 'toolforge.VideoCutTool', 'wakt',
       'toolforge.germancon-mobile', 'toolforge.simplewd', 'cewbot',
       'mm_out_of_sight', 'ptools-findfiles', 'mm_related_properties',
       'bbr-commons', 'mw-tool-oojs-ui-distro', 'langviews',
       'ptwikis-common-interwikis', 'toolforge.rangeblockfinder',
       'toolforge.dockerregistry', 'expose-data', 'lp_FMIS',
       'toolforge.spacemedia', 'mm_flickrfree', 'mm_no_information',
       'mm_herding_sheep', 'enwikistalk', 'commons-maintenance-bot',
       'CategoryWatchlistBot', 'cdnjs', 'mm_treeviews', 'mm_crosscats',
       'wikipediarequests', 'hay-sum', 'massviews', 'mormegil-randomlist',
       'WC3(WC-triple):Wikipedia Category Comprehensiveness Checker',
       'mm_catscan2', 'toolforge.vip-space-media', 'lp_BBR',
       'mm_trans_parent', 'meta-stalktoy'], dtype=object)
data_hay_tools['unavailable'].value_counts() # not true I found several not being available
0    598
Name: unavailable, dtype: int64

list_of_tools = data_hay_tools[['id','name','description','fulltext']] tools = list_of_tools.sort_values(by="id") tools.to_csv("tools.csv", index=False)

# query to toolforge parametrized with tool name
#to deploy at toolforge - else at HER or here I will need to connect to ssh and ask from command line?
#f = open('/replica.my.cnf', "r")
#lines = f.readlines()
#f.close()
username=""
password= ""
#jdbc:mariadb:
hostport= "wikidatawiki.analytics.db.svc.eqiad.wmflabs:3306"
db_name="wikidatawiki_p"
engine= "mysql+pymysql://"+username+":"+password+"@"+hostport+"/"+db_name
#mysql+pymysql://
engine = create_engine(engine)
tools_occur = dict()
def getToolOccurrence(tools):
    tooli = tools['name']
    
    #TODO: add try except and recurring try
    df= pd.read_sql_query('''select * from revision inner join comment on rev_comment_id=comment_id where comment_text LIKE %s limit 1''', engine,params=("%#"+tooli+"%",))
    if len(df)!=0:
        tools_occur[tooli] = True
    else:
        tools_occur[tooli] = False
       
data_hay_tools.apply(getToolOccurrence, axis=1)
tools_occur.to_csv("tools_in_comments.csv", index=False)
---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-17-c19dea3555ff> in <module>
----> 1 data_hay_tools.apply(getToolOccurrence, axis=1)
      2 tools_occur.to_csv("tools_in_comments.csv", index=False)

/srv/paws/lib/python3.6/site-packages/pandas/core/frame.py in apply(self, func, axis, broadcast, raw, reduce, result_type, args, **kwds)
   6485                          args=args,
   6486                          kwds=kwds)
-> 6487         return op.get_result()
   6488 
   6489     def applymap(self, func):

/srv/paws/lib/python3.6/site-packages/pandas/core/apply.py in get_result(self)
    149             return self.apply_raw()
    150 
--> 151         return self.apply_standard()
    152 
    153     def apply_empty_result(self):

/srv/paws/lib/python3.6/site-packages/pandas/core/apply.py in apply_standard(self)
    249                                           axis=self.axis,
    250                                           dummy=dummy,
--> 251                                           labels=labels)
    252                 return self.obj._constructor_sliced(result, index=labels)
    253             except Exception:

pandas/_libs/reduction.pyx in pandas._libs.reduction.reduce()

pandas/_libs/reduction.pyx in pandas._libs.reduction.Reducer.get_result()

<ipython-input-16-33fa391d3d0b> in getToolOccurrence(tools)
      4 
      5     #TODO: add try except and recurring try
----> 6     df= pd.read_sql_query('''select * from revision inner join comment on rev_comment_id=comment_id where comment_text LIKE %s limit 1''', engine,params=("%#"+tooli+"%",))
      7     if len(df)!=0:
      8         tools_occur[tooli] = True

/srv/paws/lib/python3.6/site-packages/pandas/io/sql.py in read_sql_query(sql, con, index_col, coerce_float, params, parse_dates, chunksize)
    312     return pandas_sql.read_query(
    313         sql, index_col=index_col, params=params, coerce_float=coerce_float,
--> 314         parse_dates=parse_dates, chunksize=chunksize)
    315 
    316 

/srv/paws/lib/python3.6/site-packages/pandas/io/sql.py in read_query(self, sql, index_col, coerce_float, parse_dates, params, chunksize)
   1097         args = _convert_params(sql, params)
   1098 
-> 1099         result = self.execute(*args)
   1100         columns = result.keys()
   1101 

/srv/paws/lib/python3.6/site-packages/pandas/io/sql.py in execute(self, *args, **kwargs)
    988     def execute(self, *args, **kwargs):
    989         """Simple passthrough to SQLAlchemy connectable"""
--> 990         return self.connectable.execute(*args, **kwargs)
    991 
    992     def read_table(self, table_name, index_col=None, coerce_float=True,

/srv/paws/lib/python3.6/site-packages/sqlalchemy/engine/base.py in execute(self, statement, *multiparams, **params)
   2164 
   2165         connection = self._contextual_connect(close_with_result=True)
-> 2166         return connection.execute(statement, *multiparams, **params)
   2167 
   2168     def scalar(self, statement, *multiparams, **params):

/srv/paws/lib/python3.6/site-packages/sqlalchemy/engine/base.py in execute(self, object_, *multiparams, **params)
    980         """
    981         if isinstance(object_, util.string_types[0]):
--> 982             return self._execute_text(object_, multiparams, params)
    983         try:
    984             meth = object_._execute_on_connection

/srv/paws/lib/python3.6/site-packages/sqlalchemy/engine/base.py in _execute_text(self, statement, multiparams, params)
   1153             parameters,
   1154             statement,
-> 1155             parameters,
   1156         )
   1157         if self._has_events or self.engine._has_events:

/srv/paws/lib/python3.6/site-packages/sqlalchemy/engine/base.py in _execute_context(self, dialect, constructor, statement, parameters, *args)
   1246         except BaseException as e:
   1247             self._handle_dbapi_exception(
-> 1248                 e, statement, parameters, cursor, context
   1249             )
   1250 

/srv/paws/lib/python3.6/site-packages/sqlalchemy/engine/base.py in _handle_dbapi_exception(self, e, statement, parameters, cursor, context)
   1466                 util.raise_from_cause(sqlalchemy_exception, exc_info)
   1467             else:
-> 1468                 util.reraise(*exc_info)
   1469 
   1470         finally:

/srv/paws/lib/python3.6/site-packages/sqlalchemy/util/compat.py in reraise(tp, value, tb, cause)
    127         if value.__traceback__ is not tb:
    128             raise value.with_traceback(tb)
--> 129         raise value
    130 
    131     def u(s):

/srv/paws/lib/python3.6/site-packages/sqlalchemy/engine/base.py in _execute_context(self, dialect, constructor, statement, parameters, *args)
   1242                 if not evt_handled:
   1243                     self.dialect.do_execute(
-> 1244                         cursor, statement, parameters, context
   1245                     )
   1246         except BaseException as e:

/srv/paws/lib/python3.6/site-packages/sqlalchemy/engine/default.py in do_execute(self, cursor, statement, parameters, context)
    550 
    551     def do_execute(self, cursor, statement, parameters, context=None):
--> 552         cursor.execute(statement, parameters)
    553 
    554     def do_execute_no_params(self, cursor, statement, context=None):

/srv/paws/lib/python3.6/site-packages/pymysql/cursors.py in execute(self, query, args)
    168         query = self.mogrify(query, args)
    169 
--> 170         result = self._query(query)
    171         self._executed = query
    172         return result

/srv/paws/lib/python3.6/site-packages/pymysql/cursors.py in _query(self, q)
    326         self._last_executed = q
    327         self._clear_result()
--> 328         conn.query(q)
    329         self._do_get_result()
    330         return self.rowcount

/srv/paws/lib/python3.6/site-packages/pymysql/connections.py in query(self, sql, unbuffered)
    515                 sql = sql.encode(self.encoding, 'surrogateescape')
    516         self._execute_command(COMMAND.COM_QUERY, sql)
--> 517         self._affected_rows = self._read_query_result(unbuffered=unbuffered)
    518         return self._affected_rows
    519 

/srv/paws/lib/python3.6/site-packages/pymysql/connections.py in _read_query_result(self, unbuffered)
    730         else:
    731             result = MySQLResult(self)
--> 732             result.read()
    733         self._result = result
    734         if result.server_status is not None:

/srv/paws/lib/python3.6/site-packages/pymysql/connections.py in read(self)
   1073     def read(self):
   1074         try:
-> 1075             first_packet = self.connection._read_packet()
   1076 
   1077             if first_packet.is_ok_packet():

/srv/paws/lib/python3.6/site-packages/pymysql/connections.py in _read_packet(self, packet_type)
    655         buff = b''
    656         while True:
--> 657             packet_header = self._read_bytes(4)
    658             #if DEBUG: dump_packet(packet_header)
    659 

/srv/paws/lib/python3.6/site-packages/pymysql/connections.py in _read_bytes(self, num_bytes)
    689         while True:
    690             try:
--> 691                 data = self._rfile.read(num_bytes)
    692                 break
    693             except (IOError, OSError) as e:

/usr/lib/python3.6/socket.py in readinto(self, b)
    584         while True:
    585             try:
--> 586                 return self._sock.recv_into(b)
    587             except timeout:
    588                 self._timeout_occurred = True

KeyboardInterrupt: 
df= pd.read_sql_query('''select * from revision inner join comment on rev_comment_id=comment_id where comment_text LIKE %s limit 1''', engine,params=("%#petscan%",))
---------------------------------------------------------------------------
OperationalError                          Traceback (most recent call last)
/srv/paws/lib/python3.6/site-packages/sqlalchemy/engine/base.py in _execute_context(self, dialect, constructor, statement, parameters, *args)
   1243                     self.dialect.do_execute(
-> 1244                         cursor, statement, parameters, context
   1245                     )

/srv/paws/lib/python3.6/site-packages/sqlalchemy/engine/default.py in do_execute(self, cursor, statement, parameters, context)
    551     def do_execute(self, cursor, statement, parameters, context=None):
--> 552         cursor.execute(statement, parameters)
    553 

/srv/paws/lib/python3.6/site-packages/pymysql/cursors.py in execute(self, query, args)
    169 
--> 170         result = self._query(query)
    171         self._executed = query

/srv/paws/lib/python3.6/site-packages/pymysql/cursors.py in _query(self, q)
    327         self._clear_result()
--> 328         conn.query(q)
    329         self._do_get_result()

/srv/paws/lib/python3.6/site-packages/pymysql/connections.py in query(self, sql, unbuffered)
    516         self._execute_command(COMMAND.COM_QUERY, sql)
--> 517         self._affected_rows = self._read_query_result(unbuffered=unbuffered)
    518         return self._affected_rows

/srv/paws/lib/python3.6/site-packages/pymysql/connections.py in _read_query_result(self, unbuffered)
    731             result = MySQLResult(self)
--> 732             result.read()
    733         self._result = result

/srv/paws/lib/python3.6/site-packages/pymysql/connections.py in read(self)
   1074         try:
-> 1075             first_packet = self.connection._read_packet()
   1076 

/srv/paws/lib/python3.6/site-packages/pymysql/connections.py in _read_packet(self, packet_type)
    656         while True:
--> 657             packet_header = self._read_bytes(4)
    658             #if DEBUG: dump_packet(packet_header)

/srv/paws/lib/python3.6/site-packages/pymysql/connections.py in _read_bytes(self, num_bytes)
    706             raise err.OperationalError(
--> 707                 CR.CR_SERVER_LOST, "Lost connection to MySQL server during query")
    708         return data

OperationalError: (2013, 'Lost connection to MySQL server during query')

The above exception was the direct cause of the following exception:

OperationalError                          Traceback (most recent call last)
<ipython-input-18-85bc49290c0a> in <module>
----> 1 df= pd.read_sql_query('''select * from revision inner join comment on rev_comment_id=comment_id where comment_text LIKE %s limit 1''', engine,params=("%#petscan%",))

/srv/paws/lib/python3.6/site-packages/pandas/io/sql.py in read_sql_query(sql, con, index_col, coerce_float, params, parse_dates, chunksize)
    312     return pandas_sql.read_query(
    313         sql, index_col=index_col, params=params, coerce_float=coerce_float,
--> 314         parse_dates=parse_dates, chunksize=chunksize)
    315 
    316 

/srv/paws/lib/python3.6/site-packages/pandas/io/sql.py in read_query(self, sql, index_col, coerce_float, parse_dates, params, chunksize)
   1097         args = _convert_params(sql, params)
   1098 
-> 1099         result = self.execute(*args)
   1100         columns = result.keys()
   1101 

/srv/paws/lib/python3.6/site-packages/pandas/io/sql.py in execute(self, *args, **kwargs)
    988     def execute(self, *args, **kwargs):
    989         """Simple passthrough to SQLAlchemy connectable"""
--> 990         return self.connectable.execute(*args, **kwargs)
    991 
    992     def read_table(self, table_name, index_col=None, coerce_float=True,

/srv/paws/lib/python3.6/site-packages/sqlalchemy/engine/base.py in execute(self, statement, *multiparams, **params)
   2164 
   2165         connection = self._contextual_connect(close_with_result=True)
-> 2166         return connection.execute(statement, *multiparams, **params)
   2167 
   2168     def scalar(self, statement, *multiparams, **params):

/srv/paws/lib/python3.6/site-packages/sqlalchemy/engine/base.py in execute(self, object_, *multiparams, **params)
    980         """
    981         if isinstance(object_, util.string_types[0]):
--> 982             return self._execute_text(object_, multiparams, params)
    983         try:
    984             meth = object_._execute_on_connection

/srv/paws/lib/python3.6/site-packages/sqlalchemy/engine/base.py in _execute_text(self, statement, multiparams, params)
   1153             parameters,
   1154             statement,
-> 1155             parameters,
   1156         )
   1157         if self._has_events or self.engine._has_events:

/srv/paws/lib/python3.6/site-packages/sqlalchemy/engine/base.py in _execute_context(self, dialect, constructor, statement, parameters, *args)
   1246         except BaseException as e:
   1247             self._handle_dbapi_exception(
-> 1248                 e, statement, parameters, cursor, context
   1249             )
   1250 

/srv/paws/lib/python3.6/site-packages/sqlalchemy/engine/base.py in _handle_dbapi_exception(self, e, statement, parameters, cursor, context)
   1464                 util.raise_from_cause(newraise, exc_info)
   1465             elif should_wrap:
-> 1466                 util.raise_from_cause(sqlalchemy_exception, exc_info)
   1467             else:
   1468                 util.reraise(*exc_info)

/srv/paws/lib/python3.6/site-packages/sqlalchemy/util/compat.py in raise_from_cause(exception, exc_info)
    381     exc_type, exc_value, exc_tb = exc_info
    382     cause = exc_value if exc_value is not exception else None
--> 383     reraise(type(exception), exception, tb=exc_tb, cause=cause)
    384 
    385 

/srv/paws/lib/python3.6/site-packages/sqlalchemy/util/compat.py in reraise(tp, value, tb, cause)
    126             value.__cause__ = cause
    127         if value.__traceback__ is not tb:
--> 128             raise value.with_traceback(tb)
    129         raise value
    130 

/srv/paws/lib/python3.6/site-packages/sqlalchemy/engine/base.py in _execute_context(self, dialect, constructor, statement, parameters, *args)
   1242                 if not evt_handled:
   1243                     self.dialect.do_execute(
-> 1244                         cursor, statement, parameters, context
   1245                     )
   1246         except BaseException as e:

/srv/paws/lib/python3.6/site-packages/sqlalchemy/engine/default.py in do_execute(self, cursor, statement, parameters, context)
    550 
    551     def do_execute(self, cursor, statement, parameters, context=None):
--> 552         cursor.execute(statement, parameters)
    553 
    554     def do_execute_no_params(self, cursor, statement, context=None):

/srv/paws/lib/python3.6/site-packages/pymysql/cursors.py in execute(self, query, args)
    168         query = self.mogrify(query, args)
    169 
--> 170         result = self._query(query)
    171         self._executed = query
    172         return result

/srv/paws/lib/python3.6/site-packages/pymysql/cursors.py in _query(self, q)
    326         self._last_executed = q
    327         self._clear_result()
--> 328         conn.query(q)
    329         self._do_get_result()
    330         return self.rowcount

/srv/paws/lib/python3.6/site-packages/pymysql/connections.py in query(self, sql, unbuffered)
    515                 sql = sql.encode(self.encoding, 'surrogateescape')
    516         self._execute_command(COMMAND.COM_QUERY, sql)
--> 517         self._affected_rows = self._read_query_result(unbuffered=unbuffered)
    518         return self._affected_rows
    519 

/srv/paws/lib/python3.6/site-packages/pymysql/connections.py in _read_query_result(self, unbuffered)
    730         else:
    731             result = MySQLResult(self)
--> 732             result.read()
    733         self._result = result
    734         if result.server_status is not None:

/srv/paws/lib/python3.6/site-packages/pymysql/connections.py in read(self)
   1073     def read(self):
   1074         try:
-> 1075             first_packet = self.connection._read_packet()
   1076 
   1077             if first_packet.is_ok_packet():

/srv/paws/lib/python3.6/site-packages/pymysql/connections.py in _read_packet(self, packet_type)
    655         buff = b''
    656         while True:
--> 657             packet_header = self._read_bytes(4)
    658             #if DEBUG: dump_packet(packet_header)
    659 

/srv/paws/lib/python3.6/site-packages/pymysql/connections.py in _read_bytes(self, num_bytes)
    705             self._force_close()
    706             raise err.OperationalError(
--> 707                 CR.CR_SERVER_LOST, "Lost connection to MySQL server during query")
    708         return data
    709 

OperationalError: (pymysql.err.OperationalError) (2013, 'Lost connection to MySQL server during query')
[SQL: select * from revision inner join comment on rev_comment_id=comment_id where comment_text LIKE %s limit 1]
[parameters: ('%#petscan%',)]
(Background on this error at: http://sqlalche.me/e/e3q8)