import mwparserfromhell
import pywikibot

# list of pages to amend with Wikidata items
pagetitles = []
with open('./talkpages.txt', mode='r', encoding='utf8') as fh:
    for line in fh:
        pagetitle = line.strip()
        if pagetitle != '':
            pagetitles.append(pagetitle)

pagetitles = list(reversed(pagetitles))
            
print(len(pagetitles))

simulate = False
wikidata_site = pywikibot.Site('wikidata', 'wikidata')

for i, pagetitle in enumerate(pagetitles):
    noDel = False
    page = pywikibot.Page(wikidata_site, pagetitle)
    print('#### {} (page {}/{}) ####'.format(page.title(), i+1, len(pagetitles)))
    if page.exists() == False:
        print('* page does not exist; skip')
        print()
        continue
    
    wikicode = mwparserfromhell.parse(page.get())
    wikicode_original = str(wikicode)

    for template in wikicode.filter_templates():
        if template.name.matches("Wanted footballers"):
            wikicode.remove(template)
            print('* removed template "Wanted footballers" from page "{}"'.format(page.title()))
    
    if wikicode_original == str(wikicode).strip(): # no changes
        print('* no modifications for page "{}" required'.format(page.title()))
        print()
        continue
        
    #print('* new wikitext: "{}"'.format(str(wikicode).strip()))

    contributorsCounter = page.contributors()
    contributors = []
    for contributor in contributorsCounter.elements():
        contributors.append(contributor)
    contributors = list(set(contributors))
    
    if contributors != [ 'Japan Football' ]:
        print('* complex version history: contributions by {}'.format(contributors))
        noDel = True
    
    if simulate == False:
        if noDel == False and str(wikicode).strip() == '':
            #page.delete(reason='remove [[Template:Wanted footballers]] (to be deleted), and delete empty talk page; only contributor: [[User:Japan Football]]; cf. https://www.wikidata.org/w/index.php?oldid=839363428#Category:Footballers_by_nationality_and_all_subcategories', prompt=False, mark=False, quit=True)
            print('* page "{}" deleted'.format(page.title()))
        else:
            #page.put(str(wikicode).strip(), summary='remove [[Template:Wanted footballers]] (to be deleted); cf. https://www.wikidata.org/w/index.php?oldid=839363428#Category:Footballers_by_nationality_and_all_subcategories')
            print('* page "{}" modified (new length: {} bytes)'.format(page.title(), len(str(wikicode).strip())))
    print()
    
    if i > 3:
        break
5533
#### Talk:Q20040568 (page 1/5533) ####
* removed template "Wanted footballers" from page "Talk:Q20040568"
* page "Talk:Q20040568" deleted

#### Talk:Q20040245 (page 2/5533) ####
* removed template "Wanted footballers" from page "Talk:Q20040245"
* page "Talk:Q20040245" deleted

#### Talk:Q19864403 (page 3/5533) ####
* removed template "Wanted footballers" from page "Talk:Q19864403"
* page "Talk:Q19864403" deleted

#### Talk:Q19863706 (page 4/5533) ####
* removed template "Wanted footballers" from page "Talk:Q19863706"
* page "Talk:Q19863706" deleted

#### Talk:Q19863564 (page 5/5533) ####
* removed template "Wanted footballers" from page "Talk:Q19863564"
* page "Talk:Q19863564" deleted

#### Talk:Q19847273 (page 6/5533) ####
* removed template "Wanted footballers" from page "Talk:Q19847273"
* page "Talk:Q19847273" deleted

#### Talk:Q19847268 (page 7/5533) ####
* removed template "Wanted footballers" from page "Talk:Q19847268"
* page "Talk:Q19847268" deleted

#### Talk:Q19847254 (page 8/5533) ####
* removed template "Wanted footballers" from page "Talk:Q19847254"
* page "Talk:Q19847254" deleted

#### Talk:Q19847252 (page 9/5533) ####
* removed template "Wanted footballers" from page "Talk:Q19847252"
* page "Talk:Q19847252" deleted

#### Talk:Q19823057 (page 10/5533) ####
* removed template "Wanted footballers" from page "Talk:Q19823057"
* page "Talk:Q19823057" deleted

#### Talk:Q19823054 (page 11/5533) ####
* removed template "Wanted footballers" from page "Talk:Q19823054"
* page "Talk:Q19823054" deleted

#### Talk:Q18991202 (page 12/5533) ####
* removed template "Wanted footballers" from page "Talk:Q18991202"
* page "Talk:Q18991202" deleted

#### Talk:Q18991012 (page 13/5533) ####
* removed template "Wanted footballers" from page "Talk:Q18991012"
* page "Talk:Q18991012" deleted

#### Talk:Q18818158 (page 14/5533) ####
* removed template "Wanted footballers" from page "Talk:Q18818158"
* page "Talk:Q18818158" deleted

#### Talk:Q18818149 (page 15/5533) ####
* removed template "Wanted footballers" from page "Talk:Q18818149"
* page "Talk:Q18818149" deleted

#### Talk:Q18818115 (page 16/5533) ####
* removed template "Wanted footballers" from page "Talk:Q18818115"
* page "Talk:Q18818115" deleted

#### Talk:Q18818052 (page 17/5533) ####
* removed template "Wanted footballers" from page "Talk:Q18818052"
* page "Talk:Q18818052" deleted

#### Talk:Q18817948 (page 18/5533) ####
* removed template "Wanted footballers" from page "Talk:Q18817948"
* page "Talk:Q18817948" deleted

#### Talk:Q18817931 (page 19/5533) ####
* removed template "Wanted footballers" from page "Talk:Q18817931"
* page "Talk:Q18817931" deleted

#### Talk:Q18817863 (page 20/5533) ####
* removed template "Wanted footballers" from page "Talk:Q18817863"
* page "Talk:Q18817863" deleted

#### Talk:Q18817862 (page 21/5533) ####
* removed template "Wanted footballers" from page "Talk:Q18817862"
* page "Talk:Q18817862" deleted

#### Talk:Q18817848 (page 22/5533) ####
* removed template "Wanted footballers" from page "Talk:Q18817848"
* page "Talk:Q18817848" deleted

#### Talk:Q18817703 (page 23/5533) ####
* removed template "Wanted footballers" from page "Talk:Q18817703"
* page "Talk:Q18817703" deleted

#### Talk:Q18817514 (page 24/5533) ####
* removed template "Wanted footballers" from page "Talk:Q18817514"
* page "Talk:Q18817514" deleted

#### Talk:Q18817486 (page 25/5533) ####
* removed template "Wanted footballers" from page "Talk:Q18817486"
* page "Talk:Q18817486" deleted

#### Talk:Q18817435 (page 26/5533) ####
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
/srv/paws/lib/python3.6/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
    376             try:  # Python 2.7, use buffering of HTTP responses
--> 377                 httplib_response = conn.getresponse(buffering=True)
    378             except TypeError:  # Python 2.6 and older, Python 3

TypeError: getresponse() got an unexpected keyword argument 'buffering'

During handling of the above exception, another exception occurred:

KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-16-61f2a1c24114> in <module>()
     21     page = pywikibot.Page(wikidata_site, pagetitle)
     22     print('#### {} (page {}/{}) ####'.format(page.title(), i+1, len(pagetitles)))
---> 23     if page.exists() == False:
     24         print('* page does not exist; skip')
     25         print()

/srv/paws/pwb/pywikibot/page.py in exists(self)
    804         @rtype: bool
    805         """
--> 806         return self.pageid > 0
    807 
    808     @property

/srv/paws/pwb/pywikibot/page.py in pageid(self)
    285         """
    286         if not hasattr(self, '_pageid'):
--> 287             self.site.loadpageinfo(self)
    288         return self._pageid
    289 

/srv/paws/pwb/pywikibot/site.py in loadpageinfo(self, page, preload)
   3024                                 titles=title.encode(self.encoding()),
   3025                                 inprop=inprop)
-> 3026         self._update_page(page, query)
   3027 
   3028     @need_extension('GeoData')

/srv/paws/pwb/pywikibot/site.py in _update_page(self, page, query)
   3007 
   3008     def _update_page(self, page, query):
-> 3009         for pageitem in query:
   3010             if not self.sametitle(pageitem['title'],
   3011                                   page.title(with_section=False)):

/srv/paws/pwb/pywikibot/data/api.py in __iter__(self)
   3142         """Yield results."""
   3143         self._previous_dicts = {}
-> 3144         for result in super(PropertyGenerator, self).__iter__():
   3145             yield result
   3146         for result in self._previous_dicts.values():

/srv/paws/pwb/pywikibot/data/api.py in __iter__(self)
   2966                 prev_limit, new_limit, previous_result_had_data)
   2967             if not hasattr(self, 'data'):
-> 2968                 self.data = self.request.submit()
   2969             if not self.data or not isinstance(self.data, dict):
   2970                 pywikibot.debug(

/srv/paws/pwb/pywikibot/data/api.py in submit(self)
   2173                                                                    paramstring)
   2174             rawdata, use_get = self._http_request(use_get, uri, body, headers,
-> 2175                                                   paramstring)
   2176             if rawdata is None:
   2177                 continue

/srv/paws/pwb/pywikibot/data/api.py in _http_request(self, use_get, uri, body, headers, paramstring)
   1932                 site=self.site, uri=uri,
   1933                 method='GET' if use_get else 'POST',
-> 1934                 body=body, headers=headers)
   1935         except Server504Error:
   1936             pywikibot.log('Caught HTTP 504 error; retrying')

/srv/paws/pwb/pywikibot/tools/__init__.py in wrapper(*__args, **__kw)
   1735                              cls, depth)
   1736                     del __kw[old_arg]
-> 1737             return obj(*__args, **__kw)
   1738 
   1739         if not __debug__:

/srv/paws/pwb/pywikibot/comms/http.py in request(site, uri, method, params, body, headers, data, **kwargs)
    322 
    323     baseuri = site.base_url(uri)
--> 324     r = fetch(baseuri, method, params, body, headers, **kwargs)
    325     site.throttle.retry_after = int(r.response_headers.get('retry-after', 0))
    326     return r.text

/srv/paws/pwb/pywikibot/comms/http.py in fetch(uri, method, params, body, headers, default_error_handling, use_fake_user_agent, data, **kwargs)
    519             headers['user-agent'] = fake_user_agent()
    520 
--> 521     request = _enqueue(uri, method, params, body, headers, **kwargs)
    522     # if there's no data in the answer we're in trouble
    523     assert request._data is not None

/srv/paws/pwb/pywikibot/comms/http.py in _enqueue(uri, method, params, body, headers, data, **kwargs)
    475     request = threadedhttp.HttpRequest(
    476         uri, method, params, body, all_headers, callbacks, **kwargs)
--> 477     _http_process(session, request)
    478     return request
    479 

/srv/paws/pwb/pywikibot/comms/http.py in _http_process(session, http_request)
    389                                    headers=headers, auth=auth, timeout=timeout,
    390                                    verify=not ignore_validation,
--> 391                                    **http_request.kwargs)
    392     except Exception as e:
    393         http_request.data = e

/srv/paws/lib/python3.6/site-packages/requests/sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
    510         }
    511         send_kwargs.update(settings)
--> 512         resp = self.send(prep, **send_kwargs)
    513 
    514         return resp

/srv/paws/lib/python3.6/site-packages/requests/sessions.py in send(self, request, **kwargs)
    620 
    621         # Send the request
--> 622         r = adapter.send(request, **kwargs)
    623 
    624         # Total elapsed time of the request (approximately)

/srv/paws/lib/python3.6/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
    443                     decode_content=False,
    444                     retries=self.max_retries,
--> 445                     timeout=timeout
    446                 )
    447 

/srv/paws/lib/python3.6/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    598                                                   timeout=timeout_obj,
    599                                                   body=body, headers=headers,
--> 600                                                   chunked=chunked)
    601 
    602             # If we're going to release the connection in ``finally:``, then

/srv/paws/lib/python3.6/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
    378             except TypeError:  # Python 2.6 and older, Python 3
    379                 try:
--> 380                     httplib_response = conn.getresponse()
    381                 except Exception as e:
    382                     # Remove the TypeError from the exception chain in Python 3;

/usr/lib/python3.6/http/client.py in getresponse(self)
   1329         try:
   1330             try:
-> 1331                 response.begin()
   1332             except ConnectionError:
   1333                 self.close()

/usr/lib/python3.6/http/client.py in begin(self)
    295         # read until we get a non-100 response
    296         while True:
--> 297             version, status, reason = self._read_status()
    298             if status != CONTINUE:
    299                 break

/usr/lib/python3.6/http/client.py in _read_status(self)
    256 
    257     def _read_status(self):
--> 258         line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
    259         if len(line) > _MAXLINE:
    260             raise LineTooLong("status line")

/usr/lib/python3.6/socket.py in readinto(self, b)
    584         while True:
    585             try:
--> 586                 return self._sock.recv_into(b)
    587             except timeout:
    588                 self._timeout_occurred = True

/usr/lib/python3.6/ssl.py in recv_into(self, buffer, nbytes, flags)
   1007                   "non-zero flags not allowed in calls to recv_into() on %s" %
   1008                   self.__class__)
-> 1009             return self.read(nbytes, buffer)
   1010         else:
   1011             return socket.recv_into(self, buffer, nbytes, flags)

/usr/lib/python3.6/ssl.py in read(self, len, buffer)
    869             raise ValueError("Read on closed or unwrapped SSL socket.")
    870         try:
--> 871             return self._sslobj.read(len, buffer)
    872         except SSLError as x:
    873             if x.args[0] == SSL_ERROR_EOF and self.suppress_ragged_eofs:

/usr/lib/python3.6/ssl.py in read(self, len, buffer)
    629         """
    630         if buffer is not None:
--> 631             v = self._sslobj.read(len, buffer)
    632         else:
    633             v = self._sslobj.read(len)

KeyboardInterrupt: