import pywikibot
from pywikibot import pagegenerators
from pywikibot.data.api import APIError

wikidata_site = pywikibot.Site('wikidata', 'wikidata')
repo = wikidata_site.data_repository()

# lack of given names : https://query.wikidata.org/#SELECT%20%3Fcoc%20%3FcocLabel%20%28COUNT%28%2a%29%20AS%20%3Fcnt%29%20WHERE%20%7B%0A%20%20%3Fitem%20p%3AP106%2Fps%3AP106%2Fwdt%3AP279%2a%20wd%3AQ26270618%3B%20wdt%3AP27%20%3Fcoc%20.%0A%20%20MINUS%20%7B%20%3Fitem%20wdt%3AP735%20%5B%5D%20%7D%0A%20%20SERVICE%20wikibase%3Alabel%20%7B%20bd%3AserviceParam%20wikibase%3Alanguage%20%27en%27%20%7D%0A%7D%20GROUP%20BY%20%3Fcoc%20%3FcocLabel%20ORDER%20BY%20DESC%28%3Fcnt%29
# lack of family names: https://query.wikidata.org/#SELECT%20%3Fcoc%20%3FcocLabel%20%28COUNT%28%2a%29%20AS%20%3Fcnt%29%20WHERE%20%7B%0A%20%20%3Fitem%20p%3AP106%2Fps%3AP106%2Fwdt%3AP279%2a%20wd%3AQ26270618%3B%20wdt%3AP27%20%3Fcoc%20.%0A%20%20MINUS%20%7B%20%3Fitem%20wdt%3AP734%20%5B%5D%20%7D%0A%20%20SERVICE%20wikibase%3Alabel%20%7B%20bd%3AserviceParam%20wikibase%3Alanguage%20%27en%27%20%7D%0A%7D%20GROUP%20BY%20%3Fcoc%20%3FcocLabel%20ORDER%20BY%20DESC%28%3Fcnt%29
countryOfCitizenships = [ 'Q37', 'Q33946', 'Q45', 'Q258', 'Q215', 'Q27', 'Q211', 'Q43', 'Q34', 'Q801', 'Q214', 'Q16957', 'Q191', 'Q20', 'Q43287', 'Q713750', 'Q954', 'Q33', 'Q252', 'Q217' ] # 

nameProps = [ 'P735', 'P734' ]
editSummary = 'add [[Property:{}]] claim based on English label'

dataset_query = """SELECT DISTINCT ?item WHERE {{
  ?item p:P106/ps:P106/wdt:P279* wd:Q26270618; wdt:P27 wd:{} .
  MINUS {{ ?item wdt:{} [] }}
}}"""

nameQuery = """SELECT DISTINCT ?item WHERE {{
  ?item wdt:P31/wdt:P279* wd:{}; wdt:P1705 ?native_label .
  FILTER(STR(?native_label) = "{}") .
}}"""

nameItemsRaw = {
    'P735':{'item':'Q202444','prop':'P735','type':'givenname'},
    'P734':{'item':'Q101352','prop':'P734','type':'familyname'}
}

def lookupName(nameProp, nameString):
    if nameProp not in nameItemsRaw:
        print('Unknown nameType: {}; skip item lookup'.format(nameProp))
        return None

    nameItems = [] # elements will have pywikibot.ItemPage type
    for item in pywikibot.pagegenerators.WikidataSPARQLPageGenerator(nameQuery.format(nameItemsRaw[nameProp]['item'], nameString), site=wikidata_site):
        if type(item) is pywikibot.ItemPage:
            nameItems.append(item)
            #print('Found {} item for {}: {}'.format(nameItemsRaw[nameProp]['type'], nameString, item))
        else:
            print('Found query result with wrong data type: {}'.format(type(item)))
    if len(nameItems)==1:
        return nameItems[0]

    if len(nameItems)>1:
        print('Found {} name items for {} "{}"'.format(len(nameItems), nameItemsRaw[nameProp]['type'], nameString))
    return None

for countryOfCitizenship in countryOfCitizenships:
    print('#### country of citizenship: {} ####'.format(countryOfCitizenship))
    for nameProp in nameProps:
        print('## nameProp: {} ##'.format(nameProp))
        for Qitem in pagegenerators.WikidataSPARQLPageGenerator(dataset_query.format(countryOfCitizenship, nameProp), site=wikidata_site):
            Qitem.get()

            if Qitem.claims and nameProp in Qitem.claims: # name property already set
                continue

            if not Qitem.labels or 'en' not in Qitem.labels:
                continue

            name = Qitem.labels['en'].split(' ')
            if len(name) != 2:
                continue

            if nameProp=='P735':
                nameString = name[0]
            elif nameProp=='P734':
                nameString = name[1]
            else:
                continue

            nameItem = lookupName(nameProp, nameString.strip())
            if nameItem==None:
                continue
            if type(nameItem) is not pywikibot.ItemPage:
                continue
            nameItem.get()
            nameItemNativeLabel = nameItem.claims['P1705'][0].getTarget()

            nameClaim = pywikibot.Claim(repo, nameProp)
            nameClaim.setTarget(value=nameItem)

            try:
                Qitem.addClaim(nameClaim, summary=editSummary.format(nameProp))
                print('Item {}: "{}" add item {} ("{}"@{})'.format(Qitem.title(), Qitem.labels['en'], nameItem.title(), nameItemNativeLabel.text, nameItemNativeLabel.language))
            except APIError as e:
                pass

            #if len(inobj['givennames'].split(' '))>1:
            #    ordinalQualifier = pywikibot.Claim(repo, 'P1545')
            #    ordinalQualifier.setTarget(value='{}'.format(ordinal+1))
            #    givennameClaim.addQualifier(ordinalQualifier)
        print()
    print()    

print('All done, job finished')
#### country of citizenship: Q37 ####
## nameProp: P735 ##

## nameProp: P734 ##
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
/srv/paws/lib/python3.6/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
    376             try:  # Python 2.7, use buffering of HTTP responses
--> 377                 httplib_response = conn.getresponse(buffering=True)
    378             except TypeError:  # Python 2.6 and older, Python 3

TypeError: getresponse() got an unexpected keyword argument 'buffering'

During handling of the above exception, another exception occurred:

KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-1-cc53460bf395> in <module>()
     71                 continue
     72 
---> 73             nameItem = lookupName(nameProp, nameString.strip())
     74             if nameItem==None:
     75                 continue

<ipython-input-1-cc53460bf395> in lookupName(nameProp, nameString)
     34 
     35     nameItems = [] # elements will have pywikibot.ItemPage type
---> 36     for item in pywikibot.pagegenerators.WikidataSPARQLPageGenerator(nameQuery.format(nameItemsRaw[nameProp]['item'], nameString), site=wikidata_site):
     37         if type(item) is pywikibot.ItemPage:
     38             nameItems.append(item)

/srv/paws/pwb/pywikibot/pagegenerators.py in WikidataSPARQLPageGenerator(query, site, item_name, endpoint, entity_url, result_type)
   2968     data = query_object.get_items(query,
   2969                                   item_name=item_name,
-> 2970                                   result_type=result_type)
   2971     items_pages = (pywikibot.ItemPage(repo, item) for item in data
   2972                    if pywikibot.ItemPage.is_valid_id(item))

/srv/paws/pwb/pywikibot/data/sparql.py in get_items(self, query, item_name, result_type)
    192         @rtype: same as result_type
    193         """
--> 194         res = self.select(query, full_data=True)
    195         if res:
    196             return result_type(r[item_name].getID() for r in res)

/srv/paws/pwb/pywikibot/data/sparql.py in select(self, query, full_data, headers)
    112         @return: List of query results or None if query failed
    113         """
--> 114         data = self.query(query, headers=headers)
    115         if data and 'results' in data:
    116             result = []

/srv/paws/pwb/pywikibot/data/sparql.py in query(self, query, headers)
    145         while True:
    146             try:
--> 147                 self.last_response = http.fetch(url, headers=headers)
    148                 if not self.last_response.text:
    149                     return None

/srv/paws/pwb/pywikibot/comms/http.py in fetch(uri, method, params, body, headers, default_error_handling, use_fake_user_agent, data, **kwargs)
    519             headers['user-agent'] = fake_user_agent()
    520 
--> 521     request = _enqueue(uri, method, params, body, headers, **kwargs)
    522     # if there's no data in the answer we're in trouble
    523     assert request._data is not None

/srv/paws/pwb/pywikibot/comms/http.py in _enqueue(uri, method, params, body, headers, data, **kwargs)
    475     request = threadedhttp.HttpRequest(
    476         uri, method, params, body, all_headers, callbacks, **kwargs)
--> 477     _http_process(session, request)
    478     return request
    479 

/srv/paws/pwb/pywikibot/comms/http.py in _http_process(session, http_request)
    389                                    headers=headers, auth=auth, timeout=timeout,
    390                                    verify=not ignore_validation,
--> 391                                    **http_request.kwargs)
    392     except Exception as e:
    393         http_request.data = e

/srv/paws/lib/python3.6/site-packages/requests/sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
    510         }
    511         send_kwargs.update(settings)
--> 512         resp = self.send(prep, **send_kwargs)
    513 
    514         return resp

/srv/paws/lib/python3.6/site-packages/requests/sessions.py in send(self, request, **kwargs)
    620 
    621         # Send the request
--> 622         r = adapter.send(request, **kwargs)
    623 
    624         # Total elapsed time of the request (approximately)

/srv/paws/lib/python3.6/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
    443                     decode_content=False,
    444                     retries=self.max_retries,
--> 445                     timeout=timeout
    446                 )
    447 

/srv/paws/lib/python3.6/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    598                                                   timeout=timeout_obj,
    599                                                   body=body, headers=headers,
--> 600                                                   chunked=chunked)
    601 
    602             # If we're going to release the connection in ``finally:``, then

/srv/paws/lib/python3.6/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
    378             except TypeError:  # Python 2.6 and older, Python 3
    379                 try:
--> 380                     httplib_response = conn.getresponse()
    381                 except Exception as e:
    382                     # Remove the TypeError from the exception chain in Python 3;

/usr/lib/python3.6/http/client.py in getresponse(self)
   1329         try:
   1330             try:
-> 1331                 response.begin()
   1332             except ConnectionError:
   1333                 self.close()

/usr/lib/python3.6/http/client.py in begin(self)
    295         # read until we get a non-100 response
    296         while True:
--> 297             version, status, reason = self._read_status()
    298             if status != CONTINUE:
    299                 break

/usr/lib/python3.6/http/client.py in _read_status(self)
    256 
    257     def _read_status(self):
--> 258         line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
    259         if len(line) > _MAXLINE:
    260             raise LineTooLong("status line")

/usr/lib/python3.6/socket.py in readinto(self, b)
    584         while True:
    585             try:
--> 586                 return self._sock.recv_into(b)
    587             except timeout:
    588                 self._timeout_occurred = True

/usr/lib/python3.6/ssl.py in recv_into(self, buffer, nbytes, flags)
   1007                   "non-zero flags not allowed in calls to recv_into() on %s" %
   1008                   self.__class__)
-> 1009             return self.read(nbytes, buffer)
   1010         else:
   1011             return socket.recv_into(self, buffer, nbytes, flags)

/usr/lib/python3.6/ssl.py in read(self, len, buffer)
    869             raise ValueError("Read on closed or unwrapped SSL socket.")
    870         try:
--> 871             return self._sslobj.read(len, buffer)
    872         except SSLError as x:
    873             if x.args[0] == SSL_ERROR_EOF and self.suppress_ragged_eofs:

/usr/lib/python3.6/ssl.py in read(self, len, buffer)
    629         """
    630         if buffer is not None:
--> 631             v = self._sslobj.read(len, buffer)
    632         else:
    633             v = self._sslobj.read(len)

KeyboardInterrupt: