import pywikibot
from pywikibot import pagegenerators as pg
import json

site = pywikibot.Site("sv", "wiktionary")
DATA_FILENAME = "citat.json"



def list_template_usage(site_obj, tmpl_name):
    """
    Takes Site object and template name and returns a generator.

    The function expects a Site object (pywikibot.Site()) and
    a template name (String). It creates a list of all
    pages using that template and returns them as a generator.
    It only returns pages in the 121-namespace (property-talk-pages).
    The generator will load 50 pages at a time for iteration.
    """
    name = "{}:{}".format(site.namespace(10), tmpl_name)
    tmpl_page = pywikibot.Page(site, name)
    ref_gen = pg.ReferringPageGenerator(tmpl_page, onlyTemplateInclusion=True)
    filter_gen = pg.NamespaceFilterPageGenerator(ref_gen, namespaces=[0])
    generator = site.preloadpages(filter_gen, pageprops=True)
    return generator

citat_gen = list_template_usage(site, "citat")

with open(DATA_FILENAME, mode='w', encoding='utf-8') as f:
    json.dump([], f)
    
quotes = []
page_count = 0
quote_count = 0

for page in citat_gen:
    page_count = page_count + 1
    page_str = page.get()
    tmpl_list = pywikibot.textlib.extract_templates_and_params(page_str)
    if page_count % 10 == 0:
        print("Processed {} pages.".format(page_count))
    for tmpl in tmpl_list:
        if "citat" in tmpl:
            quote_count = quote_count + 1
            t_object = {}
            t_object["year"] = tmpl[1].get("år")
            t_object["author"] = tmpl[1].get("förf")
            t_object["date"] = tmpl[1].get("datum")
            t_object["publication"] = tmpl[1].get("publ")
            t_object["content"] = tmpl[1].get("1")
            if quote_count % 10 == 0:
                print("Processed {} quotes".format(quote_count))
            with open(DATA_FILENAME, mode='w', encoding='utf-8') as feedsjson:
                quotes.append(t_object)
                json.dump(quotes, feedsjson, ensure_ascii=False)
            
Retrieving 50 pages from wiktionary:sv.
Processed 10 quotes
Processed 20 quotes
Processed 10 pages.
Processed 30 quotes
Processed 20 pages.
Processed 40 quotes
Processed 50 quotes
Processed 30 pages.
Processed 60 quotes
Processed 40 pages.
Processed 70 quotes
Processed 80 quotes
Processed 50 pages.
Retrieving 50 pages from wiktionary:sv.
Processed 90 quotes
Processed 100 quotes
Processed 60 pages.
Processed 110 quotes
Processed 70 pages.
Processed 120 quotes
Processed 80 pages.
Processed 130 quotes
Processed 90 pages.
Processed 140 quotes
Processed 100 pages.
Processed 150 quotes
Retrieving 50 pages from wiktionary:sv.
Processed 160 quotes
Processed 170 quotes
Processed 110 pages.
Processed 180 quotes
Processed 120 pages.
Processed 190 quotes
Processed 130 pages.
Processed 200 quotes
Processed 210 quotes
Processed 140 pages.
Processed 220 quotes
Processed 230 quotes
Processed 150 pages.
Retrieving 50 pages from wiktionary:sv.
Processed 240 quotes
Processed 250 quotes
Processed 160 pages.
Processed 260 quotes
Processed 270 quotes
Processed 170 pages.
Processed 280 quotes
Processed 290 quotes
Processed 180 pages.
Processed 300 quotes
Processed 190 pages.
Processed 310 quotes
Processed 200 pages.
Retrieving 50 pages from wiktionary:sv.
Processed 320 quotes
Processed 210 pages.
Processed 330 quotes
Processed 220 pages.
Processed 340 quotes
Processed 350 quotes
Processed 230 pages.
Processed 360 quotes
Processed 240 pages.
Processed 370 quotes
Processed 250 pages.
Retrieving 50 pages from wiktionary:sv.
Processed 380 quotes
Processed 390 quotes
Processed 260 pages.
Processed 400 quotes
Processed 270 pages.
Processed 410 quotes
Processed 280 pages.
Processed 420 quotes
Processed 430 quotes
Processed 290 pages.
Processed 440 quotes
Processed 300 pages.
Retrieving 50 pages from wiktionary:sv.
Processed 450 quotes
Processed 310 pages.
Processed 460 quotes
Processed 320 pages.
Processed 470 quotes
Processed 480 quotes
Processed 330 pages.
Processed 490 quotes
Processed 340 pages.
Processed 500 quotes
Processed 350 pages.
Retrieving 50 pages from wiktionary:sv.
Processed 510 quotes
Processed 360 pages.
Processed 520 quotes
Processed 370 pages.
Processed 530 quotes
Processed 540 quotes
Processed 380 pages.
Processed 550 quotes
Processed 390 pages.
Processed 560 quotes
Processed 400 pages.
Processed 570 quotes
Retrieving 50 pages from wiktionary:sv.
Processed 580 quotes
Processed 410 pages.
Processed 590 quotes
Processed 420 pages.
Processed 600 quotes
Processed 430 pages.
Processed 610 quotes
Processed 440 pages.
Processed 620 quotes
Processed 450 pages.
Processed 630 quotes
Retrieving 50 pages from wiktionary:sv.
Processed 640 quotes
Processed 460 pages.
Processed 650 quotes
Processed 470 pages.
Processed 660 quotes
Processed 670 quotes
Processed 480 pages.
Processed 680 quotes
Processed 490 pages.
Processed 690 quotes
Processed 500 pages.
Retrieving 50 pages from wiktionary:sv.
Processed 700 quotes
Processed 510 pages.
Processed 710 quotes
Processed 520 pages.
Processed 720 quotes
Processed 730 quotes
Processed 530 pages.
Processed 740 quotes
Processed 540 pages.
Processed 750 quotes
Processed 550 pages.
Retrieving 50 pages from wiktionary:sv.
Processed 760 quotes
Processed 560 pages.
Processed 770 quotes
Processed 570 pages.
Processed 780 quotes
Processed 790 quotes
Processed 580 pages.
Processed 800 quotes
Processed 590 pages.
Processed 810 quotes
Processed 600 pages.
Retrieving 50 pages from wiktionary:sv.
Processed 820 quotes
Processed 830 quotes
Processed 610 pages.
Processed 840 quotes
Processed 620 pages.
Processed 850 quotes
Processed 630 pages.
Processed 860 quotes
Processed 870 quotes
Processed 640 pages.
Processed 880 quotes
Processed 650 pages.
Retrieving 50 pages from wiktionary:sv.
Processed 890 quotes
Processed 660 pages.
Processed 900 quotes
Processed 910 quotes
Processed 670 pages.
Processed 920 quotes
Processed 680 pages.
Processed 930 quotes
Processed 690 pages.
Processed 940 quotes
Processed 700 pages.
Retrieving 50 pages from wiktionary:sv.
Processed 950 quotes
Processed 710 pages.
Processed 960 quotes
Processed 970 quotes
Processed 720 pages.
Processed 980 quotes
Processed 730 pages.
Processed 990 quotes
Processed 740 pages.
Processed 1000 quotes
Processed 750 pages.
Retrieving 50 pages from wiktionary:sv.
Processed 1010 quotes
Processed 1020 quotes
Processed 760 pages.
Processed 1030 quotes
Processed 770 pages.
Processed 1040 quotes
Processed 780 pages.
Processed 1050 quotes
Processed 790 pages.
Processed 1060 quotes
Processed 1070 quotes
Processed 800 pages.
Retrieving 50 pages from wiktionary:sv.
Processed 1080 quotes
Processed 810 pages.
Processed 1090 quotes
Processed 820 pages.
Processed 1100 quotes
Processed 1110 quotes
Processed 830 pages.
Processed 1120 quotes
Processed 840 pages.
Processed 1130 quotes
Processed 850 pages.
Retrieving 50 pages from wiktionary:sv.
Processed 1140 quotes
Processed 860 pages.
Processed 1150 quotes
Processed 1160 quotes
Processed 870 pages.
Processed 1170 quotes
Processed 1180 quotes
Processed 880 pages.
Processed 1190 quotes
Processed 890 pages.
Processed 1200 quotes
Processed 900 pages.
Retrieving 50 pages from wiktionary:sv.
Processed 1210 quotes
Processed 910 pages.
Processed 1220 quotes
Processed 920 pages.
Processed 1230 quotes
Processed 1240 quotes
Processed 930 pages.
Processed 1250 quotes
Processed 940 pages.
Processed 1260 quotes
Processed 1270 quotes
Processed 950 pages.
Retrieving 50 pages from wiktionary:sv.
Processed 1280 quotes
Processed 960 pages.
Processed 1290 quotes
Processed 970 pages.
Processed 1300 quotes
Processed 980 pages.