import pywikibot
import re
from pywikibot import pagegenerators
site = pywikibot.Site('de')

anchorRegLeft  = '(?P<left>((region|REGION|Region)(-ISO| ISO|ISO|-iso| iso|)|ISO-CODE|ISO-Code)\s*=\s*([A-Z]+(\-[A-Z0-9]+)?\/)*)'
anchorRegRight = '(?P<right>[^A-Z0-9\-])' # do mot match only part of code
# match parameter names
# match ' = '
# match multi regions like AT-7/DE-BY
anchorRefLeft  = '\g<left>' # refer to match of anchorReg
anchorRefRight = '\g<right>'

# sample replacement
# out = re.sub(r"(((region|REGION|Region)(-ISO| ISO|-iso| iso|)|ISO-CODE|ISO-Code)\s*=\s*([A-Z]+(\-[A-Z0-9]+)?\/)*)CN-33","\g<1>CN-ZJ",out)

# global pages like global lists not matched by country category
# add new lists on demand
globalPageNames = [
                u'Global Stratotype Section and Point',
                u'Weltraumbahnhof',
                u'DP World',
                u'Eisenbahnteststrecke',
                u'Liste von Regattastrecken',
                u'Links- und Rechtsverkehr',
                u'Liste der größten Bogenbrücken',
                u'Liste von Leuchttürmen in Asien',
                u'Liste in der europäischen Expansion entdeckter Inseln',
                u'Liste der höchsten Leuchttürme der Welt',
                u'Liste der höchsten Inseln der Erde',
                u'Liste von Karstquellen',
              ]
globalPagesGenerator = []
for n in globalPageNames:
    globalPagesGenerator.append(pywikibot.Page(site, n))

namespaces = [0] # namespaces to operate on

# use the iso3166changer for irregular, externally triggered updates
# use the iso3166fixer for regular code fixes for only a small number of occurrences
class iso3166changer:
    """Describe the change needed for a set of ISO 3166 changes"""
    countryCode = '' # defined in derived class
    editcomment = '' # defined in derived class, without pre and post, or defaulted
    editcommentPre = 'ISO 3166 Bot: '
    #editcommentPost = '; weitere / offene Fehler unter https://quarry.wmflabs.org/query/11685'
    editcommentPost = ''
    isoRefDate = '' # reference date of publication of new set of iso codes, in German, effective date of change
    
    category = '' # root category, without namespace

    replacements = [] # an array of pairs, defined in derived class
    extraPagesNames = [] # list of special page names that are not matched by category, defined in derived class
    
    def execute(self, wikitext): # may be overwritten in derived class
        out = wikitext
        for i in range(len(self.replacements)):
            out = re.sub(anchorRegLeft+self.replacements[i][0]+anchorRegRight,
                         anchorRefLeft+self.replacements[i][1]+anchorRefRight,
                         out)
        return out
    def getEditcomment(self):
        if (self.editcomment == ''):
            return self.editcommentPre + 'ISO-Code Änderungen gemäß [[ISO 3166-2:' + self.countryCode + \
                ']] (https://www.iso.org/obp/ui/#iso:code:3166:' + self.countryCode + '), gültig seit ' + \
                self.isoRefDate + \
                self.editcommentPost
        else:
            return self.editcommentPre + self.editcomment + self.editcommentPost
    
    def extraPagesGenerator(self):
        "returns a page generator for the extra pages"
        extraPages = []
        for n in self.extraPagesNames:
            extraPages.append(pywikibot.Page(site, n))
        return extraPages
    
    def searchFilterGenerator(self):
        return None


class iso3166fixer(iso3166changer):
    # fix a set of mistakes found quite often
    def getEditcomment(self):
        if (self.editcomment == ''):
            return self.editcommentPre + 'ISO-Code Änderungen gemäß [[ISO 3166-2:' + self.countryCode + \
                ']], falschen Code korrigiert' + \
                self.editcommentPost
        else:
            return self.editcommentPre + self.editcomment + self.editcommentPost

    def searchFilterGenerator(self):
        if len(self.replacements) == 0:
            return None
        else:
            searchFilter = []
            for i in range(len(self.replacements)):
                search = 'insource:"' + self.replacements[i][0] + '" insource:/' + self.replacements[i][0] + '/'
                searchFilter.append(pagegenerators.SearchPageGenerator(search, namespaces=namespaces, site=site))
                print ('search filter', search, 'added')
            if len(self.replacements) == 1:
                return searchFilter[0]
            else:
                return pagegenerators.CombinedPageGenerator(searchFilter) # OR the filters
# do not write back, for test only!
class TestChanger(iso3166fixer):
    countryCode = 'DE' # Deutschland
    category = 'Verkehr (Berlin)'

    replacements = [
                ['DE-B', 'DE-BE'],
                ['DE-(BB|BE)',    'DE-XXX'], # does not match both in a line ??
                #['DE-BB',    'DE-XXX'],
                #['DE-BE',    'DE-XXX'],
            ]
class AL(iso3166changer):
    countryCode = 'AL' # Albanien
    category = 'Albanien'
    isoRefDate = '2015-11-27'

    replacements = [
                ['AL-(BR|KC|SK)',    'AL-01'],
                ['AL-(DR|KR)',       'AL-02'],
                ['AL-(EL|GR|LB|PQ)', 'AL-03'],
                ['AL-(FR|LU|MK)',    'AL-04'],
                ['AL-(GJ|PR|TE)',    'AL-05'],
                ['AL-(DV|ER|KO|PG)', 'AL-06'],
                ['AL-(HA|KU|TP)',    'AL-07'],
                ['AL-(KB|LE|MR)',    'AL-08'],
                ['AL-(BU|DI|MT)',    'AL-09'],
                ['AL-(MM|PU|SH)',    'AL-10'],
                ['AL-(KA|TR)',       'AL-11'],
                ['AL-(DL|SR|VL)',    'AL-12'],
            ]
class CN(iso3166fixer):
    countryCode = 'CN' # China
    category = 'China'
    isoRefDate = '2017-11-23'

    replacements = [
                ['CN-11', 'CN-BJ'],
                ['CN-12', 'CN-TJ'],
                ['CN-13', 'CN-HE'],
                ['CN-14', 'CN-SX'],
                ['CN-15', 'CN-NM'],
                ['CN-21', 'CN-LN'],
                ['CN-22', 'CN-JL'],
                ['CN-23', 'CN-HL'],
                ['CN-31', 'CN-SH'],
                ['CN-32', 'CN-JS'],
                ['CN-33', 'CN-ZJ'],
                ['CN-34', 'CN-AH'],
                ['CN-35', 'CN-FJ'],
                ['CN-36', 'CN-JX'],
                ['CN-37', 'CN-SD'],
                ['CN-41', 'CN-HA'],
                ['CN-42', 'CN-HB'],
                ['CN-43', 'CN-HN'],
                ['CN-44', 'CN-GD'],
                ['CN-45', 'CN-GX'],
                ['CN-46', 'CN-HI'],
                ['CN-50', 'CN-CQ'],
                ['CN-51', 'CN-SC'],
                ['CN-52', 'CN-GZ'],
                ['CN-53', 'CN-YN'],
                ['CN-54', 'CN-XZ'],
                ['CN-61', 'CN-SN'],
                ['CN-62', 'CN-GS'],
                ['CN-63', 'CN-QH'],
                ['CN-64', 'CN-NX'],
                ['CN-65', 'CN-XJ'],
                ['CN-71', 'CN-TW'],
                ['CN-91', 'CN-HK'],
                ['CN-92', 'CN-MO'],
            ]
    extraPagesNames = [
        u'Spratly-Inseln',
            ] 
class DE_BE(iso3166fixer):
    countryCode = 'DE' # Deutschland
    category = 'Berlin'

    replacements = [
                ['DE-B', 'DE-BE'],
            ]
class DE_NI(iso3166fixer):
    countryCode = 'DE' # Deutschland
    category = 'Niedersachsen'

    replacements = [
                ['DE-NS', 'DE-NI'],
            ]
class DE_NW(iso3166fixer):
    countryCode = 'DE' # Deutschland
    category = 'Nordrhein-Westfalen'

    replacements = [
                ['DE-NRW', 'DE-NW'],
            ]
class DE_SL(iso3166fixer):
    countryCode = 'DE' # Deutschland
    category = 'Saarland'

    replacements = [
                ['DE-SA', 'DE-SL'],
            ]
class DE_SN(iso3166fixer):
    countryCode = 'DE' # Deutschland
    category = 'Sachsen'

    replacements = [
                ['DE-SA', 'DE-SN'],
            ]
class DE_ST(iso3166fixer):
    countryCode = 'DE' # Deutschland
    category = 'Sachsen-Anhalt'

    replacements = [
                ['DE-SA', 'DE-ST'],
            ]
class DK_81(iso3166fixer):
    countryCode = 'DK' # Dänemark
    category = 'Region Nordjylland'

    replacements = [
                ['DK-070', 'DK-81'], # tw.
                ['DK-076', 'DK-81'], # tw.
                ['DK-080', 'DK-81'],
            ]
class DK_82(iso3166fixer):
    countryCode = 'DK' # Dänemark
    category = 'Region Midtjylland'

    replacements = [
                ['DK-065', 'DK-82'],
                ['DK-070', 'DK-82'], # tw.
                ['DK-076', 'DK-82'], # tw.
            ]
class DK_83(iso3166fixer):
    countryCode = 'DK' # Dänemark
    category = 'Region Syddanmark'

    replacements = [
                ['DK-042', 'DK-83'],
                ['DK-050', 'DK-83'],
                ['DK-055', 'DK-83'],
                ['DK-060', 'DK-83'],
            ]
class DK_84(iso3166fixer):
    countryCode = 'DK' # Dänemark
    category = 'Region Hovedstaden'

    replacements = [
                ['DK-015', 'DK-84'],
                ['DK-020', 'DK-84'],
                ['DK-040', 'DK-84'],
                ['DK-101', 'DK-84'],
                ['DK-147', 'DK-84'],
            ]
class DK_85(iso3166fixer):
    countryCode = 'DK' # Dänemark
    category = 'Region Sjælland'

    replacements = [
                ['DK-025', 'DK-85'],
                ['DK-030', 'DK-85'],
                ['DK-035', 'DK-85'],
            ]
class TW(iso3166changer):
    countryCode = 'TW' # Taiwan
    category = 'Taiwan'
    isoRefDate = '2016-11-15'

    replacements = [
                ['TW-TPQ', 'TW-TPE'],
                ['TW-KHQ', 'TW-KHH'],
            ]
class isoCodeChanger:
    def __init__(self, changer, test=False, stopAfter=0):
        self.changer = changer # type iso3166changer
        self.count   = 0
        self.successCount   = 0
        self.errCount  = 0
        self.test = test
        self.stopAfter = stopAfter
        self.editComment = self.changer.getEditcomment()
        self.extraPagesGenerator = self.changer.extraPagesGenerator()
    
    def execute(self):
        cat = pywikibot.Category(site,'Category:'+self.changer.category)
        searchFilter = self.changer.searchFilterGenerator()
        pages = pagegenerators.CombinedPageGenerator ([
                        self.changer.extraPagesGenerator(),
                        globalPagesGenerator,
                        pagegenerators.DuplicateFilterPageGenerator (pagegenerators.CategorizedPageGenerator(cat,7)),
                       ])
        if searchFilter != None:
            pages = pagegenerators.intersect_generators ([searchFilter, pages])

        for page in pages:
            if page.isRedirectPage():
                print (page, 'is redirect')
            oldpage = page.get(get_redirect=True) # changing the redirect instead of the page it redirects to
            newpage= self.changer.execute(oldpage)
            if (oldpage != newpage):
                print (page, 'changed(%d)'% int(self.successCount+1))
                pywikibot.showDiff(oldpage, newpage)
                self.count = self.count + 1
                if (not self.test):
                    try: # might fail if no write access (as bot is not admin)
                        page.put(newpage, self.editComment, minorEdit=True, botflag=True)
                        self.successCount = self.successCount + 1
                    except:                        
                        self.errCount = self.errCount + 1
                    pass
                
                if (self.count == self.stopAfter):
                    break
        if self.errCount > 0:
            print ('finished:', self.successCount, 'pages changed,', self.errCount, 'pages unchanged due to error.')
        else:
            print ('finished:', self.successCount, 'pages changed,')
### for tests only !!!
cc = isoCodeChanger (TestChanger(), test=True, stopAfter=0)
print (cc.editComment)
#print (cc.extraPagesGenerator)
cc.execute()
ISO 3166 Bot: ISO-Code Änderungen gemäß [[ISO 3166-2:DE]], falschen Code korrigiert
search filter insource:"DE-B" insource:/DE-B/ added
search filter insource:"DE-(BB|BE)" insource:/DE-(BB|BE)/ added
[[de:Flughafen Berlin Brandenburg]] changed(1)
[[de:Flughafen Berlin-Schönefeld]] changed(1)
@@ -17 +17 @@ ***
- |Koordinate_Region=DE-BB ***
+ |Koordinate_Region=DE-XXXB ***

@@ -16 +16 @@ ***
- |Koordinate_Region=DE-BB ***
+ |Koordinate_Region=DE-XXXB ***

[[de:Flughafen Berlin-Tegel]] changed(1)
@@ -16 +16 @@ ***
- |Koordinate_Region=DE-BE ***
+ |Koordinate_Region=DE-XXXE ***

[[de:Radfernweg Berlin–Usedom]] changed(1)
@@ -222 +222 @@ ***
- {{Coordinate|NS= 52.5183176|EW= 13.4010108|type=landmark|region=DE-B/DE-BB/DE-MV}} ***
+ {{Coordinate|NS= 52.5183176|EW= 13.4010108|type=landmark|region=DE-B/DE-XXXB/DE-MV}} ***

[[de:Elsenbrücke]] changed(1)
[[de:Freybrücke]] changed(1)
@@ -34 +34 @@ ***
- |REGION-ISO=DE-BE ***
+ |REGION-ISO=DE-XXXE ***

@@ -34 +34 @@ ***
- |REGION-ISO=DE-BE ***
+ |REGION-ISO=DE-XXXE ***

@@ -30 +30 @@ ***
- | REGION-ISO=DE-BE ***
+ | REGION-ISO=DE-XXXE ***

@@ -34 +34 @@ ***
- |REGION-ISO=DE-BE/DE-BB ***
+ |REGION-ISO=DE-BE/DE-XXXB ***

[[de:Gertraudenbrücke]] changed(1)
[[de:Glienicker Brücke]] changed(1)
[[de:Lichtenberger Brücke]] changed(1)
[[de:Liebknechtbrücke]] changed(1)
@@ -34 +34 @@ ***
- |REGION-ISO = DE-BE ***
+ |REGION-ISO = DE-XXXE ***

@@ -31 +31 @@ ***
- |REGION-ISO=DE-BE ***
+ |REGION-ISO=DE-XXXE ***

@@ -34 +34 @@ ***
- |REGION-ISO=DE-BE ***
+ |REGION-ISO=DE-XXXE ***

[[de:Mühlendamm (Berlin)]] changed(1)
[[de:Schloßbrücke (Berlin-Mitte)]] changed(1)
[[de:Stößenseebrücke]] changed(1)
@@ -34 +34 @@ ***
- |REGION-ISO=DE-BE ***
+ |REGION-ISO=DE-XXXE ***

@@ -34 +34 @@ ***
- |REGION-ISO=DE-BE ***
+ |REGION-ISO=DE-XXXE ***

[[de:Tangentiale Verbindung Ost]] changed(1)
[[de:Bundesautobahn 100]] changed(1)
@@ -132 +132 @@ ***
- {{Coordinate |dim=6000 |NS=52.492424 |EW=13.548413 |type=landmark |name= Tangentiale Verbindung Ost |region=DE-BE}} ***
+ {{Coordinate |dim=6000 |NS=52.492424 |EW=13.548413 |type=landmark |name= Tangentiale Verbindung Ost |region=DE-XXXE}} ***

@@ -16 +16 @@ ***
- |BEGINN-REGION      = DE-BE ***
+ |BEGINN-REGION      = DE-XXXE ***

@@ -20 +20 @@ ***
- |ENDE-REGION        = DE-BE ***
+ |ENDE-REGION        = DE-XXXE ***

[[de:Bundesautobahn 103]] changed(1)
[[de:Bundesautobahn 111]] changed(1)
@@ -22 +22 @@ ***
- |BEGINN-REGION      = DE-BE ***
+ |BEGINN-REGION      = DE-XXXE ***

@@ -26 +26 @@ ***
- |ENDE-REGION        = DE-BE ***
+ |ENDE-REGION        = DE-XXXE ***

@@ -22 +22 @@ ***
- |BEGINN-REGION       = DE-BB ***
+ |BEGINN-REGION       = DE-XXXB ***

@@ -26 +26 @@ ***
- |ENDE-REGION         = DE-BE ***
+ |ENDE-REGION         = DE-XXXE ***

[[de:Bundesautobahn 113]] changed(1)
[[de:Bundesautobahn 114]] changed(1)
@@ -18 +18 @@ ***
- |BEGINN-REGION      = DE-BE ***
+ |BEGINN-REGION      = DE-XXXE ***

@@ -22 +22 @@ ***
- |ENDE-REGION        = DE-BB ***
+ |ENDE-REGION        = DE-XXXB ***

@@ -20 +20 @@ ***
- |BEGINN-REGION       = DE-BB ***
+ |BEGINN-REGION       = DE-XXXB ***

@@ -25 +25 @@ ***
- |ENDE-REGION         = DE-BE ***
+ |ENDE-REGION         = DE-XXXE ***

[[de:Bundesautobahn 115]] changed(1)
[[de:Bundesautobahn 117]] changed(1)
[[de:Bundesstraße 1]] changed(1)
@@ -23 +23 @@ ***
- |BEGINN-REGION       = DE-BE ***
+ |BEGINN-REGION       = DE-XXXE ***

@@ -27 +27 @@ ***
- |ENDE-REGION         = DE-BB ***
+ |ENDE-REGION         = DE-XXXB ***

@@ -11 +11 @@ ***
- |BEGINN-REGION       = DE-BE ***
+ |BEGINN-REGION       = DE-XXXE ***

@@ -15 +15 @@ ***
- |ENDE-REGION         = DE-BB ***
+ |ENDE-REGION         = DE-XXXB ***

@@ -23 +23 @@ ***
- |ENDE-REGION = DE-BB ***
+ |ENDE-REGION = DE-XXXB ***

[[de:Bundesstraße 2]] changed(1)
[[de:Bundesstraße 5]] changed(1)
[[de:Bundesstraße 96a]] changed(1)
@@ -10 +10 @@ ***
- |BEGINN-REGION       = DE-BB ***
+ |BEGINN-REGION       = DE-XXXB ***

@@ -14 +14 @@ ***
- |ENDE-REGION         = DE-BY ***
+ |ENDE-REGION         = DE-XXXY ***

@@ -16 +16 @@ ***
- |ENDE-REGION         = DE-BB ***
+ |ENDE-REGION         = DE-XXXB ***

@@ -10 +10 @@ ***
- |BEGINN-REGION       = DE-BB ***
+ |BEGINN-REGION       = DE-XXXB ***

@@ -14 +14 @@ ***
- |ENDE-REGION         = DE-BB ***
+ |ENDE-REGION         = DE-XXXB ***

[[de:Bundesstraße 101]] changed(1)
[[de:Bundesstraße 109]] changed(1)
[[de:Bundesstraße 158]] changed(1)
@@ -9 +9 @@ ***
- |BEGINN-REGION       = DE-BE ***
+ |BEGINN-REGION       = DE-XXXE ***

@@ -11 +11 @@ ***
- |BEGINN-REGION       = DE-BE ***
+ |BEGINN-REGION       = DE-XXXE ***

@@ -12 +12 @@ ***
- |BEGINN-REGION       = DE-BE ***
+ |BEGINN-REGION       = DE-XXXE ***

@@ -16 +16 @@ ***
- |ENDE-REGION         = DE-BB ***
+ |ENDE-REGION         = DE-XXXB ***

@@ -74 +74 @@ ***
- |BEGINN-REGION       = DE-BB ***
+ |BEGINN-REGION       = DE-XXXB ***

@@ -78 +78 @@ ***
- |ENDE-REGION         = DE-BB ***
+ |ENDE-REGION         = DE-XXXB ***

finished: 0 pages changed, 0 pages unchanged due to error.
### irregular updates from outside
cc = isoCodeChanger (CN(), test=False, stopAfter=0)
print (cc.editComment)
#print (cc.extraPagesGenerator)
cc.execute()
ISO 3166 Bot: ISO-Code Änderungen gemäß [[ISO 3166-2:CN]], falschen Code korrigiert
search filter insource:"CN-11" insource:/CN-11/ added
search filter insource:"CN-12" insource:/CN-12/ added
search filter insource:"CN-13" insource:/CN-13/ added
search filter insource:"CN-14" insource:/CN-14/ added
search filter insource:"CN-15" insource:/CN-15/ added
search filter insource:"CN-21" insource:/CN-21/ added
search filter insource:"CN-22" insource:/CN-22/ added
search filter insource:"CN-23" insource:/CN-23/ added
search filter insource:"CN-31" insource:/CN-31/ added
search filter insource:"CN-32" insource:/CN-32/ added
search filter insource:"CN-33" insource:/CN-33/ added
search filter insource:"CN-34" insource:/CN-34/ added
search filter insource:"CN-35" insource:/CN-35/ added
search filter insource:"CN-36" insource:/CN-36/ added
search filter insource:"CN-37" insource:/CN-37/ added
search filter insource:"CN-41" insource:/CN-41/ added
search filter insource:"CN-42" insource:/CN-42/ added
search filter insource:"CN-43" insource:/CN-43/ added
search filter insource:"CN-44" insource:/CN-44/ added
search filter insource:"CN-45" insource:/CN-45/ added
search filter insource:"CN-46" insource:/CN-46/ added
search filter insource:"CN-50" insource:/CN-50/ added
search filter insource:"CN-51" insource:/CN-51/ added
search filter insource:"CN-52" insource:/CN-52/ added
search filter insource:"CN-53" insource:/CN-53/ added
search filter insource:"CN-54" insource:/CN-54/ added
search filter insource:"CN-61" insource:/CN-61/ added
search filter insource:"CN-62" insource:/CN-62/ added
search filter insource:"CN-63" insource:/CN-63/ added
search filter insource:"CN-64" insource:/CN-64/ added
search filter insource:"CN-65" insource:/CN-65/ added
search filter insource:"CN-71" insource:/CN-71/ added
search filter insource:"CN-91" insource:/CN-91/ added
search filter insource:"CN-92" insource:/CN-92/ added
[[de:Liste der größten Bogenbrücken]] changed(1)
@@ -260 +260 @@ ***
- | [[Zhuhai]], [[Guangdong]]<br /><small>{{Coordinate|simple=y|NS=22.155917|EW=113.458806|type=landmark|region=CN-44|name=Zweite Hengqin-Brücke|text=DMS}}</small> ***
+ | [[Zhuhai]], [[Guangdong]]<br /><small>{{Coordinate|simple=y|NS=22.155917|EW=113.458806|type=landmark|region=CN-GD|name=Zweite Hengqin-Brücke|text=DMS}}</small> ***

WARNING: API error badtoken: Invalid CSRF token.
Sleeping for 9.7 seconds, 2018-02-14 00:38:27
Page [[de:Liste der größten Bogenbrücken]] saved
finished: 1 pages changed, 0 pages unchanged due to error.
### regular error fixes
#for c in [DE_BE(),DE_NI(),DE_SL(),DE_SN(),DE_ST(),DE_NW(),DK_81(),DK_82(),DK_83(),DK_84(),DK_85()]:
#for c in [DE_BE()]:
for c in [DE_SN()]:
    cc = isoCodeChanger (c, test=False, stopAfter=0)
    print (cc.editComment)
    #print (cc.extraPagesGenerator)
    cc.execute()
ISO 3166 Bot: ISO-Code Änderungen gemäß [[ISO 3166-2:DE]], falschen Code korrigiert
search filter insource:"DE-SA" insource:/DE-SA/ added
[[de:Hotel Bayerischer Hof Dresden]] changed(1)
@@ -25 +25 @@ ***
- {{Coordinate |NS=51.064102 |EW=13.739019 |type=landmark |region=DE-SA}} ***
+ {{Coordinate |NS=51.064102 |EW=13.739019 |type=landmark |region=DE-SN}} ***

WARNING: API error badtoken: Invalid CSRF token.
Sleeping for 9.9 seconds, 2018-02-16 14:17:33
Page [[de:Hotel Bayerischer Hof Dresden]] saved
finished: 1 pages changed, 0 pages unchanged due to error.
# test
cat = pywikibot.Category(site,'Category:'+'Salzburg')
pages = pagegenerators.intersect_generators ([
                pagegenerators.SearchPageGenerator('insource:"AT-5" insource:/AT-5/', total=None, namespaces=[0], site=site),
                pagegenerators.DuplicateFilterPageGenerator (pagegenerators.CategorizedPageGenerator(cat,7)),
               ])

for page in pages:
    print (page)
print ('finished')