python3.5
  File "<ipython-input-1-142e0592aba2>", line 1
    python3.5
            ^
SyntaxError: invalid syntax
python3
---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-2-ea017c41fe00> in <module>
----> 1 python3

NameError: name 'python3' is not defined
python
---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-3-32886514c262> in <module>
----> 1 python

NameError: name 'python' is not defined
print("hello")
hello
ls
Untitled1.ipynb  Untitled.ipynb
pwd
'/home/paws'
mkdir skynet
ls
  File "<ipython-input-8-096f78a022b3>", line 1
    mkdir skynet
               ^
SyntaxError: invalid syntax
mkdir skynet
ls
skynet/  Untitled1.ipynb  Untitled.ipynb
rm skynet/
rm: cannot remove 'skynet/': Is a directory
rm -f skynet/
rm: cannot remove 'skynet/': Is a directory
rm -r skynet/
import mwapi
session = mwapi.Session('https://commons.wikimedia.org') #TODO add user agent
Sending requests with default User-Agent.  Set 'user_agent' on mwapi.Session to quiet this message.
print(session.get(action='query', meta='userinfo'))
{'query': {'userinfo': {'anon': '', 'name': 'Jtmorgan', 'id': 0}},
 'batchcomplete': ''}
{'batchcomplete': '', 'query': {'userinfo': {'id': 0, 'name': '172.16.1.161', 'anon': ''}}}
{'query': {'userinfo': {'anon': '', 'name': 'Jtmorgan', 'id': 0}},
 'batchcomplete': ''}
print(session.get(action='query', meta='userinfo'))
{'query': {'userinfo': {'anon': '', 'name': 'Eatcha', 'id': 0}},
 'batchcomplete': ''}
{'batchcomplete': '', 'query': {'userinfo': {'id': 0, 'name': '172.16.1.161', 'anon': ''}}}
{'query': {'userinfo': {'anon': '', 'name': 'Eatcha', 'id': 0}},
 'batchcomplete': ''}
pip install git+https://github.com/mediawiki-utilities/python-mwviews.git
    
  File "<ipython-input-5-a9612037908a>", line 1
    pip install git+https://github.com/mediawiki-utilities/python-mwviews.git
              ^
SyntaxError: invalid syntax
pip install git https://github.com/mediawiki-utilities/python-mwviews.git
Collecting https://github.com/mediawiki-utilities/python-mwviews.git
  Downloading https://github.com/mediawiki-utilities/python-mwviews.git
     \ 102kB 6.0MB/s
  Cannot unpack file /tmp/pip-92lbtzkw-unpack/python-mwviews.git (downloaded from /tmp/pip-spy9hwcm-build, content-type: text/html; charset=utf-8); cannot detect archive format
Cannot determine archive format of /tmp/pip-spy9hwcm-build
Note: you may need to restart the kernel to use updated packages.
pip install mwviews
Collecting mwviews
  Downloading https://files.pythonhosted.org/packages/39/f9/97e8491c92a0a2774d19dd190e1104f12822376c9280973c6aa0c8719d14/mwviews-0.1.0.tar.gz
Requirement already satisfied: mwapi in /srv/paws/lib/python3.6/site-packages (from mwviews)
Requirement already satisfied: mwcli in /srv/paws/lib/python3.6/site-packages (from mwviews)
Requirement already satisfied: requests in /srv/paws/lib/python3.6/site-packages (from mwviews)
Requirement already satisfied: docopt in /srv/paws/lib/python3.6/site-packages (from mwcli->mwviews)
Requirement already satisfied: para in /srv/paws/lib/python3.6/site-packages (from mwcli->mwviews)
Requirement already satisfied: idna<2.9,>=2.5 in /srv/paws/lib/python3.6/site-packages (from requests->mwviews)
Requirement already satisfied: certifi>=2017.4.17 in /srv/paws/lib/python3.6/site-packages (from requests->mwviews)
Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /srv/paws/lib/python3.6/site-packages (from requests->mwviews)
Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /srv/paws/lib/python3.6/site-packages (from requests->mwviews)
Building wheels for collected packages: mwviews
  Running setup.py bdist_wheel for mwviews ... error
  Complete output from command /srv/paws/bin/python3.6 -u -c "import setuptools, tokenize;__file__='/tmp/pip-build-b7qrntoh/mwviews/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\r\n', '\n');f.close();exec(compile(code, __file__, 'exec'))" bdist_wheel -d /tmp/tmpbf01bpi2pip-wheel- --python-tag cp36:
  usage: -c [global_opts] cmd1 [cmd1_opts] [cmd2 [cmd2_opts] ...]
     or: -c --help [cmd1 cmd2 ...]
     or: -c --help-commands
     or: -c cmd --help
  
  error: invalid command 'bdist_wheel'
  
  ----------------------------------------
  Failed building wheel for mwviews
  Running setup.py clean for mwviews
Failed to build mwviews
Installing collected packages: mwviews
  Running setup.py install for mwviews ... done
Successfully installed mwviews-0.1.0
Note: you may need to restart the kernel to use updated packages.
from mwviews.api import PageviewsClient
p = PageviewsClient()

p.article_views('en.wikipedia', ['Selfie', 'Cat', 'Dog'])
p.project_views(['ro.wikipedia', 'de.wikipedia', 'commons.wikimedia'])
p.top_articles('en.wikipedia', limit=10)
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-9-2e02614b46d0> in <module>
----> 1 p = PageviewsClient()
      2 
      3 p.article_views('en.wikipedia', ['Selfie', 'Cat', 'Dog'])
      4 p.project_views(['ro.wikipedia', 'de.wikipedia', 'commons.wikimedia'])
      5 p.top_articles('en.wikipedia', limit=10)

TypeError: __init__() missing 1 required positional argument: 'user_agent'
session = mwapi.Session('https://commons.wikimedia.org').user_agent('google-bot')
Sending requests with default User-Agent.  Set 'user_agent' on mwapi.Session to quiet this message.
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-10-817b632bc089> in <module>
----> 1 session = mwapi.Session('https://commons.wikimedia.org').user_agent('google-bot')

AttributeError: 'Session' object has no attribute 'user_agent'
ls /bin/
bash*          date*           lessecho*       pwd*         uname*
bunzip2*       dd*             lessfile@       rbash@       uncompress*
bzcat*         df*             lesskey*        readlink*    vdir*
bzcmp@         dir*            lesspipe*       rm*          wdctl*
bzdiff*        dmesg*          ln*             rmdir*       which*
bzegrep@       dnsdomainname@  login*          rnano@       ypdomainname@
bzexe*         domainname@     ls*             run-parts*   zcat*
bzfgrep@       echo*           lsblk*          sed*         zcmp*
bzgrep*        egrep*          mkdir*          sh@          zdiff*
bzip2*         false*          mknod*          sh.distrib@  zegrep*
bzip2recover*  fgrep*          mktemp*         sleep*       zfgrep*
bzless@        findmnt*        more*           stty*        zforce*
bzmore*        grep*           mount*          su*          zgrep*
cat*           gunzip*         mountpoint*     sync*        zless*
chgrp*         gzexe*          mv*             tar*         zmore*
chmod*         gzip*           nano*           tempfile*    znew*
chown*         hostname*       nisdomainname@  touch*
cp*            kill*           pidof@          true*
dash*          less*           ps*             umount*
pwb.py login
  File "<ipython-input-12-167fa9e6a77a>", line 1
    pwb.py login
               ^
SyntaxError: invalid syntax
python3 pwb.py login
  File "<ipython-input-13-a0c48554221a>", line 1
    python3 pwb.py login
              ^
SyntaxError: invalid syntax
$ pwb.py login
  File "<ipython-input-14-6b204071d938>", line 1
    $ pwb.py login
    ^
SyntaxError: invalid syntax
pip install pywikibot
Requirement already satisfied: pywikibot in /srv/paws/pwb
Requirement already satisfied: requests>=2.20.0 in /srv/paws/lib/python3.6/site-packages (from pywikibot)
Requirement already satisfied: idna<2.9,>=2.5 in /srv/paws/lib/python3.6/site-packages (from requests>=2.20.0->pywikibot)
Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /srv/paws/lib/python3.6/site-packages (from requests>=2.20.0->pywikibot)
Requirement already satisfied: certifi>=2017.4.17 in /srv/paws/lib/python3.6/site-packages (from requests>=2.20.0->pywikibot)
Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /srv/paws/lib/python3.6/site-packages (from requests>=2.20.0->pywikibot)
Note: you may need to restart the kernel to use updated packages.
pywikibot.py login
  File "<ipython-input-1-76e5e04b36d8>", line 1
    pywikibot.py login
                     ^
SyntaxError: invalid syntax
login
---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-2-689a9b4a0dc0> in <module>
----> 1 login

NameError: name 'login' is not defined
ls
Untitled1.ipynb  Untitled.ipynb
cd /srv/paws/lib/python3.6/site-packages
/srv/paws/lib/python3.6/site-packages
ls
alembic/
alembic-1.0.10.egg-info/
asn1crypto/
asn1crypto-0.24.0.dist-info/
async_generator/
async_generator-1.10.dist-info/
atomicwrites/
atomicwrites-1.3.0.dist-info/
attr/
attrs-19.1.0.dist-info/
backcall/
backcall-0.1.0.egg-info/
bash_kernel/
bash_kernel-0.7.1.dist-info/
beautifulsoup4-4.7.1.dist-info/
bin/
bleach/
bleach-3.1.0.dist-info/
blinker/
blinker-1.4.egg-info/
bokeh/
bokeh-1.1.0.egg-info/
bqplot/
bqplot-0.11.5.dist-info/
branca/
branca-0.3.1.dist-info/
brotli/
brotlipy-0.7.0.dist-info/
bs4/
certifi/
certifi-2019.3.9.dist-info/
cffi/
cffi-1.12.3.dist-info/
_cffi_backend.cpython-36m-x86_64-linux-gnu.so*
chardet/
chardet-3.0.4.dist-info/
click/
Click-7.0.dist-info/
cli_helpers/
cli_helpers-1.2.0.dist-info/
codecov/
codecov-2.0.15.dist-info/
configobj-5.0.6.egg-info/
configobj.py
coverage/
coverage-4.5.3.dist-info/
crontab/
crontab-0.22.5.egg-info/
cryptography/
cryptography-2.6.1.dist-info/
cycler-0.10.0.dist-info/
cycler.py
dateutil/
decorator-4.4.0.dist-info/
decorator.py
defusedxml/
defusedxml-0.6.0.dist-info/
deltas/
deltas-0.4.7.dist-info/
docopt-0.6.2.egg-info/
docopt.py
dot_parser.py
easy-install.pth
easy_install.py
editor.py*
entrypoints-0.3.dist-info/
entrypoints.py
flake8/
flake8-3.7.7.dist-info/
flake8_docstrings-1.3.0.dist-info/
flake8_docstrings.py
flake8_mock-0.3.egg-info/
flake8_mock.py
flake8_polyfill/
flake8_polyfill-1.0.2.dist-info/
flask/
Flask-1.0.3.dist-info/
flickrapi/
flickrapi-2.4.0.dist-info/
google-2.0.2.egg-info/
googlesearch/
heat.py
httpbin/
httpbin-0.7.0.dist-info/
idna/
idna-2.8.dist-info/
importlib_metadata/
importlib_metadata-0.13.dist-info/
inflect-2.1.0.dist-info/
inflect.py
ipydatawidgets/
ipydatawidgets-4.0.0.dist-info/
ipykernel/
ipykernel-5.1.1.dist-info/
ipykernel_launcher.py
ipyleaflet/
ipyleaflet-0.10.2.dist-info/
IPython/
ipython-7.5.0.dist-info/
ipython_genutils/
ipython_genutils-0.2.0.dist-info/
ipywidgets/
ipywidgets-7.4.2.dist-info/
irc/
irc-17.1.dist-info/
isympy.py
itsdangerous/
itsdangerous-1.1.0.dist-info/
jaraco/
jaraco.classes-2.0.dist-info/
jaraco.collections-2.0.dist-info/
jaraco.functools-2.0.dist-info/
jaraco.itertools-4.4.2.dist-info/
jaraco.logging-2.0.dist-info/
jaraco.stream-2.0.dist-info/
jaraco.text-3.0.dist-info/
jedi/
jedi-0.13.3.dist-info/
jinja2/
Jinja2-2.10.1.dist-info/
joblib/
joblib-0.13.2.dist-info/
jsonable/
jsonable-0.3.1.dist-info/
jsonschema/
jsonschema-3.0.1.dist-info/
jupyter-1.0.0.dist-info/
jupyter_client/
jupyter_client-5.2.4.dist-info/
jupyter_console/
jupyter_console-6.0.0.dist-info/
jupyter_core/
jupyter_core-4.4.0.dist-info/
jupyterhub/
jupyterhub-0.9.0.dist-info/
jupyterlab/
jupyterlab-0.35.6.dist-info/
jupyterlab_server/
jupyterlab_server-0.2.0.dist-info/
jupyter.py
jwt/
kiwisolver-1.1.0.dist-info/
kiwisolver.cpython-36m-x86_64-linux-gnu.so*
lasagne/
Lasagne-0.1.egg-info/
lxml/
lxml-4.3.3.dist-info/
mako/
Mako-1.0.10.egg-info/
markupsafe/
MarkupSafe-1.1.1.dist-info/
matplotlib/
matplotlib-3.1.0.dist-info/
matplotlib-3.1.0-py3.6-nspkg.pth
mccabe-0.6.1.dist-info/
mccabe.py
memento_client/
memento_client-0.6.1.dist-info/
mistune-0.8.4.dist-info/
mistune.py
more_itertools/
more_itertools-7.0.0.dist-info/
mpl_toolkits/
mpmath/
mpmath-1.1.0.egg-info/
mwapi/
mwapi-0.5.1.dist-info/
mwcli/
mwcli-0.0.2.dist-info/
mwdb/
mwdb-0.1.2.dist-info/
mwdiffs/
mwdiffs-0.0.2.egg-info/
mwoauth/
mwoauth-0.3.3.dist-info/
mwparserfromhell/
mwparserfromhell-0.5.4.egg-info/
mwpersistence/
mwpersistence-0.2.4.dist-info/
mwreverts/
mwreverts-0.1.5.dist-info/
mwsessions/
mwsessions-0.0.2.egg-info/
mwtypes/
mwtypes-0.3.0.dist-info/
mwviews/
mwviews-0.1.0.egg-info/
mwxml/
mwxml-0.3.3.dist-info/
mycli/
mycli-1.19.0.dist-info/
nbconvert/
nbconvert-5.5.0.dist-info/
nbformat/
nbformat-4.4.0.dist-info/
nbpawspublic/
nbpawspublic-0.1.0.egg-info/
nbresuse/
nbresuse-0.3.1.egg-info/
nose/
nose-1.3.7.dist-info/
nosetrim/
nosetrim-0.1.dev0.egg-info/
notebook/
notebook-5.7.8.dist-info/
numpy/
numpy-1.16.3.dist-info/
oauth2/
oauthlib/
oauthlib-3.0.1.dist-info/
packaging/
packaging-19.0.dist-info/
pamela-1.0.0.dist-info/
pamela.py
pandas/
pandas-0.24.2.dist-info/
pandocfilters-1.4.2.egg-info/
pandocfilters.py
para/
para-0.0.6.dist-info/
parso/
parso-0.4.0.dist-info/
paws/
paws-0.1.egg-info/
pep257-0.7.0.dist-info/
pep257.py
pexpect/
pexpect-4.7.0.dist-info/
pickleshare-0.7.5.dist-info/
pickleshare.py
PIL/
Pillow-6.0.0.dist-info/
pip/
pip-9.0.1.dist-info/
pkg_resources/
pkg_resources-0.0.0.dist-info/
pluggy/
pluggy-0.11.0.dist-info/
pprofile-2.0.2.egg-info/
pprofile.py
prometheus_client/
prometheus_client-0.6.0.egg-info/
prompt_toolkit/
prompt_toolkit-2.0.9.dist-info/
psutil/
psutil-5.6.2.egg-info/
ptr.py
ptyprocess/
ptyprocess-0.6.0.dist-info/
pvectorc.cpython-36m-x86_64-linux-gnu.so*
py/
py-1.8.0.dist-info/
__pycache__/
pycodestyle-2.5.0.dist-info/
pycodestyle.py
pycountry/
pycountry-18.12.8.dist-info/
pycparser/
pycparser-2.19.egg-info/
pydocstyle/
pydocstyle-3.0.0.dist-info/
pydot-1.4.1.dist-info/
pydot.py
pyflakes/
pyflakes-2.1.1.dist-info/
pygments/
Pygments-2.4.0.dist-info/
pyheat/
py_heat-0.0.6.egg-info/
py_heat_magic-0.0.2.egg-info/
PyJWT-1.7.1.dist-info/
pylab.py
pymysql/
PyMySQL-0.9.3.dist-info/
pyparsing-2.4.0.dist-info/
pyparsing.py
pyrsistent/
pyrsistent-0.15.2.egg-info/
_pyrsistent_version.py
_pytest/
pytest-4.5.0.dist-info/
pytest_attrib/
pytest_attrib-0.1.3.egg-info/
pytest_cov/
pytest_cov-2.7.1.dist-info/
pytest-cov.pth
pytest_httpbin/
pytest_httpbin-1.0.0.dist-info/
pytest.py
pytest_runner-4.4.dist-info/
pytest_timeout-1.3.3.dist-info/
pytest_timeout.py
python_dateutil-2.8.0.dist-info/
python_editor-1.0.4.dist-info/
python_oauth2-1.1.0.egg-info/
python_stdnum-1.11.dist-info/
python_wdqs-0.1.egg-info/
pythreejs/
pythreejs-2.0.2.dist-info/
pytz/
pytz-2019.1.dist-info/
pywikibot.egg-link
PyYAML-5.1.egg-info/
pyzmq-18.0.1.dist-info/
qtconsole/
qtconsole-4.4.4.dist-info/
raven/
raven-6.10.0.dist-info/
requests/
requests-2.22.0.dist-info/
requests_oauthlib/
requests_oauthlib-1.2.0.dist-info/
requests_toolbelt/
requests_toolbelt-0.9.1.dist-info/
_rinterface_cffi.py
rise/
rise-5.5.0.dist-info/
rpy2/
rpy2-3.0.4.egg-info/
scikit_learn-0.21.1.dist-info/
scikit_neuralnetwork-0.7.egg-info/
scipy/
scipy-1.3.0.dist-info/
seaborn/
seaborn-0.9.0.dist-info/
send2trash/
Send2Trash-1.5.0.dist-info/
setuptools/
setuptools-39.0.1.dist-info/
simplegeneric-0.8.1.egg-info/
simplegeneric.py
six-1.12.0.dist-info/
six.py
sklearn/
sknn/
snowballstemmer/
snowballstemmer-1.2.1.dist-info/
soupsieve/
soupsieve-1.9.1.dist-info/
sqlalchemy/
SQLAlchemy-1.3.3.egg-info/
sqlparse/
sqlparse-0.2.4.dist-info/
sseclient-0.0.24.egg-info/
sseclient.py
stdnum/
stopit/
stopit-1.1.2.egg-info/
sympy/
sympy-1.4.dist-info/
tabulate-0.8.3.egg-info/
tabulate.py
tempora/
tempora-1.14.1.dist-info/
terminado/
terminado-0.8.2.dist-info/
terminaltables/
terminaltables-3.1.0.egg-info/
testpath/
testpath-0.4.2.dist-info/
theano/
Theano-1.0.4.egg-info/
tornado/
tornado-6.0.2.egg-info/
traitlets/
traitlets-4.3.2.dist-info/
traittypes/
traittypes-0.2.1.dist-info/
urllib3/
urllib3-1.25.2.dist-info/
validate.py
_version.py
wcwidth/
wcwidth-0.1.7.dist-info/
wdqs/
webencodings/
webencodings-0.5.1.dist-info/
werkzeug/
Werkzeug-0.15.4.dist-info/
widgetsnbextension/
widgetsnbextension-3.4.2.dist-info/
xarray/
xarray-0.12.1.dist-info/
yaml/
yamlconf/
yamlconf-0.2.4.dist-info/
yamlconf.py
zipp-0.5.1.dist-info/
zipp.py
zmq/
zpprofile.py
cd pywikibot.egg-link
[Errno 20] Not a directory: 'pywikibot.egg-link'
/srv/paws/lib/python3.6/site-packages
pwb.py add_text -help
  File "<ipython-input-7-bacda7118b26>", line 1
    pwb.py add_text -help
                  ^
SyntaxError: invalid syntax
pwb.py
---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-9-d215821285a5> in <module>
----> 1 pwb.py

NameError: name 'pwb' is not defined
import pywikibot

site = pywikibot.Site('test', 'wikipedia')
page = pywikibot.Page(site, 'User:Eatcha')

page.text = 'Hello world!'
page.save()
Sleeping for 9.4 seconds, 2019-07-24 11:28:30
Page [[User:Eatcha]] saved
import pywikibot

site = pywikibot.Site('commons', 'commons')
page = pywikibot.Page(site, 'User:Eatcha/test')

page.text = 'Hello world! 6'
page.save()
Sleeping for 9.6 seconds, 2019-07-24 12:05:51
Page [[commons:User:Eatcha/test]] saved
import urllib2
from bs4 import BeautifulSoup
url = 'http://www.google.co.in/'

conn = urllib2.urlopen(url)
html = conn.read()

soup = BeautifulSoup(html)
links = soup.find_all('a')

for tag in links:
    link = tag.get('href',None)
    if link is not None:
        print (link)
---------------------------------------------------------------------------
ModuleNotFoundError                       Traceback (most recent call last)
<ipython-input-2-86aeb8efa446> in <module>
----> 1 import urllib2
      2 from bs4 import BeautifulSoup
      3 url = 'http://www.google.co.in/'
      4 
      5 conn = urllib2.urlopen(url)

ModuleNotFoundError: No module named 'urllib2'
pip install urllib2
Collecting urllib2
  Could not find a version that satisfies the requirement urllib2 (from versions: )
No matching distribution found for urllib2
Note: you may need to restart the kernel to use updated packages.
import urllib.request, urllib.error, urllib.parse
from bs4 import BeautifulSoup
url = 'http://www.google.co.in/'

conn = urllib.request.urlopen(url)
html = conn.read()

soup = BeautifulSoup(html)
links = soup.find_all('a')

for tag in links:
    link = tag.get('href',None)
    if link is not None:
        print (link)
http://www.google.com/imghp?hl=en&tab=wi
http://maps.google.com/maps?hl=en&tab=wl
https://play.google.com/?hl=en&tab=w8
http://www.youtube.com/?gl=US&tab=w1
http://news.google.com/nwshp?hl=en&tab=wn
https://mail.google.com/mail/?tab=wm
https://drive.google.com/?tab=wo
https://www.google.com/intl/en/about/products?tab=wh
http://www.google.com/history/optout?hl=en
/preferences?hl=en
https://accounts.google.com/ServiceLogin?hl=en&passive=true&continue=http://www.google.co.in/
/advanced_search?hl=en&authuser=0
/language_tools?hl=en&authuser=0
/intl/en/ads/
/services/
/intl/en/about.html
http://www.google.co.in/setprefdomain?prefdom=US&sig=K_MJYW9lyCeEz2lNf5OQMsGnsnfOU%3D
/intl/en/policies/privacy/
/intl/en/policies/terms/
import requests
from bs4 import BeautifulSoup

url = "http://www.python.org"
response = requests.get(url)
# parse html
page = str(BeautifulSoup(response.content))


def getURL(page):
    """

    :param page: html of web page (here: Python home page) 
    :return: urls in that page 
    """
    start_link = page.find("a href")
    if start_link == -1:
        return None, 0
    start_quote = page.find('"', start_link)
    end_quote = page.find('"', start_quote + 1)
    url = page[start_quote + 1: end_quote]
    return url, end_quote

def doWaybackup(self, metadata):
       
        urfields = [u'url', u'idrefurl', u'refurl', u'describedbyurl', u'imagesourceurl']
        doneurls = []
        for urlfield in urfields:
            url = metadata.get(urlfield)
            if url and url not in doneurls:
                print (u'Backing up this url to the Wayback Machine: %s' % (url,))
                waybackUrl = u'https://web.archive.org/save/%s' % (url,)
                waybackPage = requests.get(waybackUrl)
                doneurls.append(url)

while True:
    url, n = getURL(page)
    page = page[n:]
    if url:
        print(url)
    else:
        break
http://browsehappy.com/
#content
/psf-landing/
https://docs.python.org
https://pypi.python.org/
/jobs/
/community/
/
https://www.facebook.com/pythonlang?fref=ts
https://twitter.com/ThePSF
/community/irc/
/about/apps/
/about/quotes/
/about/gettingstarted/
/about/help/
http://brochure.getpython.info/
/downloads/
/downloads/source/
/downloads/windows/
/downloads/mac-osx/
/download/other/
https://docs.python.org/3/license.html
/download/alternatives
/doc/
/doc/av
https://wiki.python.org/moin/BeginnersGuide
https://devguide.python.org/
https://docs.python.org/faq/
http://wiki.python.org/moin/Languages
http://python.org/dev/peps/
https://wiki.python.org/moin/PythonBooks
/doc/essays/
/community/survey
/community/diversity/
/community/lists/
/community/irc/
/community/forums/
/psf/annual-report/2019/
/community/workshops/
/community/sigs/
/community/logos/
https://wiki.python.org/moin/
/community/merchandise/
/community/awards
https://www.python.org/psf/codeofconduct/
/success-stories/category/arts/
/success-stories/category/business/
/success-stories/category/education/
/success-stories/category/engineering/
/success-stories/category/government/
/success-stories/category/scientific/
/success-stories/category/software-development/
/blogs/
http://planetpython.org/
http://pyfound.blogspot.com/
http://pycon.blogspot.com/
/events/python-events
/events/python-user-group/
/events/python-events/past/
/events/python-user-group/past/
https://wiki.python.org/moin/PythonEventsCalendar#Submitting_an_Event
//docs.python.org/3/tutorial/controlflow.html#defining-functions
//docs.python.org/3/tutorial/introduction.html#lists
http://docs.python.org/3/tutorial/introduction.html#using-python-as-a-calculator
//docs.python.org/3/tutorial/
//docs.python.org/3/tutorial/controlflow.html
/about/gettingstarted/
/downloads/release/python-374/
https://docs.python.org
//jobs.python.org
https://blog.python.org
http://feedproxy.google.com/~r/PythonInsider/~3/MwuRB1u_KNQ/pypi-now-supports-uploading-via-api.html
http://feedproxy.google.com/~r/PythonSoftwareFoundationNews/~3/PyfIYGrs4Vo/pypi-now-supports-uploading-via-api.html
https://mailchi.mp/python/psf-2019-q2-newsletter
http://feedproxy.google.com/~r/PythonSoftwareFoundationNews/~3/9sGpXmeE1-c/2019-psf-fundraiser-thank-you-debrief.html
http://feedproxy.google.com/~r/PythonSoftwareFoundationNews/~3/hf9N-_dx7VQ/the-python-software-foundation-is.html
/events/calendars/
/events/python-events/776/
/events/python-events/792/
/events/python-events/847/
/events/python-events/854/
/events/python-events/846/
/success-stories/
/success-stories/python-seo-link-analyzer/
/success-stories/python-seo-link-analyzer/
/about/apps
/dev/peps/
/psf/
/about/
/about/apps/
/about/quotes/
/about/gettingstarted/
/about/help/
http://brochure.getpython.info/
/downloads/
/downloads/
/downloads/source/
/downloads/windows/
/downloads/mac-osx/
/download/other/
https://docs.python.org/3/license.html
/download/alternatives
/doc/
/doc/
/doc/av
https://wiki.python.org/moin/BeginnersGuide
https://devguide.python.org/
https://docs.python.org/faq/
http://wiki.python.org/moin/Languages
http://python.org/dev/peps/
https://wiki.python.org/moin/PythonBooks
/doc/essays/
/community/
/community/survey
/community/diversity/
/community/lists/
/community/irc/
/community/forums/
/psf/annual-report/2019/
/community/workshops/
/community/sigs/
/community/logos/
https://wiki.python.org/moin/
/community/merchandise/
/community/awards
https://www.python.org/psf/codeofconduct/
/success-stories/
/success-stories/category/arts/
/success-stories/category/business/
/success-stories/category/education/
/success-stories/category/engineering/
/success-stories/category/government/
/success-stories/category/scientific/
/success-stories/category/software-development/
/blogs/
/blogs/
http://planetpython.org/
http://pyfound.blogspot.com/
http://pycon.blogspot.com/
/events/
/events/python-events
/events/python-user-group/
/events/python-events/past/
/events/python-user-group/past/
https://wiki.python.org/moin/PythonEventsCalendar#Submitting_an_Event
/dev/
https://devguide.python.org/
https://bugs.python.org/
https://mail.python.org/mailman/listinfo/python-dev
/dev/core-mentorship/
/news/security/
/about/help/
/community/diversity/
https://github.com/python/pythondotorg/issues
https://status.python.org/
/psf-landing/
/about/legal/
/privacy/
/psf/sponsorship/sponsors/#heroku
import httplib2
from BeautifulSoup import BeautifulSoup, SoupStrainer

http = httplib2.Http()
status, response = http.request('http://www.nytimes.com')

for link in BeautifulSoup(response, parse_only=SoupStrainer('a')):
    if link.has_attr('href'):
        print(link['href'])
---------------------------------------------------------------------------
ModuleNotFoundError                       Traceback (most recent call last)
<ipython-input-6-b4252a14d0b2> in <module>
----> 1 import httplib2
      2 from BeautifulSoup import BeautifulSoup, SoupStrainer
      3 
      4 http = httplib2.Http()
      5 status, response = http.request('http://www.nytimes.com')

ModuleNotFoundError: No module named 'httplib2'
pip install httplib2
Collecting httplib2
  Downloading https://files.pythonhosted.org/packages/60/55/3902b9f33ad9c15abf447ad91b86ef2d0835a1ae78530f1410c115cf8fe3/httplib2-0.13.1-py3-none-any.whl (94kB)
    100% |████████████████████████████████| 102kB 2.5MB/s ta 0:00:01
Installing collected packages: httplib2
Successfully installed httplib2-0.13.1
Note: you may need to restart the kernel to use updated packages.
import httplib2
from BeautifulSoup import BeautifulSoup, SoupStrainer

http = httplib2.Http()
status, response = http.request('http://www.nytimes.com')

for link in BeautifulSoup(response, parse_only=SoupStrainer('a')):
    if link.has_attr('href'):
        print(link['href'])
---------------------------------------------------------------------------
ModuleNotFoundError                       Traceback (most recent call last)
<ipython-input-8-b4252a14d0b2> in <module>
      1 import httplib2
----> 2 from BeautifulSoup import BeautifulSoup, SoupStrainer
      3 
      4 http = httplib2.Http()
      5 status, response = http.request('http://www.nytimes.com')

ModuleNotFoundError: No module named 'BeautifulSoup'
pip install BeautifulSoup
Collecting BeautifulSoup
  Downloading https://files.pythonhosted.org/packages/1e/ee/295988deca1a5a7accd783d0dfe14524867e31abb05b6c0eeceee49c759d/BeautifulSoup-3.2.1.tar.gz
    Complete output from command python setup.py egg_info:
    Traceback (most recent call last):
      File "<string>", line 1, in <module>
      File "/tmp/pip-build-x9ywuc54/BeautifulSoup/setup.py", line 22
        print "Unit tests have failed!"
                                      ^
    SyntaxError: Missing parentheses in call to 'print'. Did you mean print("Unit tests have failed!")?
    
    ----------------------------------------
Command "python setup.py egg_info" failed with error code 1 in /tmp/pip-build-x9ywuc54/BeautifulSoup/
Note: you may need to restart the kernel to use updated packages.
import httplib2
from BeautifulSoup import BeautifulSoup, SoupStrainer

http = httplib2.Http()
status, response = http.request('http://www.nytimes.com')

for link in BeautifulSoup(response, parse_only=SoupStrainer('a')):
    if link.has_attr('href'):
        print(link['href'])
---------------------------------------------------------------------------
ModuleNotFoundError                       Traceback (most recent call last)
<ipython-input-10-b4252a14d0b2> in <module>
      1 import httplib2
----> 2 from BeautifulSoup import BeautifulSoup, SoupStrainer
      3 
      4 http = httplib2.Http()
      5 status, response = http.request('http://www.nytimes.com')

ModuleNotFoundError: No module named 'BeautifulSoup'
pip install BeautifulSoup
Collecting BeautifulSoup
  Using cached https://files.pythonhosted.org/packages/1e/ee/295988deca1a5a7accd783d0dfe14524867e31abb05b6c0eeceee49c759d/BeautifulSoup-3.2.1.tar.gz
    Complete output from command python setup.py egg_info:
    Traceback (most recent call last):
      File "<string>", line 1, in <module>
      File "/tmp/pip-build-yg2oqq10/BeautifulSoup/setup.py", line 22
        print "Unit tests have failed!"
                                      ^
    SyntaxError: Missing parentheses in call to 'print'. Did you mean print("Unit tests have failed!")?
    
    ----------------------------------------
Command "python setup.py egg_info" failed with error code 1 in /tmp/pip-build-yg2oqq10/BeautifulSoup/
Note: you may need to restart the kernel to use updated packages.
pip install beautifulsoup4
Requirement already satisfied: beautifulsoup4 in /srv/paws/lib/python3.6/site-packages
Requirement already satisfied: soupsieve>=1.2 in /srv/paws/lib/python3.6/site-packages (from beautifulsoup4)
Note: you may need to restart the kernel to use updated packages.
import httplib2
from beautifulsoup4 import BeautifulSoup, SoupStrainer

http = httplib2.Http()
status, response = http.request('http://www.nytimes.com')

for link in BeautifulSoup(response, parse_only=SoupStrainer('a')):
    if link.has_attr('href'):
        print(link['href'])
---------------------------------------------------------------------------
ModuleNotFoundError                       Traceback (most recent call last)
<ipython-input-13-45f47ac7c382> in <module>
      1 import httplib2
----> 2 from beautifulsoup4 import BeautifulSoup, SoupStrainer
      3 
      4 http = httplib2.Http()
      5 status, response = http.request('http://www.nytimes.com')

ModuleNotFoundError: No module named 'beautifulsoup4'
import httplib2
from bs4 import BeautifulSoup, SoupStrainer

http = httplib2.Http()
status, response = http.request('http://www.nytimes.com')

for link in BeautifulSoup(response, parse_only=SoupStrainer('a')):
    if link.has_attr('href'):
        print(link['href'])
        
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-2-0a4e6f78f066> in <module>
      6 
      7 for link in BeautifulSoup(response, parse_only=SoupStrainer('a')):
----> 8     if link.has_attr('href'):
      9         print(link['href'])
     10 

/srv/paws/lib/python3.6/site-packages/bs4/element.py in __getattr__(self, attr)
    740             raise AttributeError(
    741                 "'%s' object has no attribute '%s'" % (
--> 742                     self.__class__.__name__, attr))
    743 
    744     def output_ready(self, formatter="minimal"):

AttributeError: 'Doctype' object has no attribute 'has_attr'
import requests
from bs4 import BeautifulSoup

url = "http://www.python.org"
response = requests.get(url)
# parse html
page = str(BeautifulSoup(response.content))


def getURL(page):
    """

    :param page: html of web page (here: Python home page) 
    :return: urls in that page 
    """
    start_link = page.find("a href")
    if start_link == -1:
        return None, 0
    start_quote = page.find('"', start_link)
    end_quote = page.find('"', start_quote + 1)
    url = page[start_quote + 1: end_quote]
    return url, end_quote

while True:
    url, n = getURL(page)
    page = page[n:]
    if url:
        print(url)
    else:
        break
http://browsehappy.com/
#content
/psf-landing/
https://docs.python.org
https://pypi.python.org/
/jobs/
/community/
/
https://www.facebook.com/pythonlang?fref=ts
https://twitter.com/ThePSF
/community/irc/
/about/apps/
/about/quotes/
/about/gettingstarted/
/about/help/
http://brochure.getpython.info/
/downloads/
/downloads/source/
/downloads/windows/
/downloads/mac-osx/
/download/other/
https://docs.python.org/3/license.html
/download/alternatives
/doc/
/doc/av
https://wiki.python.org/moin/BeginnersGuide
https://devguide.python.org/
https://docs.python.org/faq/
http://wiki.python.org/moin/Languages
http://python.org/dev/peps/
https://wiki.python.org/moin/PythonBooks
/doc/essays/
/community/survey
/community/diversity/
/community/lists/
/community/irc/
/community/forums/
/psf/annual-report/2019/
/community/workshops/
/community/sigs/
/community/logos/
https://wiki.python.org/moin/
/community/merchandise/
/community/awards
https://www.python.org/psf/codeofconduct/
/success-stories/category/arts/
/success-stories/category/business/
/success-stories/category/education/
/success-stories/category/engineering/
/success-stories/category/government/
/success-stories/category/scientific/
/success-stories/category/software-development/
/blogs/
http://planetpython.org/
http://pyfound.blogspot.com/
http://pycon.blogspot.com/
/events/python-events
/events/python-user-group/
/events/python-events/past/
/events/python-user-group/past/
https://wiki.python.org/moin/PythonEventsCalendar#Submitting_an_Event
//docs.python.org/3/tutorial/controlflow.html#defining-functions
//docs.python.org/3/tutorial/introduction.html#lists
http://docs.python.org/3/tutorial/introduction.html#using-python-as-a-calculator
//docs.python.org/3/tutorial/
//docs.python.org/3/tutorial/controlflow.html
/about/gettingstarted/
/downloads/release/python-374/
https://docs.python.org
//jobs.python.org
https://blog.python.org
http://feedproxy.google.com/~r/PythonInsider/~3/MwuRB1u_KNQ/pypi-now-supports-uploading-via-api.html
http://feedproxy.google.com/~r/PythonSoftwareFoundationNews/~3/PyfIYGrs4Vo/pypi-now-supports-uploading-via-api.html
https://mailchi.mp/python/psf-2019-q2-newsletter
http://feedproxy.google.com/~r/PythonSoftwareFoundationNews/~3/9sGpXmeE1-c/2019-psf-fundraiser-thank-you-debrief.html
http://feedproxy.google.com/~r/PythonSoftwareFoundationNews/~3/hf9N-_dx7VQ/the-python-software-foundation-is.html
/events/calendars/
/events/python-events/776/
/events/python-events/792/
/events/python-events/847/
/events/python-events/854/
/events/python-events/846/
/success-stories/
/success-stories/python-seo-link-analyzer/
/success-stories/python-seo-link-analyzer/
/about/apps
/dev/peps/
/psf/
/about/
/about/apps/
/about/quotes/
/about/gettingstarted/
/about/help/
http://brochure.getpython.info/
/downloads/
/downloads/
/downloads/source/
/downloads/windows/
/downloads/mac-osx/
/download/other/
https://docs.python.org/3/license.html
/download/alternatives
/doc/
/doc/
/doc/av
https://wiki.python.org/moin/BeginnersGuide
https://devguide.python.org/
https://docs.python.org/faq/
http://wiki.python.org/moin/Languages
http://python.org/dev/peps/
https://wiki.python.org/moin/PythonBooks
/doc/essays/
/community/
/community/survey
/community/diversity/
/community/lists/
/community/irc/
/community/forums/
/psf/annual-report/2019/
/community/workshops/
/community/sigs/
/community/logos/
https://wiki.python.org/moin/
/community/merchandise/
/community/awards
https://www.python.org/psf/codeofconduct/
/success-stories/
/success-stories/category/arts/
/success-stories/category/business/
/success-stories/category/education/
/success-stories/category/engineering/
/success-stories/category/government/
/success-stories/category/scientific/
/success-stories/category/software-development/
/blogs/
/blogs/
http://planetpython.org/
http://pyfound.blogspot.com/
http://pycon.blogspot.com/
/events/
/events/python-events
/events/python-user-group/
/events/python-events/past/
/events/python-user-group/past/
https://wiki.python.org/moin/PythonEventsCalendar#Submitting_an_Event
/dev/
https://devguide.python.org/
https://bugs.python.org/
https://mail.python.org/mailman/listinfo/python-dev
/dev/core-mentorship/
/news/security/
/about/help/
/community/diversity/
https://github.com/python/pythondotorg/issues
https://status.python.org/
/psf-landing/
/about/legal/
/privacy/
/psf/sponsorship/sponsors/#heroku