mirror of
https://github.com/ArchiveBox/ArchiveBox
synced 2024-11-10 06:34:16 +00:00
test: Refactor args tests to use local webserver instead of remote requests
This commit is contained in:
Cristian
parent
bf417f50a4
commit
8f2543fa61
4 changed files with 458 additions and 10 deletions
|
|
@ -1,8 +1,17 @@
|
|||
from bottle import route, run
|
||||
from os.path import abspath
|
||||
from os import getcwd
|
||||
from pathlib import Path
|
||||
|
||||
@route('/')
|
||||
from bottle import route, run, static_file
|
||||
|
||||
@route("/")
|
||||
def index():
|
||||
return "Hello"
|
||||
|
||||
@route("/static/<filename>")
|
||||
def static_path(filename):
|
||||
template_path = abspath(getcwd()) / Path("tests/mock_server/templates")
|
||||
return static_file(filename, root=template_path)
|
||||
|
||||
def start():
|
||||
run(host='localhost', port=8080)
|
||||
49
tests/mock_server/templates/example.com.html
Normal file
49
tests/mock_server/templates/example.com.html
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Example Domain</title>
|
||||
|
||||
<meta charset="utf-8"/>
|
||||
<meta http-equiv="Content-type" content="text/html; charset=utf-8"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<style type="text/css">
|
||||
body {
|
||||
background-color: #f0f0f2;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
font-family: -apple-system, system-ui, BlinkMacSystemFont, "Segoe UI", "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
|
||||
|
||||
}
|
||||
div {
|
||||
width: 600px;
|
||||
margin: 5em auto;
|
||||
padding: 2em;
|
||||
background-color: #fdfdff;
|
||||
border-radius: 0.5em;
|
||||
box-shadow: 2px 3px 7px 2px rgba(0, 0, 0, 0.02);
|
||||
}
|
||||
a:link,
|
||||
a:visited {
|
||||
color: #38488f;
|
||||
text-decoration: none;
|
||||
}
|
||||
@media(max-width: 700px) {
|
||||
div {
|
||||
margin: 0 auto;
|
||||
width: auto;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<div>
|
||||
<h1>Example Domain</h1>
|
||||
<p>This domain is for use in illustrative examples in documents. You may use this
|
||||
domain in literature without prior coordination or asking for permission.</p>
|
||||
<p>
|
||||
<a href="http://localhost:8080/static/iana.org.html">More information...</a>
|
||||
</p>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
390
tests/mock_server/templates/iana.org.html
Normal file
390
tests/mock_server/templates/iana.org.html
Normal file
|
|
@ -0,0 +1,390 @@
|
|||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<title>IANA — IANA-managed Reserved Domains</title>
|
||||
|
||||
<meta charset="utf-8"/>
|
||||
<meta http-equiv="Content-type" content="text/html; charset=utf-8"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
|
||||
<link rel="stylesheet" media="screen" href="/_css/2015.1/screen.css"/>
|
||||
<link rel="stylesheet" media="print" href="/_css/2015.1/print.css"/>
|
||||
<link rel="shortcut icon" type="image/ico" href="/_img/bookmark_icon.ico"/>
|
||||
|
||||
<script type="text/javascript" src="/_js/2013.1/jquery.js"></script>
|
||||
<script type="text/javascript" src="/_js/2013.1/iana.js"></script>
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<body>
|
||||
|
||||
<header>
|
||||
<div id="header">
|
||||
<div id="logo">
|
||||
<a href="/"><img src="/_img/2013.1/iana-logo-header.svg" alt="Homepage"/></a>
|
||||
</div>
|
||||
<div class="navigation">
|
||||
<ul>
|
||||
<li><a href="/domains">Domains</a></li>
|
||||
<li><a href="/numbers">Numbers</a></li>
|
||||
<li><a href="/protocols">Protocols</a></li>
|
||||
<li><a href="/about">About Us</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<div id="body">
|
||||
|
||||
|
||||
<div id="main_right">
|
||||
|
||||
|
||||
<h1>IANA-managed Reserved Domains</h1>
|
||||
|
||||
<p>Certain domains are set aside, and nominally registered to “IANA”, for specific
|
||||
policy or technical purposes.</p>
|
||||
|
||||
<h2>Example domains</h2>
|
||||
|
||||
<p>As described in
|
||||
<a href="/go/rfc2606">RFC 2606</a>
|
||||
and
|
||||
<a href="/go/rfc6761">RFC 6761</a>,
|
||||
a number of domains such as
|
||||
<span class="domain label">example.com</span>
|
||||
and
|
||||
<span class="domain label">example.org</span>
|
||||
are maintained for documentation purposes. These domains may be used as illustrative
|
||||
examples in documents without prior coordination with us. They are
|
||||
not available for registration or transfer.</p>
|
||||
|
||||
<h2>Test IDN top-level domains</h2>
|
||||
|
||||
<p>These domains were temporarily delegated by IANA for the
|
||||
<a href="http://www.icann.org/topics/idn/">IDN Evaluation</a>
|
||||
being conducted by
|
||||
<a href="http://www.icann.org/">ICANN</a>.</p>
|
||||
|
||||
<div class="iana-table-frame">
|
||||
<table id="arpa-table" class="iana-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Domain</th>
|
||||
<th>Domain (A-label)</th>
|
||||
<th>Language</th>
|
||||
<th>Script</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>إختبار</td>
|
||||
<td>
|
||||
<span class="domain label">
|
||||
<a href="/domains/root/db/xn--kgbechtv.html">XN--KGBECHTV</a>
|
||||
</span>
|
||||
</td>
|
||||
<td>Arabic</td>
|
||||
<td>Arabic</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>آزمایشی</td>
|
||||
<td>
|
||||
<span class="domain label">
|
||||
<a href="/domains/root/db/xn--hgbk6aj7f53bba.html">XN--HGBK6AJ7F53BBA</a>
|
||||
</span>
|
||||
</td>
|
||||
<td>Persian</td>
|
||||
<td>Arabic</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>测试</td>
|
||||
<td>
|
||||
<span class="domain label">
|
||||
<a href="/domains/root/db/xn--0zwm56d.html">XN--0ZWM56D</a>
|
||||
</span>
|
||||
</td>
|
||||
<td>Chinese</td>
|
||||
<td>Han (Simplified variant)</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>測試</td>
|
||||
<td>
|
||||
<span class="domain label">
|
||||
<a href="/domains/root/db/xn--g6w251d.html">XN--G6W251D</a>
|
||||
</span>
|
||||
</td>
|
||||
<td>Chinese</td>
|
||||
<td>Han (Traditional variant)</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>испытание</td>
|
||||
<td>
|
||||
<span class="domain label">
|
||||
<a href="/domains/root/db/xn--80akhbyknj4f.html">XN--80AKHBYKNJ4F</a>
|
||||
</span>
|
||||
</td>
|
||||
<td>Russian</td>
|
||||
<td>Cyrillic</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>परीक्षा</td>
|
||||
<td>
|
||||
<span class="domain label">
|
||||
<a href="/domains/root/db/xn--11b5bs3a9aj6g.html">XN--11B5BS3A9AJ6G</a>
|
||||
</span>
|
||||
</td>
|
||||
<td>Hindi</td>
|
||||
<td>Devanagari (Nagari)</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>δοκιμή</td>
|
||||
<td>
|
||||
<span class="domain label">
|
||||
<a href="/domains/root/db/xn--jxalpdlp.html">XN--JXALPDLP</a>
|
||||
</span>
|
||||
</td>
|
||||
<td>Greek, Modern (1453-)</td>
|
||||
<td>Greek</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>테스트</td>
|
||||
<td>
|
||||
<span class="domain label">
|
||||
<a href="/domains/root/db/xn--9t4b11yi5a.html">XN--9T4B11YI5A</a>
|
||||
</span>
|
||||
</td>
|
||||
<td>Korean</td>
|
||||
<td>Hangul (Hangŭl, Hangeul)</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>טעסט</td>
|
||||
<td>
|
||||
<span class="domain label">
|
||||
<a href="/domains/root/db/xn--deba0ad.html">XN--DEBA0AD</a>
|
||||
</span>
|
||||
</td>
|
||||
<td>Yiddish</td>
|
||||
<td>Hebrew</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>テスト</td>
|
||||
<td>
|
||||
<span class="domain label">
|
||||
<a href="/domains/root/db/xn--zckzah.html">XN--ZCKZAH</a>
|
||||
</span>
|
||||
</td>
|
||||
<td>Japanese</td>
|
||||
<td>Katakana</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>பரிட்சை</td>
|
||||
<td>
|
||||
<span class="domain label">
|
||||
<a href="/domains/root/db/xn--hlcj6aya9esc7a.html">XN--HLCJ6AYA9ESC7A</a>
|
||||
</span>
|
||||
</td>
|
||||
<td>Tamil</td>
|
||||
<td>Tamil</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<h2>Policy-reserved domains</h2>
|
||||
|
||||
<p>We act as both the registrant and registrar for a select number of domains
|
||||
which have been reserved under policy grounds. These exclusions are
|
||||
typically indicated in either technical standards (RFC documents),
|
||||
or
|
||||
<a href="http://www.icann.org/en/registries/agreements.htm">contractual limitations</a>.</p>
|
||||
|
||||
<p>Domains which are described as registered to IANA or ICANN on policy
|
||||
grounds are not available for registration or transfer, with the exception
|
||||
of
|
||||
<span class="domain label">
|
||||
<i>country-name</i>.info</span>
|
||||
domains. These domains are available for release
|
||||
by the ICANN Governmental Advisory Committee Secretariat.</p>
|
||||
|
||||
<h2>Other Special-Use Domains</h2>
|
||||
|
||||
<p>There is additionally a
|
||||
<a href="/assignments/special-use-domain-names">Special-Use Domain Names</a>
|
||||
registry documenting special-use domains designated by technical standards. For further information, see
|
||||
<a href="/go/rfc6761">Special-Use Domain Names</a>
|
||||
(RFC 6761).</p>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div id="sidebar_left">
|
||||
<div class="navigation_box">
|
||||
<h2>Domain Names</h2>
|
||||
<ul>
|
||||
<li id="nav_dom_top">
|
||||
<a href="/domains">Overview</a>
|
||||
</li>
|
||||
<li id="nav_dom_root">
|
||||
<a href="/domains/root">Root Zone Management</a>
|
||||
</li>
|
||||
<ul id="nav_dom_root_sub">
|
||||
<li id="nav_dom_root_top">
|
||||
<a href="/domains/root">Overview</a>
|
||||
</li>
|
||||
<li id="nav_dom_root_db">
|
||||
<a href="/domains/root/db">Root Database</a>
|
||||
</li>
|
||||
<li id="nav_dom_root_files">
|
||||
<a href="/domains/root/files">Hint and Zone Files</a>
|
||||
</li>
|
||||
<li id="nav_dom_root_manage">
|
||||
<a href="/domains/root/manage">Change Requests</a>
|
||||
</li>
|
||||
<li id="nav_dom_root_procedures">
|
||||
<a href="/domains/root/help">Instructions & Guides</a>
|
||||
</li>
|
||||
<li id="nav_dom_root_servers">
|
||||
<a href="/domains/root/servers">Root Servers</a>
|
||||
</li>
|
||||
</ul>
|
||||
<li id="nav_dom_int">
|
||||
<a href="/domains/int">.INT Registry</a>
|
||||
</li>
|
||||
<ul id="nav_dom_int_sub">
|
||||
<li id="nav_dom_int_top">
|
||||
<a href="/domains/int">Overview</a>
|
||||
</li>
|
||||
<li id="nav_dom_int_manage">
|
||||
<a href="/domains/int/manage">Register/modify an .INT domain</a>
|
||||
</li>
|
||||
<li id="nav_dom_int_policy">
|
||||
<a href="/domains/int/policy">Eligibility</a>
|
||||
</li>
|
||||
</ul>
|
||||
<li id="nav_dom_arpa">
|
||||
<a href="/domains/arpa">.ARPA Registry</a>
|
||||
</li>
|
||||
<li id="nav_dom_idn">
|
||||
<a href="/domains/idn-tables">IDN Practices Repository</a>
|
||||
</li>
|
||||
<ul id="nav_dom_idn_sub">
|
||||
<li id="nav_dom_idn_top">
|
||||
<a href="/domains/idn-tables">Overview</a>
|
||||
</li>
|
||||
<!-- <li id="nav_dom_idn_tables"><a href="/domains/idn-tables/db">Tables</a></li> -->
|
||||
<li id="nav_dom_idn_submit">
|
||||
<a href="/procedures/idn-repository.html">Submit a table</a>
|
||||
</li>
|
||||
</ul>
|
||||
<li id="nav_dom_dnssec">
|
||||
<a href="/dnssec">Root Key Signing Key (DNSSEC)</a>
|
||||
</li>
|
||||
<ul id="nav_dom_dnssec_sub">
|
||||
<li id="nav_dom_dnssec_top">
|
||||
<a href="/dnssec">Overview</a>
|
||||
</li>
|
||||
<li id="nav_dom_dnssec_ksk">
|
||||
<a href="/dnssec/files">Trusts Anchors and Keys</a>
|
||||
</li>
|
||||
<li id="nav_dom_dnssec_ceremonies">
|
||||
<a href="/dnssec/ceremonies">Root KSK Ceremonies</a>
|
||||
</li>
|
||||
<li id="nav_dom_dnssec_dps">
|
||||
<a href="/dnssec/dps">Practice Statement</a>
|
||||
</li>
|
||||
<li id="nav_dom_dnssec_tcrs">
|
||||
<a href="/dnssec/tcrs">Community Representatives</a>
|
||||
</li>
|
||||
</ul>
|
||||
<li id="nav_dom_special">
|
||||
<a href="/domains/reserved">Reserved Domains</a>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<footer>
|
||||
<div id="footer">
|
||||
<table class="navigation">
|
||||
<tr>
|
||||
<td class="section">
|
||||
<a href="/domains">Domain Names</a>
|
||||
</td>
|
||||
<td class="subsection">
|
||||
<ul>
|
||||
<li><a href="/domains/root">Root Zone Registry</a></li>
|
||||
<li><a href="/domains/int">.INT Registry</a></li>
|
||||
<li><a href="/domains/arpa">.ARPA Registry</a></li>
|
||||
<li><a href="/domains/idn-tables">IDN Repository</a></li>
|
||||
</ul>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="section">
|
||||
<a href="/numbers">Number Resources</a>
|
||||
</td>
|
||||
<td class="subsection">
|
||||
<ul>
|
||||
<li><a href="/abuse">Abuse Information</a></li>
|
||||
</ul>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="section">
|
||||
<a href="/protocols">Protocols</a>
|
||||
</td>
|
||||
<td class="subsection">
|
||||
<ul>
|
||||
<li><a href="/protocols">Protocol Registries</a></li>
|
||||
<li><a href="/time-zones">Time Zone Database</a></li>
|
||||
</ul>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="section">
|
||||
<a href="/about">About Us</a>
|
||||
</td>
|
||||
<td class="subsection">
|
||||
<ul>
|
||||
<li><a href="/about/presentations">Presentations</a></li>
|
||||
<li><a href="/reports">Reports</a></li>
|
||||
<li><a href="/performance">Performance</a></li>
|
||||
<li><a href="/reviews">Reviews</a></li>
|
||||
<li><a href="/about/excellence">Excellence</a></li>
|
||||
<li><a href="/contact">Contact Us</a></li>
|
||||
</ul>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<div id="custodian">
|
||||
<p>The IANA functions coordinate the Internet’s globally unique identifiers, and
|
||||
are provided by
|
||||
<a href="http://pti.icann.org">Public Technical Identifiers</a>, an affiliate of
|
||||
<a href="http://www.icann.org/">ICANN</a>.</p>
|
||||
</div>
|
||||
|
||||
<div id="legalnotice">
|
||||
<ul>
|
||||
<li><a href="https://www.icann.org/privacy/policy">Privacy Policy</a></li>
|
||||
<li><a href="https://www.icann.org/privacy/tos">Terms of Service</a></li>
|
||||
</ul>
|
||||
</p>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</body>
|
||||
</html></footer><script>
|
||||
$(document).ready(function () {
|
||||
$("#nav_dom_special").addClass("selected")
|
||||
$("#nav_dom_int_sub").hide()
|
||||
$("#nav_dom_idn_sub").hide()
|
||||
$("#nav_dom_dnssec_sub").hide()
|
||||
$("#nav_dom_tools_sub").hide()
|
||||
$("#nav_dom_root_sub").hide()
|
||||
});</script></body></html>
|
||||
|
|
@ -4,25 +4,25 @@ import json
|
|||
from .fixtures import *
|
||||
|
||||
def test_depth_flag_is_accepted(process):
|
||||
arg_process = subprocess.run(["archivebox", "add", "https://example.com", "--depth=0"], capture_output=True)
|
||||
arg_process = subprocess.run(["archivebox", "add", "http://localhost:8080/static/example.com.html", "--depth=0"], capture_output=True)
|
||||
assert 'unrecognized arguments: --depth' not in arg_process.stderr.decode("utf-8")
|
||||
|
||||
def test_depth_flag_fails_if_it_is_not_0_or_1(process):
|
||||
arg_process = subprocess.run(["archivebox", "add", "https://example.com", "--depth=5"], capture_output=True)
|
||||
arg_process = subprocess.run(["archivebox", "add", "http://localhost:8080/static/example.com.html", "--depth=5"], capture_output=True)
|
||||
assert 'invalid choice' in arg_process.stderr.decode("utf-8")
|
||||
arg_process = subprocess.run(["archivebox", "add", "https://example.com", "--depth=-1"], capture_output=True)
|
||||
arg_process = subprocess.run(["archivebox", "add", "http://localhost:8080/static/example.com.html", "--depth=-1"], capture_output=True)
|
||||
assert 'invalid choice' in arg_process.stderr.decode("utf-8")
|
||||
|
||||
def test_depth_flag_0_crawls_only_the_arg_page(tmp_path, process):
|
||||
arg_process = subprocess.run(["archivebox", "add", "https://example.com", "--depth=0"], capture_output=True)
|
||||
arg_process = subprocess.run(["archivebox", "add", "http://localhost:8080/static/example.com.html", "--depth=0"], capture_output=True)
|
||||
archived_item_path = list(tmp_path.glob('archive/**/*'))[0]
|
||||
with open(archived_item_path / "index.json", "r") as f:
|
||||
output_json = json.load(f)
|
||||
assert output_json["base_url"] == "example.com"
|
||||
assert output_json["base_url"] == "localhost:8080/static/example.com.html"
|
||||
|
||||
def test_depth_flag_1_crawls_the_page_AND_links(tmp_path, process):
|
||||
arg_process = subprocess.run(["archivebox", "add", "https://example.com", "--depth=1"], capture_output=True)
|
||||
arg_process = subprocess.run(["archivebox", "add", "http://localhost:8080/static/example.com.html", "--depth=1"], capture_output=True)
|
||||
with open(tmp_path / "index.json", "r") as f:
|
||||
archive_file = f.read()
|
||||
assert "https://example.com" in archive_file
|
||||
assert "https://www.iana.org/domains/example" in archive_file
|
||||
assert "http://localhost:8080/static/example.com.html" in archive_file
|
||||
assert "http://localhost:8080/static/iana.org.html" in archive_file
|
||||
|
|
|
|||
Loading…
Reference in a new issue