<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>astropy.io.votable.validator.main — Astropy v0.2.4</title>
<link rel="stylesheet" href="../../../../../_static/bootstrap-astropy.css" type="text/css" />
<link rel="stylesheet" href="../../../../../_static/pygments.css" type="text/css" />
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT: '../../../../../',
VERSION: '0.2.4',
COLLAPSE_INDEX: false,
FILE_SUFFIX: '.html',
HAS_SOURCE: true
};
</script>
<script type="text/javascript" src="../../../../../_static/jquery.js"></script>
<script type="text/javascript" src="../../../../../_static/underscore.js"></script>
<script type="text/javascript" src="../../../../../_static/doctools.js"></script>
<script type="text/javascript" src="../../../../../_static/sidebar.js"></script>
<link rel="shortcut icon" href="../../../../../_static/astropy_logo.ico"/>
<link rel="top" title="Astropy v0.2.4" href="../../../../../index.html" />
<link rel="up" title="Module code" href="../../../../index.html" />
</head>
<body>
<div class="topbar">
<a class="brand" title="Documentation Home" href="../../../../../index.html"></a>
<ul>
<li><a class="homelink" title="AstroPy Homepage" href="http://www.astropy.org"></a></li>
<li><a title="General Index" href="../../../../../genindex.html">Index</a></li>
<li><a title="Python Module Index" href="../../../../../py-modindex.html">Modules</a></li>
<li>
<form action="../../../../../search.html" method="get">
<input type="text" name="q" placeholder="Search" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</li>
</ul>
</div>
<div class="related">
<h3>Navigation</h3>
<ul>
<li>
<a href="../../../../../index.html">Astropy v0.2.4</a>
»
</li>
<li><a href="../../../../index.html" accesskey="U">Module code</a> »</li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body">
<h1>Source code for astropy.io.votable.validator.main</h1><div class="highlight"><pre>
<span class="c"># Licensed under a 3-clause BSD style license - see LICENSE.rst</span>
<span class="sd">"""</span>
<span class="sd">Validates a large collection of web-accessible VOTable files,</span>
<span class="sd">and generates a report as a directory tree of HTML files.</span>
<span class="sd">"""</span>
<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">print_function</span>
<span class="c"># STDLIB</span>
<span class="kn">import</span> <span class="nn">os</span>
<span class="c"># LOCAL</span>
<span class="kn">from</span> <span class="nn">....utils.data</span> <span class="kn">import</span> <span class="n">get_pkg_data_filename</span>
<span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">html</span>
<span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">result</span>
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s">'make_validation_report'</span><span class="p">]</span>
<span class="k">def</span> <span class="nf">get_srcdir</span><span class="p">():</span>
<span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="n">__file__</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">get_urls</span><span class="p">(</span><span class="n">destdir</span><span class="p">,</span> <span class="n">s</span><span class="p">):</span>
<span class="kn">import</span> <span class="nn">gzip</span>
<span class="n">types</span> <span class="o">=</span> <span class="p">[</span><span class="s">'good'</span><span class="p">,</span> <span class="s">'broken'</span><span class="p">,</span> <span class="s">'incorrect'</span><span class="p">]</span>
<span class="n">seen</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
<span class="n">urls</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="nb">type</span> <span class="ow">in</span> <span class="n">types</span><span class="p">:</span>
<span class="n">filename</span> <span class="o">=</span> <span class="n">get_pkg_data_filename</span><span class="p">(</span>
<span class="s">'urls/cone.{0}.dat.gz'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">type</span><span class="p">))</span>
<span class="k">with</span> <span class="n">gzip</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s">'rb'</span><span class="p">)</span> <span class="k">as</span> <span class="n">fd</span><span class="p">:</span>
<span class="k">for</span> <span class="n">url</span> <span class="ow">in</span> <span class="n">fd</span><span class="o">.</span><span class="n">readlines</span><span class="p">():</span>
<span class="n">s</span><span class="o">.</span><span class="n">next</span><span class="p">()</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">url</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
<span class="k">if</span> <span class="n">url</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">seen</span><span class="p">:</span>
<span class="k">with</span> <span class="n">result</span><span class="o">.</span><span class="n">Result</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">root</span><span class="o">=</span><span class="n">destdir</span><span class="p">)</span> <span class="k">as</span> <span class="n">r</span><span class="p">:</span>
<span class="n">r</span><span class="p">[</span><span class="s">'expected'</span><span class="p">]</span> <span class="o">=</span> <span class="nb">type</span>
<span class="n">urls</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
<span class="n">seen</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
<span class="k">return</span> <span class="n">urls</span>
<span class="k">def</span> <span class="nf">download</span><span class="p">(</span><span class="n">args</span><span class="p">):</span>
<span class="n">url</span><span class="p">,</span> <span class="n">destdir</span> <span class="o">=</span> <span class="n">args</span>
<span class="k">with</span> <span class="n">result</span><span class="o">.</span><span class="n">Result</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">root</span><span class="o">=</span><span class="n">destdir</span><span class="p">)</span> <span class="k">as</span> <span class="n">r</span><span class="p">:</span>
<span class="n">r</span><span class="o">.</span><span class="n">download_xml_content</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">validate_vo</span><span class="p">(</span><span class="n">args</span><span class="p">):</span>
<span class="n">url</span><span class="p">,</span> <span class="n">destdir</span> <span class="o">=</span> <span class="n">args</span>
<span class="k">with</span> <span class="n">result</span><span class="o">.</span><span class="n">Result</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">root</span><span class="o">=</span><span class="n">destdir</span><span class="p">)</span> <span class="k">as</span> <span class="n">r</span><span class="p">:</span>
<span class="n">r</span><span class="o">.</span><span class="n">validate_vo</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">votlint_validate</span><span class="p">(</span><span class="n">args</span><span class="p">):</span>
<span class="n">path_to_stilts_jar</span><span class="p">,</span> <span class="n">url</span><span class="p">,</span> <span class="n">destdir</span> <span class="o">=</span> <span class="n">args</span>
<span class="k">with</span> <span class="n">result</span><span class="o">.</span><span class="n">Result</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">root</span><span class="o">=</span><span class="n">destdir</span><span class="p">)</span> <span class="k">as</span> <span class="n">r</span><span class="p">:</span>
<span class="k">if</span> <span class="n">r</span><span class="p">[</span><span class="s">'network_error'</span><span class="p">]</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="n">r</span><span class="o">.</span><span class="n">validate_with_votlint</span><span class="p">(</span><span class="n">path_to_stilts_jar</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">write_html_result</span><span class="p">(</span><span class="n">args</span><span class="p">):</span>
<span class="n">url</span><span class="p">,</span> <span class="n">destdir</span> <span class="o">=</span> <span class="n">args</span>
<span class="k">with</span> <span class="n">result</span><span class="o">.</span><span class="n">Result</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">root</span><span class="o">=</span><span class="n">destdir</span><span class="p">)</span> <span class="k">as</span> <span class="n">r</span><span class="p">:</span>
<span class="n">html</span><span class="o">.</span><span class="n">write_result</span><span class="p">(</span><span class="n">r</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">write_subindex</span><span class="p">(</span><span class="n">args</span><span class="p">):</span>
<span class="n">subset</span><span class="p">,</span> <span class="n">destdir</span><span class="p">,</span> <span class="n">total</span> <span class="o">=</span> <span class="n">args</span>
<span class="n">html</span><span class="o">.</span><span class="n">write_index_table</span><span class="p">(</span><span class="n">destdir</span><span class="p">,</span> <span class="o">*</span><span class="n">subset</span><span class="p">,</span> <span class="n">total</span><span class="o">=</span><span class="n">total</span><span class="p">)</span>
<div class="viewcode-block" id="make_validation_report"><a class="viewcode-back" href="../../../../../_generated/astropy.io.votable.validator.main.make_validation_report.html#astropy.io.votable.validator.main.make_validation_report">[docs]</a><span class="k">def</span> <span class="nf">make_validation_report</span><span class="p">(</span>
<span class="n">urls</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">destdir</span><span class="o">=</span><span class="s">'astropy.io.votable.validator.results'</span><span class="p">,</span>
<span class="n">multiprocess</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span> <span class="n">stilts</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Validates a large collection of web-accessible VOTable files.</span>
<span class="sd"> Generates a report as a directory tree of HTML files.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> urls : list of strings, optional</span>
<span class="sd"> If provided, is a list of HTTP urls to download VOTable files</span>
<span class="sd"> from. If not provided, a built-in set of ~22,000 urls</span>
<span class="sd"> compiled by HEASARC will be used.</span>
<span class="sd"> destdir : path, optional</span>
<span class="sd"> The directory to write the report to. By default, this is a</span>
<span class="sd"> directory called `astropy.io.votable.validator.results` in the</span>
<span class="sd"> current directory. If the directory does not exist, it will</span>
<span class="sd"> be created.</span>
<span class="sd"> multiprocess : bool, optional</span>
<span class="sd"> If `True` (default), perform validations in parallel using all</span>
<span class="sd"> of the cores on this machine.</span>
<span class="sd"> stilts : path, optional</span>
<span class="sd"> To perform validation with `votlint` from the the Java-based</span>
<span class="sd"> `STILTS <http://www.star.bris.ac.uk/~mbt/stilts/>`_ VOTable</span>
<span class="sd"> parser, in addition to `astropy.io.votable`, set this to the</span>
<span class="sd"> path of the `stilts.jar` file. `java` on the system shell</span>
<span class="sd"> path will be used to run it.</span>
<span class="sd"> Notes</span>
<span class="sd"> -----</span>
<span class="sd"> Downloads of each given URL will be performed only once and cached</span>
<span class="sd"> locally in *destdir*. To refresh the cache, remove *destdir*</span>
<span class="sd"> first.</span>
<span class="sd"> """</span>
<span class="kn">from</span> <span class="nn">....utils.console</span> <span class="kn">import</span> <span class="p">(</span><span class="n">color_print</span><span class="p">,</span> <span class="n">ProgressBar</span><span class="p">,</span> <span class="n">Spinner</span><span class="p">)</span>
<span class="k">if</span> <span class="n">stilts</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">stilts</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
<span class="s">'{0} does not exist.'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">stilts</span><span class="p">))</span>
<span class="n">destdir</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="n">destdir</span><span class="p">)</span>
<span class="k">if</span> <span class="n">urls</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="k">with</span> <span class="n">Spinner</span><span class="p">(</span><span class="s">'Loading URLs'</span><span class="p">,</span> <span class="s">'green'</span><span class="p">)</span> <span class="k">as</span> <span class="n">s</span><span class="p">:</span>
<span class="n">urls</span> <span class="o">=</span> <span class="n">get_urls</span><span class="p">(</span><span class="n">destdir</span><span class="p">,</span> <span class="n">s</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">color_print</span><span class="p">(</span><span class="s">'Marking URLs'</span><span class="p">,</span> <span class="s">'green'</span><span class="p">)</span>
<span class="k">for</span> <span class="n">url</span> <span class="ow">in</span> <span class="n">ProgressBar</span><span class="o">.</span><span class="n">iterate</span><span class="p">(</span><span class="n">urls</span><span class="p">):</span>
<span class="k">with</span> <span class="n">result</span><span class="o">.</span><span class="n">Result</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">root</span><span class="o">=</span><span class="n">destdir</span><span class="p">)</span> <span class="k">as</span> <span class="n">r</span><span class="p">:</span>
<span class="n">r</span><span class="p">[</span><span class="s">'expected'</span><span class="p">]</span> <span class="o">=</span> <span class="nb">type</span>
<span class="n">args</span> <span class="o">=</span> <span class="p">[(</span><span class="n">url</span><span class="p">,</span> <span class="n">destdir</span><span class="p">)</span> <span class="k">for</span> <span class="n">url</span> <span class="ow">in</span> <span class="n">urls</span><span class="p">]</span>
<span class="n">color_print</span><span class="p">(</span><span class="s">'Downloading VO files'</span><span class="p">,</span> <span class="s">'green'</span><span class="p">)</span>
<span class="n">ProgressBar</span><span class="o">.</span><span class="n">map</span><span class="p">(</span>
<span class="n">download</span><span class="p">,</span> <span class="n">args</span><span class="p">,</span> <span class="n">multiprocess</span><span class="o">=</span><span class="n">multiprocess</span><span class="p">)</span>
<span class="n">color_print</span><span class="p">(</span><span class="s">'Validating VO files'</span><span class="p">,</span> <span class="s">'green'</span><span class="p">)</span>
<span class="n">ProgressBar</span><span class="o">.</span><span class="n">map</span><span class="p">(</span>
<span class="n">validate_vo</span><span class="p">,</span> <span class="n">args</span><span class="p">,</span> <span class="n">multiprocess</span><span class="o">=</span><span class="n">multiprocess</span><span class="p">)</span>
<span class="k">if</span> <span class="n">stilts</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
<span class="n">color_print</span><span class="p">(</span><span class="s">'Validating with votlint'</span><span class="p">,</span> <span class="s">'green'</span><span class="p">)</span>
<span class="n">votlint_args</span> <span class="o">=</span> <span class="p">[(</span><span class="n">stilts</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">destdir</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">urls</span><span class="p">]</span>
<span class="n">ProgressBar</span><span class="o">.</span><span class="n">map</span><span class="p">(</span>
<span class="n">votlint_validate</span><span class="p">,</span> <span class="n">votlint_args</span><span class="p">,</span> <span class="n">multiprocess</span><span class="o">=</span><span class="n">multiprocess</span><span class="p">)</span>
<span class="n">color_print</span><span class="p">(</span><span class="s">'Generating HTML files'</span><span class="p">,</span> <span class="s">'green'</span><span class="p">)</span>
<span class="n">ProgressBar</span><span class="o">.</span><span class="n">map</span><span class="p">(</span>
<span class="n">write_html_result</span><span class="p">,</span> <span class="n">args</span><span class="p">,</span> <span class="n">multiprocess</span><span class="o">=</span><span class="n">multiprocess</span><span class="p">)</span>
<span class="k">with</span> <span class="n">Spinner</span><span class="p">(</span><span class="s">'Grouping results'</span><span class="p">,</span> <span class="s">'green'</span><span class="p">)</span> <span class="k">as</span> <span class="n">s</span><span class="p">:</span>
<span class="n">subsets</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get_result_subsets</span><span class="p">(</span><span class="n">urls</span><span class="p">,</span> <span class="n">destdir</span><span class="p">,</span> <span class="n">s</span><span class="p">)</span>
<span class="n">color_print</span><span class="p">(</span><span class="s">'Generating index'</span><span class="p">,</span> <span class="s">'green'</span><span class="p">)</span>
<span class="n">html</span><span class="o">.</span><span class="n">write_index</span><span class="p">(</span><span class="n">subsets</span><span class="p">,</span> <span class="n">urls</span><span class="p">,</span> <span class="n">destdir</span><span class="p">)</span>
<span class="n">color_print</span><span class="p">(</span><span class="s">'Generating subindices'</span><span class="p">,</span> <span class="s">'green'</span><span class="p">)</span>
<span class="n">subindex_args</span> <span class="o">=</span> <span class="p">[(</span><span class="n">subset</span><span class="p">,</span> <span class="n">destdir</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">urls</span><span class="p">))</span> <span class="k">for</span> <span class="n">subset</span> <span class="ow">in</span> <span class="n">subsets</span><span class="p">]</span>
<span class="n">ProgressBar</span><span class="o">.</span><span class="n">map</span><span class="p">(</span>
<span class="n">write_subindex</span><span class="p">,</span> <span class="n">subindex_args</span><span class="p">,</span> <span class="n">multiprocess</span><span class="o">=</span><span class="n">multiprocess</span><span class="p">)</span></div>
</pre></div>
</div>
</div>
</div>
<div class="sphinxsidebar">
<div class="sphinxsidebarwrapper"><h3>Page Contents</h3>
</div>
</div>
<div class="clearer"></div>
</div>
<footer class="footer">
<p class="pull-right">
<a href="#">Back to Top</a></p>
<p>
© Copyright 2011-2013, The Astropy Developers.<br/>
Created using <a href="http://sphinx.pocoo.org/">Sphinx</a> 1.1.3.
Last built 22 Oct 2013. <br/>
</p>
</footer>
</body>
</html>
