<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>astropy.utils.data — Astropy v0.2.4</title>
<link rel="stylesheet" href="../../../_static/bootstrap-astropy.css" type="text/css" />
<link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT: '../../../',
VERSION: '0.2.4',
COLLAPSE_INDEX: false,
FILE_SUFFIX: '.html',
HAS_SOURCE: true
};
</script>
<script type="text/javascript" src="../../../_static/jquery.js"></script>
<script type="text/javascript" src="../../../_static/underscore.js"></script>
<script type="text/javascript" src="../../../_static/doctools.js"></script>
<script type="text/javascript" src="../../../_static/sidebar.js"></script>
<link rel="shortcut icon" href="../../../_static/astropy_logo.ico"/>
<link rel="top" title="Astropy v0.2.4" href="../../../index.html" />
<link rel="up" title="astropy.utils" href="../utils.html" />
</head>
<body>
<div class="topbar">
<a class="brand" title="Documentation Home" href="../../../index.html"></a>
<ul>
<li><a class="homelink" title="AstroPy Homepage" href="http://www.astropy.org"></a></li>
<li><a title="General Index" href="../../../genindex.html">Index</a></li>
<li><a title="Python Module Index" href="../../../py-modindex.html">Modules</a></li>
<li>
<form action="../../../search.html" method="get">
<input type="text" name="q" placeholder="Search" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</li>
</ul>
</div>
<div class="related">
<h3>Navigation</h3>
<ul>
<li>
<a href="../../../index.html">Astropy v0.2.4</a>
»
</li>
<li><a href="../../index.html" >Module code</a> »</li>
<li><a href="../utils.html" accesskey="U">astropy.utils</a> »</li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body">
<h1>Source code for astropy.utils.data</h1><div class="highlight"><pre>
<span class="c"># Licensed under a 3-clause BSD style license - see LICENSE.rst</span>
<span class="sd">""" This module contains helper functions for accessing, downloading, and</span>
<span class="sd">caching data files.</span>
<span class="sd">"""</span>
<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">division</span>
<span class="kn">import</span> <span class="nn">os</span>
<span class="kn">import</span> <span class="nn">io</span>
<span class="kn">import</span> <span class="nn">sys</span>
<span class="kn">import</span> <span class="nn">atexit</span>
<span class="kn">import</span> <span class="nn">contextlib</span>
<span class="kn">import</span> <span class="nn">urllib2</span>
<span class="kn">from</span> <span class="nn">..config.configuration</span> <span class="kn">import</span> <span class="n">ConfigurationItem</span>
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s">'get_readable_fileobj'</span><span class="p">,</span> <span class="s">'get_file_contents'</span><span class="p">,</span> <span class="s">'get_pkg_data_fileobj'</span><span class="p">,</span>
<span class="s">'get_pkg_data_filename'</span><span class="p">,</span> <span class="s">'get_pkg_data_contents'</span><span class="p">,</span>
<span class="s">'get_pkg_data_fileobjs'</span><span class="p">,</span> <span class="s">'get_pkg_data_filenames'</span><span class="p">,</span> <span class="s">'compute_hash'</span><span class="p">,</span>
<span class="s">'clear_download_cache'</span><span class="p">,</span> <span class="s">'CacheMissingWarning'</span><span class="p">,</span>
<span class="s">'get_free_space_in_dir'</span><span class="p">,</span> <span class="s">'check_free_space_in_dir'</span><span class="p">,</span> <span class="s">'download_file'</span><span class="p">,</span>
<span class="s">'download_files_in_parallel'</span><span class="p">]</span>
<span class="n">DATAURL</span> <span class="o">=</span> <span class="n">ConfigurationItem</span><span class="p">(</span>
<span class="s">'dataurl'</span><span class="p">,</span> <span class="s">'http://data.astropy.org/'</span><span class="p">,</span> <span class="s">'URL for astropy remote data site.'</span><span class="p">)</span>
<span class="n">REMOTE_TIMEOUT</span> <span class="o">=</span> <span class="n">ConfigurationItem</span><span class="p">(</span>
<span class="s">'remote_timeout'</span><span class="p">,</span> <span class="mf">3.</span><span class="p">,</span> <span class="s">'Time to wait for remote data query (in seconds).'</span><span class="p">)</span>
<span class="n">COMPUTE_HASH_BLOCK_SIZE</span> <span class="o">=</span> <span class="n">ConfigurationItem</span><span class="p">(</span>
<span class="s">'hash_block_size'</span><span class="p">,</span> <span class="mi">2</span> <span class="o">**</span> <span class="mi">16</span><span class="p">,</span> <span class="c"># 64K</span>
<span class="s">'Block size for computing MD5 file hashes.'</span><span class="p">)</span>
<span class="n">DOWNLOAD_CACHE_BLOCK_SIZE</span> <span class="o">=</span> <span class="n">ConfigurationItem</span><span class="p">(</span>
<span class="s">'download_block_size'</span><span class="p">,</span> <span class="mi">2</span> <span class="o">**</span> <span class="mi">16</span><span class="p">,</span> <span class="c"># 64K</span>
<span class="s">'Number of bytes of remote data to download per step.'</span><span class="p">)</span>
<span class="n">DOWNLOAD_CACHE_LOCK_ATTEMPTS</span> <span class="o">=</span> <span class="n">ConfigurationItem</span><span class="p">(</span>
<span class="s">'download_cache_lock_attempts'</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="s">'Number of times to try to get the lock '</span> <span class="o">+</span>
<span class="s">'while accessing the data cache before giving up.'</span><span class="p">)</span>
<span class="n">DELETE_TEMPORARY_DOWNLOADS_AT_EXIT</span> <span class="o">=</span> <span class="n">ConfigurationItem</span><span class="p">(</span>
<span class="s">'delete_temporary_downloads_at_exit'</span><span class="p">,</span> <span class="bp">True</span><span class="p">,</span> <span class="s">'If True, temporary download'</span> <span class="o">+</span>
<span class="s">' files created when the cache is inacessible will be deleted at the end'</span> <span class="o">+</span>
<span class="s">' of the python session.'</span><span class="p">)</span>
<span class="n">PY3K</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">>=</span> <span class="mi">3</span>
<div class="viewcode-block" id="CacheMissingWarning"><a class="viewcode-back" href="../../../_generated/astropy.utils.data.CacheMissingWarning.html#astropy.utils.data.CacheMissingWarning">[docs]</a><span class="k">class</span> <span class="nc">CacheMissingWarning</span><span class="p">(</span><span class="ne">Warning</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> This warning indicates the standard cache directory is not accessible, with</span>
<span class="sd"> the first argument providing the warning message. If args[1] is present, it</span>
<span class="sd"> is a filename indicating the path to a temporary file that was created to</span>
<span class="sd"> store a remote data download in the absence of the cache.</span>
<span class="sd"> """</span>
</div>
<span class="k">def</span> <span class="nf">_is_url</span><span class="p">(</span><span class="n">string</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Test whether a string is a valid URL</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> string : str</span>
<span class="sd"> The string to test</span>
<span class="sd"> """</span>
<span class="kn">from</span> <span class="nn">urlparse</span> <span class="kn">import</span> <span class="n">urlparse</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">string</span><span class="p">)</span>
<span class="c"># url[0]==url.scheme, but url[0] is py 2.6-compat</span>
<span class="c"># we can't just check that url[0] is not an empty string, because</span>
<span class="c"># file paths in windows would return a non-empty scheme (e.g. e:\\</span>
<span class="c"># returns 'e').</span>
<span class="k">return</span> <span class="n">url</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="ow">in</span> <span class="p">[</span><span class="s">'http'</span><span class="p">,</span> <span class="s">'https'</span><span class="p">,</span> <span class="s">'ftp'</span><span class="p">]</span>
<span class="k">def</span> <span class="nf">_is_inside</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">parent_path</span><span class="p">):</span>
<span class="c"># We have to try realpath too to avoid issues with symlinks, but we leave</span>
<span class="c"># abspath because some systems like debian have the absolute path (with no</span>
<span class="c"># symlinks followed) match, but the real directories in different</span>
<span class="c"># locations, so need to try both cases.</span>
<span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="n">path</span><span class="p">)</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="n">parent_path</span><span class="p">))</span> \
<span class="ow">or</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">realpath</span><span class="p">(</span><span class="n">path</span><span class="p">)</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">realpath</span><span class="p">(</span><span class="n">parent_path</span><span class="p">))</span>
<span class="nd">@contextlib.contextmanager</span>
<div class="viewcode-block" id="get_readable_fileobj"><a class="viewcode-back" href="../../../_generated/astropy.utils.data.get_readable_fileobj.html#astropy.utils.data.get_readable_fileobj">[docs]</a><span class="k">def</span> <span class="nf">get_readable_fileobj</span><span class="p">(</span><span class="n">name_or_obj</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="bp">False</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Given a filename or a readable file-like object, return a context</span>
<span class="sd"> manager that yields a readable file-like object.</span>
<span class="sd"> This supports passing filenames, URLs, and readable file-like</span>
<span class="sd"> objects, any of which can be compressed in gzip or bzip2.</span>
<span class="sd"> Notes</span>
<span class="sd"> -----</span>
<span class="sd"> This function is a context manager, and should be used for example</span>
<span class="sd"> as::</span>
<span class="sd"> with get_readable_fileobj('file.dat') as f:</span>
<span class="sd"> contents = f.read()</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> name_or_obj : str or file-like object</span>
<span class="sd"> The filename of the file to access (if given as a string), or</span>
<span class="sd"> the file-like object to access.</span>
<span class="sd"> If a file-like object, it must be opened in binary mode.</span>
<span class="sd"> encoding : str, optional</span>
<span class="sd"> When `None` (default), returns a file-like object with a</span>
<span class="sd"> `read` method that on Python 2.x returns `bytes` objects and</span>
<span class="sd"> on Python 3.x returns `str` (`unicode`) objects, using</span>
<span class="sd"> `locale.getpreferredencoding()` as an encoding. This matches</span>
<span class="sd"> the default behavior of the built-in `open` when no `mode`</span>
<span class="sd"> argument is provided.</span>
<span class="sd"> When `'binary'`, returns a file-like object where its `read`</span>
<span class="sd"> method returns `bytes` objects.</span>
<span class="sd"> When another string, it is the name of an encoding, and the</span>
<span class="sd"> file-like object's `read` method will return `str` (`unicode`)</span>
<span class="sd"> objects, decoded from binary using the given encoding.</span>
<span class="sd"> cache : bool, optional</span>
<span class="sd"> Whether to cache the contents of remote URLs.</span>
<span class="sd"> """</span>
<span class="kn">import</span> <span class="nn">tempfile</span>
<span class="c"># close_fds is a list of file handles created by this function</span>
<span class="c"># that need to be closed. We don't want to always just close the</span>
<span class="c"># returned file handle, because it may simply be the file handle</span>
<span class="c"># passed in. In that case it is not the responsibility of this</span>
<span class="c"># function to close it: doing so could result in a "double close"</span>
<span class="c"># and an "invalid file descriptor" exception.</span>
<span class="n">close_fds</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">delete_fds</span> <span class="o">=</span> <span class="p">[]</span>
<span class="c"># Get a file object to the content</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">name_or_obj</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">):</span>
<span class="k">if</span> <span class="n">_is_url</span><span class="p">(</span><span class="n">name_or_obj</span><span class="p">):</span>
<span class="n">name_or_obj</span> <span class="o">=</span> <span class="n">download_file</span><span class="p">(</span><span class="n">name_or_obj</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>
<span class="k">if</span> <span class="n">PY3K</span><span class="p">:</span>
<span class="n">fileobj</span> <span class="o">=</span> <span class="n">io</span><span class="o">.</span><span class="n">FileIO</span><span class="p">(</span><span class="n">name_or_obj</span><span class="p">,</span> <span class="s">'r'</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">fileobj</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">name_or_obj</span><span class="p">,</span> <span class="s">'rb'</span><span class="p">)</span>
<span class="n">close_fds</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">fileobj</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">fileobj</span> <span class="o">=</span> <span class="n">name_or_obj</span>
<span class="c"># Check if the file object supports random access, and if not,</span>
<span class="c"># then wrap it in a BytesIO buffer. It would be nicer to use a</span>
<span class="c"># BufferedReader to avoid reading loading the whole file first,</span>
<span class="c"># but that is not compatible with streams or urllib2.urlopen</span>
<span class="c"># objects on Python 2.x.</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">fileobj</span><span class="p">,</span> <span class="s">'seek'</span><span class="p">):</span>
<span class="n">fileobj</span> <span class="o">=</span> <span class="n">io</span><span class="o">.</span><span class="n">BytesIO</span><span class="p">(</span><span class="n">fileobj</span><span class="o">.</span><span class="n">read</span><span class="p">())</span>
<span class="c"># Now read enough bytes to look at signature</span>
<span class="n">signature</span> <span class="o">=</span> <span class="n">fileobj</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="mi">4</span><span class="p">)</span>
<span class="n">fileobj</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
<span class="k">if</span> <span class="n">signature</span><span class="p">[:</span><span class="mi">3</span><span class="p">]</span> <span class="o">==</span> <span class="n">b</span><span class="s">'</span><span class="se">\x1f\x8b\x08</span><span class="s">'</span><span class="p">:</span> <span class="c"># gzip</span>
<span class="kn">import</span> <span class="nn">struct</span>
<span class="k">try</span><span class="p">:</span>
<span class="kn">from</span> <span class="nn">.compat</span> <span class="kn">import</span> <span class="n">gzip</span>
<span class="n">fileobj_new</span> <span class="o">=</span> <span class="n">gzip</span><span class="o">.</span><span class="n">GzipFile</span><span class="p">(</span><span class="n">fileobj</span><span class="o">=</span><span class="n">fileobj</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s">'rb'</span><span class="p">)</span>
<span class="n">fileobj_new</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span> <span class="c"># need to check that the file is really gzip</span>
<span class="k">except</span> <span class="p">(</span><span class="ne">IOError</span><span class="p">,</span> <span class="ne">EOFError</span><span class="p">):</span> <span class="c"># invalid gzip file</span>
<span class="n">fileobj</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
<span class="n">fileobj_new</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
<span class="k">except</span> <span class="n">struct</span><span class="o">.</span><span class="n">error</span><span class="p">:</span> <span class="c"># invalid gzip file on Python 3</span>
<span class="n">fileobj</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
<span class="n">fileobj_new</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">fileobj_new</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
<span class="n">fileobj</span> <span class="o">=</span> <span class="n">fileobj_new</span>
<span class="k">elif</span> <span class="n">signature</span><span class="p">[:</span><span class="mi">3</span><span class="p">]</span> <span class="o">==</span> <span class="n">b</span><span class="s">'BZh'</span><span class="p">:</span> <span class="c"># bzip2</span>
<span class="k">try</span><span class="p">:</span>
<span class="c"># bz2.BZ2File does not support file objects, only filenames, so we</span>
<span class="c"># need to write the data to a temporary file</span>
<span class="n">tmp</span> <span class="o">=</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="s">"wb"</span><span class="p">,</span> <span class="n">delete</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span>
<span class="n">tmp</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">fileobj</span><span class="o">.</span><span class="n">read</span><span class="p">())</span>
<span class="n">tmp</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
<span class="n">delete_fds</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">tmp</span><span class="p">)</span>
<span class="kn">import</span> <span class="nn">bz2</span>
<span class="n">fileobj_new</span> <span class="o">=</span> <span class="n">bz2</span><span class="o">.</span><span class="n">BZ2File</span><span class="p">(</span><span class="n">tmp</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s">'rb'</span><span class="p">)</span>
<span class="n">fileobj_new</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span> <span class="c"># need to check that the file is really bzip2</span>
<span class="k">except</span> <span class="ne">IOError</span><span class="p">:</span> <span class="c"># invalid bzip2 file</span>
<span class="n">fileobj</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
<span class="n">fileobj_new</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">fileobj_new</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
<span class="n">close_fds</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">fileobj_new</span><span class="p">)</span>
<span class="n">fileobj</span> <span class="o">=</span> <span class="n">fileobj_new</span>
<span class="c"># By this point, we have a file, io.FileIO, gzip.GzipFile, or</span>
<span class="c"># bz2.BZ2File instance opened in binary mode (that is, read</span>
<span class="c"># returns bytes). Now we need to, if requested, wrap it in a</span>
<span class="c"># io.TextIOWrapper so read will return unicode based on the</span>
<span class="c"># encoding parameter.</span>
<span class="k">if</span> <span class="n">PY3K</span><span class="p">:</span>
<span class="n">needs_textio_wrapper</span> <span class="o">=</span> <span class="n">encoding</span> <span class="o">!=</span> <span class="s">'binary'</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">needs_textio_wrapper</span> <span class="o">=</span> <span class="n">encoding</span> <span class="o">!=</span> <span class="s">'binary'</span> <span class="ow">and</span> <span class="n">encoding</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span>
<span class="k">if</span> <span class="n">needs_textio_wrapper</span><span class="p">:</span>
<span class="c"># A bz2.BZ2File can not be wrapped by a TextIOWrapper,</span>
<span class="c"># so we decompress it to a temporary file and then</span>
<span class="c"># return a handle to that.</span>
<span class="kn">import</span> <span class="nn">bz2</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">fileobj</span><span class="p">,</span> <span class="n">bz2</span><span class="o">.</span><span class="n">BZ2File</span><span class="p">):</span>
<span class="n">tmp</span> <span class="o">=</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="s">"wb"</span><span class="p">,</span> <span class="n">delete</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">fileobj</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
<span class="n">tmp</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
<span class="n">tmp</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
<span class="n">delete_fds</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">tmp</span><span class="p">)</span>
<span class="k">if</span> <span class="n">PY3K</span><span class="p">:</span>
<span class="n">fileobj</span> <span class="o">=</span> <span class="n">io</span><span class="o">.</span><span class="n">FileIO</span><span class="p">(</span><span class="n">tmp</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="s">'r'</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">fileobj</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">tmp</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="s">'rb'</span><span class="p">)</span>
<span class="n">close_fds</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">fileobj</span><span class="p">)</span>
<span class="c"># On Python 2.x, we need to first wrap the regular `file`</span>
<span class="c"># instance in a `io.FileIO` object before it can be</span>
<span class="c"># wrapped in a `TextIOWrapper`. We don't just create an</span>
<span class="c"># `io.FileIO` object in the first place, because we can't</span>
<span class="c"># get a raw file descriptor out of it on Python 2.x, which</span>
<span class="c"># is required for the XML iterparser.</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">PY3K</span> <span class="ow">and</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">fileobj</span><span class="p">,</span> <span class="nb">file</span><span class="p">):</span>
<span class="n">fileobj</span> <span class="o">=</span> <span class="n">io</span><span class="o">.</span><span class="n">FileIO</span><span class="p">(</span><span class="n">fileobj</span><span class="o">.</span><span class="n">fileno</span><span class="p">())</span>
<span class="n">fileobj</span> <span class="o">=</span> <span class="n">io</span><span class="o">.</span><span class="n">BufferedReader</span><span class="p">(</span><span class="n">fileobj</span><span class="p">)</span>
<span class="n">fileobj</span> <span class="o">=</span> <span class="n">io</span><span class="o">.</span><span class="n">TextIOWrapper</span><span class="p">(</span><span class="n">fileobj</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="n">encoding</span><span class="p">)</span>
<span class="c"># Ensure that file is at the start - io.FileIO will for</span>
<span class="c"># example not always be at the start:</span>
<span class="c"># >>> import io</span>
<span class="c"># >>> f = open('test.fits', 'rb')</span>
<span class="c"># >>> f.read(4)</span>
<span class="c"># 'SIMP'</span>
<span class="c"># >>> f.seek(0)</span>
<span class="c"># >>> fileobj = io.FileIO(f.fileno())</span>
<span class="c"># >>> fileobj.tell()</span>
<span class="c"># 4096L</span>
<span class="n">fileobj</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">yield</span> <span class="n">fileobj</span>
<span class="k">finally</span><span class="p">:</span>
<span class="k">for</span> <span class="n">fd</span> <span class="ow">in</span> <span class="n">close_fds</span><span class="p">:</span>
<span class="n">fd</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
<span class="k">for</span> <span class="n">fd</span> <span class="ow">in</span> <span class="n">delete_fds</span><span class="p">:</span>
<span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">fd</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="get_file_contents"><a class="viewcode-back" href="../../../_generated/astropy.utils.data.get_file_contents.html#astropy.utils.data.get_file_contents">[docs]</a><span class="k">def</span> <span class="nf">get_file_contents</span><span class="p">(</span><span class="n">name_or_obj</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="bp">False</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Retrieves the contents of a filename or file-like object.</span>
<span class="sd"> See the `get_readable_fileobj` docstring for details on parameters.</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> content</span>
<span class="sd"> The content of the file (as requested by `encoding`).</span>
<span class="sd"> """</span>
<span class="k">with</span> <span class="n">get_readable_fileobj</span><span class="p">(</span><span class="n">name_or_obj</span><span class="p">,</span> <span class="n">encoding</span><span class="p">,</span> <span class="n">cache</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
<span class="k">return</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
</div>
<div class="viewcode-block" id="get_pkg_data_fileobj"><a class="viewcode-back" href="../../../_generated/astropy.utils.data.get_pkg_data_fileobj.html#astropy.utils.data.get_pkg_data_fileobj">[docs]</a><span class="k">def</span> <span class="nf">get_pkg_data_fileobj</span><span class="p">(</span><span class="n">data_name</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="bp">True</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Retrieves a data file from the standard locations for the package and</span>
<span class="sd"> provides the file as a file-like object that reads bytes.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> data_name : str</span>
<span class="sd"> Name/location of the desired data file. One of the following:</span>
<span class="sd"> * The name of a data file included in the source</span>
<span class="sd"> distribution. The path is relative to the module</span>
<span class="sd"> calling this function. For example, if calling from</span>
<span class="sd"> `astropy.pkname`, use ``'data/file.dat'`` to get the</span>
<span class="sd"> file in ``astropy/pkgname/data/file.dat``. Double-dots</span>
<span class="sd"> can be used to go up a level. In the same example, use</span>
<span class="sd"> ``'../data/file.dat'`` to get ``astropy/data/file.dat``.</span>
<span class="sd"> * If a matching local file does not exist, the Astropy</span>
<span class="sd"> data server will be queried for the file.</span>
<span class="sd"> * A hash like that produced by `compute_hash` can be</span>
<span class="sd"> requested, prefixed by 'hash/'</span>
<span class="sd"> e.g. 'hash/395dd6493cc584df1e78b474fb150840'. The hash</span>
<span class="sd"> will first be searched for locally, and if not found,</span>
<span class="sd"> the Astropy data server will be queried.</span>
<span class="sd"> encoding : str, optional</span>
<span class="sd"> When `None` (default), returns a file-like object with a</span>
<span class="sd"> `read` method that on Python 2.x returns `bytes` objects and</span>
<span class="sd"> on Python 3.x returns `str` (`unicode`) objects, using</span>
<span class="sd"> `locale.getpreferredencoding()` as an encoding. This matches</span>
<span class="sd"> the default behavior of the built-in `open` when no `mode`</span>
<span class="sd"> argument is provided.</span>
<span class="sd"> When `'binary'`, returns a file-like object where its `read`</span>
<span class="sd"> method returns `bytes` objects.</span>
<span class="sd"> When another string, it is the name of an encoding, and the</span>
<span class="sd"> file-like object's `read` method will return `str` (`unicode`)</span>
<span class="sd"> objects, decoded from binary using the given encoding.</span>
<span class="sd"> cache : bool</span>
<span class="sd"> If True, the file will be downloaded and saved locally or the</span>
<span class="sd"> already-cached local copy will be accessed. If False, the file-like</span>
<span class="sd"> object will directly access the resource (e.g. if a remote URL is</span>
<span class="sd"> accessed, an object like that from `urllib2.urlopen` is returned).</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> fileobj : file-like</span>
<span class="sd"> An object with the contents of the data file available via</span>
<span class="sd"> :func:`read`. Can be used as part of a ``with`` statement,</span>
<span class="sd"> automatically closing itself after the ``with`` block.</span>
<span class="sd"> Raises</span>
<span class="sd"> ------</span>
<span class="sd"> urllib2.URLError</span>
<span class="sd"> If a remote file cannot be found.</span>
<span class="sd"> IOError</span>
<span class="sd"> If problems occur writing or reading a local file.</span>
<span class="sd"> Examples</span>
<span class="sd"> --------</span>
<span class="sd"> This will retrieve a data file and its contents for the `astropy.wcs`</span>
<span class="sd"> tests::</span>
<span class="sd"> from astropy.utils.data import get_pkg_data_fileobj</span>
<span class="sd"> with get_pkg_data_fileobj('data/3d_cd.hdr') as fobj:</span>
<span class="sd"> fcontents = fobj.read()</span>
<span class="sd"> This would download a data file from the astropy data server</span>
<span class="sd"> because the ``standards/vega.fits`` file is not present in the</span>
<span class="sd"> source distribution. It will also save the file locally so the</span>
<span class="sd"> next time it is accessed it won't need to be downloaded.::</span>
<span class="sd"> from astropy.utils.data import get_pkg_data_fileobj</span>
<span class="sd"> with get_pkg_data_fileobj('standards/vega.fits') as fobj:</span>
<span class="sd"> fcontents = fobj.read()</span>
<span class="sd"> This does the same thing but does *not* cache it locally::</span>
<span class="sd"> with get_pkg_data_fileobj('standards/vega.fits', cache=False) as fobj:</span>
<span class="sd"> fcontents = fobj.read()</span>
<span class="sd"> See Also</span>
<span class="sd"> --------</span>
<span class="sd"> get_pkg_data_contents : returns the contents of a file or url as a bytes object</span>
<span class="sd"> get_pkg_data_filename : returns a local name for a file containing the data</span>
<span class="sd"> """</span>
<span class="n">datafn</span> <span class="o">=</span> <span class="n">_find_pkg_data_path</span><span class="p">(</span><span class="n">data_name</span><span class="p">)</span>
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isdir</span><span class="p">(</span><span class="n">datafn</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">IOError</span><span class="p">(</span><span class="s">"Tried to access a data file that's actually "</span>
<span class="s">"a package data directory"</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">datafn</span><span class="p">):</span> <span class="c"># local file</span>
<span class="k">return</span> <span class="n">get_readable_fileobj</span><span class="p">(</span><span class="n">datafn</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="n">encoding</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span> <span class="c"># remote file</span>
<span class="k">return</span> <span class="n">get_readable_fileobj</span><span class="p">(</span><span class="n">DATAURL</span><span class="p">()</span> <span class="o">+</span> <span class="n">datafn</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="n">encoding</span><span class="p">,</span>
<span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="get_pkg_data_filename"><a class="viewcode-back" href="../../../_generated/astropy.utils.data.get_pkg_data_filename.html#astropy.utils.data.get_pkg_data_filename">[docs]</a><span class="k">def</span> <span class="nf">get_pkg_data_filename</span><span class="p">(</span><span class="n">data_name</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Retrieves a data file from the standard locations for the package and</span>
<span class="sd"> provides a local filename for the data.</span>
<span class="sd"> This function is similar to `get_pkg_data_fileobj` but returns the</span>
<span class="sd"> file *name* instead of a readable file-like object. This means</span>
<span class="sd"> that this function must always cache remote files locally, unlike</span>
<span class="sd"> `get_pkg_data_fileobj`.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> data_name : str</span>
<span class="sd"> Name/location of the desired data file. One of the following:</span>
<span class="sd"> * The name of a data file included in the source</span>
<span class="sd"> distribution. The path is relative to the module</span>
<span class="sd"> calling this function. For example, if calling from</span>
<span class="sd"> `astropy.pkname`, use ``'data/file.dat'`` to get the</span>
<span class="sd"> file in ``astropy/pkgname/data/file.dat``. Double-dots</span>
<span class="sd"> can be used to go up a level. In the same example, use</span>
<span class="sd"> ``'../data/file.dat'`` to get ``astropy/data/file.dat``.</span>
<span class="sd"> * If a matching local file does not exist, the Astropy</span>
<span class="sd"> data server will be queried for the file.</span>
<span class="sd"> * A hash like that produced by `compute_hash` can be</span>
<span class="sd"> requested, prefixed by 'hash/'</span>
<span class="sd"> e.g. 'hash/395dd6493cc584df1e78b474fb150840'. The hash</span>
<span class="sd"> will first be searched for locally, and if not found,</span>
<span class="sd"> the Astropy data server will be queried.</span>
<span class="sd"> Raises</span>
<span class="sd"> ------</span>
<span class="sd"> urllib2.URLError</span>
<span class="sd"> If a remote file cannot be found.</span>
<span class="sd"> IOError</span>
<span class="sd"> If problems occur writing or reading a local file.</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> filename : str</span>
<span class="sd"> A file path on the local file system corresponding to the data</span>
<span class="sd"> requested in `data_name`.</span>
<span class="sd"> Examples</span>
<span class="sd"> --------</span>
<span class="sd"> This will retrieve the contents of the data file for the `astropy.wcs`</span>
<span class="sd"> tests::</span>
<span class="sd"> from astropy.utils.data import get_pkg_data_filename</span>
<span class="sd"> fn = get_pkg_data_filename('data/3d_cd.hdr')</span>
<span class="sd"> with open(fn) as f:</span>
<span class="sd"> fcontents = f.read()</span>
<span class="sd"> This retrieves a data file by hash either locally or from the astropy data</span>
<span class="sd"> server::</span>
<span class="sd"> from astropy.utils.data import get_pkg_data_filename</span>
<span class="sd"> fn = get_pkg_data_filename('hash/da34a7b07ef153eede67387bf950bb32')</span>
<span class="sd"> with open(fn) as f:</span>
<span class="sd"> fcontents = f.read()</span>
<span class="sd"> See Also</span>
<span class="sd"> --------</span>
<span class="sd"> get_pkg_data_contents : returns the contents of a file or url as a bytes object</span>
<span class="sd"> get_pkg_data_fileobj : returns a file-like object with the data</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="n">data_name</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s">'hash/'</span><span class="p">):</span>
<span class="c"># first try looking for a local version if a hash is specified</span>
<span class="n">hashfn</span> <span class="o">=</span> <span class="n">_find_hash_fn</span><span class="p">(</span><span class="n">data_name</span><span class="p">[</span><span class="mi">5</span><span class="p">:])</span>
<span class="k">if</span> <span class="n">hashfn</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">download_file</span><span class="p">(</span><span class="n">DATAURL</span><span class="p">()</span> <span class="o">+</span> <span class="n">data_name</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="n">hashfn</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">datafn</span> <span class="o">=</span> <span class="n">_find_pkg_data_path</span><span class="p">(</span><span class="n">data_name</span><span class="p">)</span>
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isdir</span><span class="p">(</span><span class="n">datafn</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">IOError</span><span class="p">(</span><span class="s">"Tried to access a data file that's actually "</span>
<span class="s">"a package data directory"</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">datafn</span><span class="p">):</span> <span class="c"># local file</span>
<span class="k">return</span> <span class="n">datafn</span>
<span class="k">else</span><span class="p">:</span> <span class="c"># remote file</span>
<span class="k">return</span> <span class="n">download_file</span><span class="p">(</span><span class="n">DATAURL</span><span class="p">()</span> <span class="o">+</span> <span class="n">data_name</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="get_pkg_data_contents"><a class="viewcode-back" href="../../../_generated/astropy.utils.data.get_pkg_data_contents.html#astropy.utils.data.get_pkg_data_contents">[docs]</a><span class="k">def</span> <span class="nf">get_pkg_data_contents</span><span class="p">(</span><span class="n">data_name</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="bp">True</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Retrieves a data file from the standard locations and returns its</span>
<span class="sd"> contents as a bytes object.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> data_name : str</span>
<span class="sd"> Name/location of the desired data file. One of the following:</span>
<span class="sd"> * The name of a data file included in the source</span>
<span class="sd"> distribution. The path is relative to the module</span>
<span class="sd"> calling this function. For example, if calling from</span>
<span class="sd"> `astropy.pkname`, use ``'data/file.dat'`` to get the</span>
<span class="sd"> file in ``astropy/pkgname/data/file.dat``. Double-dots</span>
<span class="sd"> can be used to go up a level. In the same example, use</span>
<span class="sd"> ``'../data/file.dat'`` to get ``astropy/data/file.dat``.</span>
<span class="sd"> * If a matching local file does not exist, the Astropy</span>
<span class="sd"> data server will be queried for the file.</span>
<span class="sd"> * A hash like that produced by `compute_hash` can be</span>
<span class="sd"> requested, prefixed by 'hash/'</span>
<span class="sd"> e.g. 'hash/395dd6493cc584df1e78b474fb150840'. The hash</span>
<span class="sd"> will first be searched for locally, and if not found,</span>
<span class="sd"> the Astropy data server will be queried.</span>
<span class="sd"> * A URL to some other file.</span>
<span class="sd"> encoding : str, optional</span>
<span class="sd"> When `None` (default), returns a file-like object with a</span>
<span class="sd"> `read` method that on Python 2.x returns `bytes` objects and</span>
<span class="sd"> on Python 3.x returns `str` (`unicode`) objects, using</span>
<span class="sd"> `locale.getpreferredencoding()` as an encoding. This matches</span>
<span class="sd"> the default behavior of the built-in `open` when no `mode`</span>
<span class="sd"> argument is provided.</span>
<span class="sd"> When `'binary'`, returns a file-like object where its `read`</span>
<span class="sd"> method returns `bytes` objects.</span>
<span class="sd"> When another string, it is the name of an encoding, and the</span>
<span class="sd"> file-like object's `read` method will return `str` (`unicode`)</span>
<span class="sd"> objects, decoded from binary using the given encoding.</span>
<span class="sd"> cache : bool</span>
<span class="sd"> If True, the file will be downloaded and saved locally or the</span>
<span class="sd"> already-cached local copy will be accessed. If False, the file-like</span>
<span class="sd"> object will directly access the resource (e.g. if a remote URL is</span>
<span class="sd"> accessed, an object like that from `urllib2.urlopen` is returned).</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> contents : bytes</span>
<span class="sd"> The complete contents of the file as a bytes object.</span>
<span class="sd"> Raises</span>
<span class="sd"> ------</span>
<span class="sd"> urllib2.URLError</span>
<span class="sd"> If a remote file cannot be found.</span>
<span class="sd"> IOError</span>
<span class="sd"> If problems occur writing or reading a local file.</span>
<span class="sd"> See Also</span>
<span class="sd"> --------</span>
<span class="sd"> get_pkg_data_fileobj : returns a file-like object with the data</span>
<span class="sd"> get_pkg_data_filename : returns a local name for a file containing the data</span>
<span class="sd"> """</span>
<span class="k">with</span> <span class="n">get_pkg_data_fileobj</span><span class="p">(</span><span class="n">data_name</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="n">encoding</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span> <span class="k">as</span> <span class="n">fd</span><span class="p">:</span>
<span class="n">contents</span> <span class="o">=</span> <span class="n">fd</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
<span class="k">return</span> <span class="n">contents</span>
</div>
<div class="viewcode-block" id="get_pkg_data_filenames"><a class="viewcode-back" href="../../../_generated/astropy.utils.data.get_pkg_data_filenames.html#astropy.utils.data.get_pkg_data_filenames">[docs]</a><span class="k">def</span> <span class="nf">get_pkg_data_filenames</span><span class="p">(</span><span class="n">datadir</span><span class="p">,</span> <span class="n">pattern</span><span class="o">=</span><span class="s">'*'</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Returns the path of all of the data files in a given directory</span>
<span class="sd"> that match a given glob pattern.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> datadir : str</span>
<span class="sd"> Name/location of the desired data files. One of the following:</span>
<span class="sd"> * The name of a directory included in the source</span>
<span class="sd"> distribution. The path is relative to the module</span>
<span class="sd"> calling this function. For example, if calling from</span>
<span class="sd"> `astropy.pkname`, use ``'data'`` to get the</span>
<span class="sd"> files in ``astropy/pkgname/data``.</span>
<span class="sd"> * Remote URLs are not currently supported.</span>
<span class="sd"> pattern : str, optional</span>
<span class="sd"> A UNIX-style filename glob pattern to match files. See the</span>
<span class="sd"> `glob` module in the standard library for more information.</span>
<span class="sd"> By default, matches all files.</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> filenames : iterator of str</span>
<span class="sd"> Paths on the local filesystem in *datadir* matching *pattern*.</span>
<span class="sd"> Examples</span>
<span class="sd"> --------</span>
<span class="sd"> This will retrieve the contents of the data file for the `astropy.wcs`</span>
<span class="sd"> tests::</span>
<span class="sd"> from astropy.utils.data import get_pkg_data_filenames</span>
<span class="sd"> for fn in get_pkg_data_filename('maps', '*.hdr'):</span>
<span class="sd"> with open(fn) as f:</span>
<span class="sd"> fcontents = f.read()</span>
<span class="sd"> """</span>
<span class="kn">import</span> <span class="nn">fnmatch</span>
<span class="kn">from</span> <span class="nn">os.path</span> <span class="kn">import</span> <span class="n">isdir</span><span class="p">,</span> <span class="n">isfile</span><span class="p">,</span> <span class="n">join</span>
<span class="kn">from</span> <span class="nn">os</span> <span class="kn">import</span> <span class="n">listdir</span>
<span class="n">path</span> <span class="o">=</span> <span class="n">_find_pkg_data_path</span><span class="p">(</span><span class="n">datadir</span><span class="p">)</span>
<span class="k">if</span> <span class="n">isfile</span><span class="p">(</span><span class="n">path</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">IOError</span><span class="p">(</span>
<span class="s">"Tried to access a data directory that's actually "</span>
<span class="s">"a package data file"</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">isdir</span><span class="p">(</span><span class="n">path</span><span class="p">):</span>
<span class="k">for</span> <span class="n">filename</span> <span class="ow">in</span> <span class="n">listdir</span><span class="p">(</span><span class="n">path</span><span class="p">):</span>
<span class="k">if</span> <span class="n">fnmatch</span><span class="o">.</span><span class="n">fnmatch</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="n">pattern</span><span class="p">):</span>
<span class="k">yield</span> <span class="n">join</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">filename</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">IOError</span><span class="p">(</span><span class="s">"Path not found"</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="get_pkg_data_fileobjs"><a class="viewcode-back" href="../../../_generated/astropy.utils.data.get_pkg_data_fileobjs.html#astropy.utils.data.get_pkg_data_fileobjs">[docs]</a><span class="k">def</span> <span class="nf">get_pkg_data_fileobjs</span><span class="p">(</span><span class="n">datadir</span><span class="p">,</span> <span class="n">pattern</span><span class="o">=</span><span class="s">'*'</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Returns readable file objects for all of the data files in a given</span>
<span class="sd"> directory that match a given glob pattern.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> datadir : str</span>
<span class="sd"> Name/location of the desired data files. One of the following:</span>
<span class="sd"> * The name of a directory included in the source</span>
<span class="sd"> distribution. The path is relative to the module</span>
<span class="sd"> calling this function. For example, if calling from</span>
<span class="sd"> `astropy.pkname`, use ``'data'`` to get the</span>
<span class="sd"> files in ``astropy/pkgname/data``</span>
<span class="sd"> * Remote URLs are not currently supported</span>
<span class="sd"> pattern : str, optional</span>
<span class="sd"> A UNIX-style filename glob pattern to match files. See the</span>
<span class="sd"> `glob` module in the standard library for more information.</span>
<span class="sd"> By default, matches all files.</span>
<span class="sd"> encoding : str, optional</span>
<span class="sd"> When `None` (default), returns a file-like object with a</span>
<span class="sd"> `read` method that on Python 2.x returns `bytes` objects and</span>
<span class="sd"> on Python 3.x returns `str` (`unicode`) objects, using</span>
<span class="sd"> `locale.getpreferredencoding()` as an encoding. This matches</span>
<span class="sd"> the default behavior of the built-in `open` when no `mode`</span>
<span class="sd"> argument is provided.</span>
<span class="sd"> When `'binary'`, returns a file-like object where its `read`</span>
<span class="sd"> method returns `bytes` objects.</span>
<span class="sd"> When another string, it is the name of an encoding, and the</span>
<span class="sd"> file-like object's `read` method will return `str` (`unicode`)</span>
<span class="sd"> objects, decoded from binary using the given encoding.</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> fileobjs : iterator of file objects</span>
<span class="sd"> File objects for each of the files on the local filesystem in</span>
<span class="sd"> *datadir* matching *pattern*.</span>
<span class="sd"> Examples</span>
<span class="sd"> --------</span>
<span class="sd"> This will retrieve the contents of the data file for the `astropy.wcs`</span>
<span class="sd"> tests::</span>
<span class="sd"> from astropy.utils.data import get_pkg_data_filenames</span>
<span class="sd"> for fd in get_pkg_data_filename('maps', '*.hdr'):</span>
<span class="sd"> fcontents = fd.read()</span>
<span class="sd"> """</span>
<span class="k">for</span> <span class="n">fn</span> <span class="ow">in</span> <span class="n">get_pkg_data_filenames</span><span class="p">(</span><span class="n">datadir</span><span class="p">,</span> <span class="n">pattern</span><span class="p">):</span>
<span class="k">with</span> <span class="n">get_pkg_data_fileobj</span><span class="p">(</span><span class="n">fn</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="n">encoding</span><span class="p">)</span> <span class="k">as</span> <span class="n">fd</span><span class="p">:</span>
<span class="k">yield</span> <span class="n">fd</span>
</div>
<div class="viewcode-block" id="compute_hash"><a class="viewcode-back" href="../../../_generated/astropy.utils.data.compute_hash.html#astropy.utils.data.compute_hash">[docs]</a><span class="k">def</span> <span class="nf">compute_hash</span><span class="p">(</span><span class="n">localfn</span><span class="p">):</span>
<span class="sd">""" Computes the MD5 hash for a file.</span>
<span class="sd"> The hash for a data file is used for looking up data files in a unique</span>
<span class="sd"> fashion. This is of particular use for tests; a test may require a</span>
<span class="sd"> particular version of a particular file, in which case it can be accessed</span>
<span class="sd"> via hash to get the appropriate version.</span>
<span class="sd"> Typically, if you wish to write a test that requires a particular data</span>
<span class="sd"> file, you will want to submit that file to the astropy data servers, and</span>
<span class="sd"> use e.g. ``get_pkg_data_filename('hash/a725fa6ba642587436612c2df0451956')``,</span>
<span class="sd"> but with the hash for your file in place of the hash in the example.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> localfn : str</span>
<span class="sd"> The path to the file for which the hash should be generated.</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> md5hash : str</span>
<span class="sd"> The hex digest of the MD5 hash for the contents of the `localfn` file.</span>
<span class="sd"> """</span>
<span class="kn">import</span> <span class="nn">hashlib</span>
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">localfn</span><span class="p">,</span> <span class="s">'rb'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
<span class="n">h</span> <span class="o">=</span> <span class="n">hashlib</span><span class="o">.</span><span class="n">md5</span><span class="p">()</span>
<span class="n">block</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="n">COMPUTE_HASH_BLOCK_SIZE</span><span class="p">())</span>
<span class="k">while</span> <span class="n">block</span><span class="p">:</span>
<span class="n">h</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">block</span><span class="p">)</span>
<span class="n">block</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="n">COMPUTE_HASH_BLOCK_SIZE</span><span class="p">())</span>
<span class="k">return</span> <span class="n">h</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span>
</div>
<span class="k">def</span> <span class="nf">_find_pkg_data_path</span><span class="p">(</span><span class="n">data_name</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Look for data in the source-included data directories and return the</span>
<span class="sd"> path.</span>
<span class="sd"> """</span>
<span class="kn">from</span> <span class="nn">os.path</span> <span class="kn">import</span> <span class="n">dirname</span><span class="p">,</span> <span class="n">join</span>
<span class="kn">from</span> <span class="nn">..utils.misc</span> <span class="kn">import</span> <span class="n">find_current_module</span>
<span class="n">module</span> <span class="o">=</span> <span class="n">find_current_module</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="bp">True</span><span class="p">)</span>
<span class="k">if</span> <span class="n">module</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="c"># not called from inside an astropy package. So just pass name through</span>
<span class="k">return</span> <span class="n">data_name</span>
<span class="n">rootpkgname</span> <span class="o">=</span> <span class="n">module</span><span class="o">.</span><span class="n">__package__</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s">'.'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">rootpkg</span> <span class="o">=</span> <span class="nb">__import__</span><span class="p">(</span><span class="n">rootpkgname</span><span class="p">)</span>
<span class="n">module_path</span> <span class="o">=</span> <span class="n">dirname</span><span class="p">(</span><span class="n">module</span><span class="o">.</span><span class="n">__file__</span><span class="p">)</span>
<span class="n">path</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">module_path</span><span class="p">,</span> <span class="n">data_name</span><span class="p">)</span>
<span class="n">root_dir</span> <span class="o">=</span> <span class="n">dirname</span><span class="p">(</span><span class="n">rootpkg</span><span class="o">.</span><span class="n">__file__</span><span class="p">)</span>
<span class="k">assert</span> <span class="n">_is_inside</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">root_dir</span><span class="p">),</span> \
<span class="p">(</span><span class="s">"attempted to get a local data file outside "</span>
<span class="s">"of the "</span> <span class="o">+</span> <span class="n">rootpkgname</span> <span class="o">+</span> <span class="s">" tree"</span><span class="p">)</span>
<span class="k">return</span> <span class="n">path</span>
<span class="k">def</span> <span class="nf">_find_hash_fn</span><span class="p">(</span><span class="nb">hash</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Looks for a local file by hash - returns file name if found and a valid</span>
<span class="sd"> file, otherwise returns None.</span>
<span class="sd"> """</span>
<span class="kn">from</span> <span class="nn">os.path</span> <span class="kn">import</span> <span class="n">isfile</span><span class="p">,</span> <span class="n">join</span>
<span class="kn">from</span> <span class="nn">warnings</span> <span class="kn">import</span> <span class="n">warn</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">dldir</span><span class="p">,</span> <span class="n">urlmapfn</span> <span class="o">=</span> <span class="n">_get_download_cache_locs</span><span class="p">()</span>
<span class="k">except</span> <span class="p">(</span><span class="ne">IOError</span><span class="p">,</span> <span class="ne">OSError</span><span class="p">)</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="n">msg</span> <span class="o">=</span> <span class="s">'Could not access cache directory to search for data file: '</span>
<span class="n">warn</span><span class="p">(</span><span class="n">CacheMissingWarning</span><span class="p">(</span><span class="n">msg</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)))</span>
<span class="k">return</span> <span class="bp">None</span>
<span class="n">hashfn</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">dldir</span><span class="p">,</span> <span class="nb">hash</span><span class="p">)</span>
<span class="k">if</span> <span class="n">isfile</span><span class="p">(</span><span class="n">hashfn</span><span class="p">):</span>
<span class="k">return</span> <span class="n">hashfn</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="bp">None</span>
<div class="viewcode-block" id="get_free_space_in_dir"><a class="viewcode-back" href="../../../_generated/astropy.utils.data.get_free_space_in_dir.html#astropy.utils.data.get_free_space_in_dir">[docs]</a><span class="k">def</span> <span class="nf">get_free_space_in_dir</span><span class="p">(</span><span class="n">path</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Given a path to a directory, returns the amount of free space (in</span>
<span class="sd"> bytes) on that filesystem.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> path : str</span>
<span class="sd"> The path to a directory</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> bytes : int</span>
<span class="sd"> The amount of free space on the partition that the directory</span>
<span class="sd"> is on.</span>
<span class="sd"> """</span>
<span class="n">stat</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">statvfs</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
<span class="k">return</span> <span class="n">stat</span><span class="o">.</span><span class="n">f_bavail</span> <span class="o">*</span> <span class="n">stat</span><span class="o">.</span><span class="n">f_frsize</span>
</div>
<div class="viewcode-block" id="check_free_space_in_dir"><a class="viewcode-back" href="../../../_generated/astropy.utils.data.check_free_space_in_dir.html#astropy.utils.data.check_free_space_in_dir">[docs]</a><span class="k">def</span> <span class="nf">check_free_space_in_dir</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">size</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Determines if a given directory has enough space to hold a file of</span>
<span class="sd"> a given size. Raises an IOError if the file would be too large.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> path : str</span>
<span class="sd"> The path to a directory</span>
<span class="sd"> size : int</span>
<span class="sd"> A proposed filesize (in bytes)</span>
<span class="sd"> Raises</span>
<span class="sd"> -------</span>
<span class="sd"> IOError : There is not enough room on the filesystem</span>
<span class="sd"> """</span>
<span class="kn">from</span> <span class="nn">..utils.console</span> <span class="kn">import</span> <span class="n">human_file_size</span>
<span class="n">space</span> <span class="o">=</span> <span class="n">get_free_space_in_dir</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
<span class="k">if</span> <span class="n">space</span> <span class="o"><</span> <span class="n">size</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">IOError</span><span class="p">(</span>
<span class="s">"Not enough free space in '{0}' "</span>
<span class="s">"to download a {1} file"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="n">path</span><span class="p">,</span> <span class="n">human_file_size</span><span class="p">(</span><span class="n">size</span><span class="p">)))</span>
</div>
<div class="viewcode-block" id="download_file"><a class="viewcode-back" href="../../../_generated/astropy.utils.data.download_file.html#astropy.utils.data.download_file">[docs]</a><span class="k">def</span> <span class="nf">download_file</span><span class="p">(</span><span class="n">remote_url</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="bp">False</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Accepts a URL, downloads and optionally caches the result</span>
<span class="sd"> returning the filename, with a name determined by the file's MD5</span>
<span class="sd"> hash. If ``cache=True`` and the file is present in the cache, just</span>
<span class="sd"> returns the filename.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> remote_url : str</span>
<span class="sd"> The URL of the file to download</span>
<span class="sd"> cache : bool, optional</span>
<span class="sd"> Whether to use the cache</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> local_path : str</span>
<span class="sd"> Returns the local path that the file was download to.</span>
<span class="sd"> Raises</span>
<span class="sd"> ------</span>
<span class="sd"> `urllib2.URLError`</span>
<span class="sd"> Whenever there's a problem getting the remote file.</span>
<span class="sd"> """</span>
<span class="kn">import</span> <span class="nn">hashlib</span>
<span class="kn">import</span> <span class="nn">socket</span>
<span class="kn">from</span> <span class="nn">contextlib</span> <span class="kn">import</span> <span class="n">closing</span>
<span class="kn">from</span> <span class="nn">tempfile</span> <span class="kn">import</span> <span class="n">NamedTemporaryFile</span><span class="p">,</span> <span class="n">gettempdir</span>
<span class="kn">from</span> <span class="nn">shutil</span> <span class="kn">import</span> <span class="n">move</span>
<span class="kn">from</span> <span class="nn">warnings</span> <span class="kn">import</span> <span class="n">warn</span>
<span class="kn">from</span> <span class="nn">..utils.console</span> <span class="kn">import</span> <span class="n">ProgressBarOrSpinner</span>
<span class="n">missing_cache</span> <span class="o">=</span> <span class="bp">False</span>
<span class="k">if</span> <span class="n">cache</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">dldir</span><span class="p">,</span> <span class="n">urlmapfn</span> <span class="o">=</span> <span class="n">_get_download_cache_locs</span><span class="p">()</span>
<span class="k">except</span> <span class="p">(</span><span class="ne">IOError</span><span class="p">,</span> <span class="ne">OSError</span><span class="p">)</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="n">msg</span> <span class="o">=</span> <span class="s">'Remote data cache could not be accessed due to '</span>
<span class="n">estr</span> <span class="o">=</span> <span class="s">''</span> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="n">args</span><span class="p">)</span> <span class="o"><</span> <span class="mi">1</span> <span class="k">else</span> <span class="p">(</span><span class="s">': '</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
<span class="n">warn</span><span class="p">(</span><span class="n">CacheMissingWarning</span><span class="p">(</span><span class="n">msg</span> <span class="o">+</span> <span class="n">e</span><span class="o">.</span><span class="n">__class__</span><span class="o">.</span><span class="n">__name__</span> <span class="o">+</span> <span class="n">estr</span><span class="p">))</span>
<span class="n">cache</span> <span class="o">=</span> <span class="bp">False</span>
<span class="n">missing_cache</span> <span class="o">=</span> <span class="bp">True</span> <span class="c"># indicates that the cache is missing to raise a warning later</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">if</span> <span class="n">cache</span><span class="p">:</span>
<span class="c"># We don't need to acquire the lock here, since we are only reading</span>
<span class="k">with</span> <span class="n">_open_shelve</span><span class="p">(</span><span class="n">urlmapfn</span><span class="p">,</span> <span class="bp">True</span><span class="p">)</span> <span class="k">as</span> <span class="n">url2hash</span><span class="p">:</span>
<span class="k">if</span> <span class="nb">str</span><span class="p">(</span><span class="n">remote_url</span><span class="p">)</span> <span class="ow">in</span> <span class="n">url2hash</span><span class="p">:</span>
<span class="k">return</span> <span class="n">url2hash</span><span class="p">[</span><span class="nb">str</span><span class="p">(</span><span class="n">remote_url</span><span class="p">)]</span>
<span class="k">with</span> <span class="n">closing</span><span class="p">(</span><span class="n">urllib2</span><span class="o">.</span><span class="n">urlopen</span><span class="p">(</span><span class="n">remote_url</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">REMOTE_TIMEOUT</span><span class="p">()))</span> <span class="k">as</span> <span class="n">remote</span><span class="p">:</span>
<span class="c">#keep a hash to rename the local file to the hashed name</span>
<span class="nb">hash</span> <span class="o">=</span> <span class="n">hashlib</span><span class="o">.</span><span class="n">md5</span><span class="p">()</span>
<span class="n">info</span> <span class="o">=</span> <span class="n">remote</span><span class="o">.</span><span class="n">info</span><span class="p">()</span>
<span class="k">if</span> <span class="s">'Content-Length'</span> <span class="ow">in</span> <span class="n">info</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">size</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">info</span><span class="p">[</span><span class="s">'Content-Length'</span><span class="p">])</span>
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
<span class="n">size</span> <span class="o">=</span> <span class="bp">None</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">size</span> <span class="o">=</span> <span class="bp">None</span>
<span class="k">if</span> <span class="n">size</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
<span class="n">check_free_space_in_dir</span><span class="p">(</span><span class="n">gettempdir</span><span class="p">(),</span> <span class="n">size</span><span class="p">)</span>
<span class="k">if</span> <span class="n">cache</span><span class="p">:</span>
<span class="n">check_free_space_in_dir</span><span class="p">(</span><span class="n">dldir</span><span class="p">,</span> <span class="n">size</span><span class="p">)</span>
<span class="n">dlmsg</span> <span class="o">=</span> <span class="s">"Downloading {0}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">remote_url</span><span class="p">)</span>
<span class="k">with</span> <span class="n">ProgressBarOrSpinner</span><span class="p">(</span><span class="n">size</span><span class="p">,</span> <span class="n">dlmsg</span><span class="p">)</span> <span class="k">as</span> <span class="n">p</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">with</span> <span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="n">delete</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
<span class="n">bytes_read</span> <span class="o">=</span> <span class="mi">0</span>
<span class="n">block</span> <span class="o">=</span> <span class="n">remote</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="n">DOWNLOAD_CACHE_BLOCK_SIZE</span><span class="p">())</span>
<span class="k">while</span> <span class="n">block</span><span class="p">:</span>
<span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">block</span><span class="p">)</span>
<span class="nb">hash</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">block</span><span class="p">)</span>
<span class="n">bytes_read</span> <span class="o">+=</span> <span class="nb">len</span><span class="p">(</span><span class="n">block</span><span class="p">)</span>
<span class="n">p</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">bytes_read</span><span class="p">)</span>
<span class="n">block</span> <span class="o">=</span> <span class="n">remote</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="n">DOWNLOAD_CACHE_BLOCK_SIZE</span><span class="p">())</span>
<span class="k">except</span><span class="p">:</span>
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="p">):</span>
<span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
<span class="k">raise</span>
<span class="k">if</span> <span class="n">cache</span><span class="p">:</span>
<span class="n">_acquire_download_cache_lock</span><span class="p">()</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">with</span> <span class="n">_open_shelve</span><span class="p">(</span><span class="n">urlmapfn</span><span class="p">,</span> <span class="bp">True</span><span class="p">)</span> <span class="k">as</span> <span class="n">url2hash</span><span class="p">:</span>
<span class="c"># We check now to see if another process has</span>
<span class="c"># inadvertently written the file underneath us</span>
<span class="c"># already</span>
<span class="k">if</span> <span class="nb">str</span><span class="p">(</span><span class="n">remote_url</span><span class="p">)</span> <span class="ow">in</span> <span class="n">url2hash</span><span class="p">:</span>
<span class="k">return</span> <span class="n">url2hash</span><span class="p">[</span><span class="nb">str</span><span class="p">(</span><span class="n">remote_url</span><span class="p">)]</span>
<span class="n">local_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">dldir</span><span class="p">,</span> <span class="nb">hash</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">())</span>
<span class="n">move</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">local_path</span><span class="p">)</span>
<span class="n">url2hash</span><span class="p">[</span><span class="nb">str</span><span class="p">(</span><span class="n">remote_url</span><span class="p">)]</span> <span class="o">=</span> <span class="n">local_path</span>
<span class="k">finally</span><span class="p">:</span>
<span class="n">_release_download_cache_lock</span><span class="p">()</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">local_path</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">name</span>
<span class="k">if</span> <span class="n">missing_cache</span><span class="p">:</span>
<span class="n">msg</span> <span class="o">=</span> <span class="p">(</span><span class="s">'File downloaded to temporary location due to problem '</span>
<span class="s">'with cache directory and will not be cached.'</span><span class="p">)</span>
<span class="n">warn</span><span class="p">(</span><span class="n">CacheMissingWarning</span><span class="p">(</span><span class="n">msg</span><span class="p">,</span> <span class="n">local_path</span><span class="p">))</span>
<span class="k">if</span> <span class="n">DELETE_TEMPORARY_DOWNLOADS_AT_EXIT</span><span class="p">():</span>
<span class="k">global</span> <span class="n">_tempfilestodel</span>
<span class="n">_tempfilestodel</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">local_path</span><span class="p">)</span>
<span class="k">except</span> <span class="n">urllib2</span><span class="o">.</span><span class="n">URLError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">e</span><span class="p">,</span> <span class="s">'reason'</span><span class="p">)</span> <span class="ow">and</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="n">reason</span><span class="p">,</span> <span class="s">'errno'</span><span class="p">)</span> <span class="ow">and</span> <span class="n">e</span><span class="o">.</span><span class="n">reason</span><span class="o">.</span><span class="n">errno</span> <span class="o">==</span> <span class="mi">8</span><span class="p">:</span>
<span class="n">e</span><span class="o">.</span><span class="n">reason</span><span class="o">.</span><span class="n">strerror</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">reason</span><span class="o">.</span><span class="n">strerror</span> <span class="o">+</span> <span class="s">'. requested URL: '</span> <span class="o">+</span> <span class="n">remote_url</span>
<span class="n">e</span><span class="o">.</span><span class="n">reason</span><span class="o">.</span><span class="n">args</span> <span class="o">=</span> <span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="n">reason</span><span class="o">.</span><span class="n">errno</span><span class="p">,</span> <span class="n">e</span><span class="o">.</span><span class="n">reason</span><span class="o">.</span><span class="n">strerror</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">e</span>
<span class="k">except</span> <span class="n">socket</span><span class="o">.</span><span class="n">timeout</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="c"># this isn't supposed to happen, but occasionally a socket.timeout gets</span>
<span class="c"># through. It's supposed to be caught in `urrlib2` and raised in this</span>
<span class="c"># way, but for some reason in mysterious circumstances it doesn't. So</span>
<span class="c"># we'll just re-raise it here instead</span>
<span class="k">raise</span> <span class="n">urllib2</span><span class="o">.</span><span class="n">URLError</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
<span class="k">return</span> <span class="n">local_path</span>
</div>
<span class="k">def</span> <span class="nf">_do_download_files_in_parallel</span><span class="p">(</span><span class="n">args</span><span class="p">):</span>
<span class="n">orig_stdout</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">stdout</span>
<span class="n">sys</span><span class="o">.</span><span class="n">stdout</span> <span class="o">=</span> <span class="n">io</span><span class="o">.</span><span class="n">BytesIO</span><span class="p">()</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">return</span> <span class="n">download_file</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">)</span>
<span class="k">finally</span><span class="p">:</span>
<span class="n">sys</span><span class="o">.</span><span class="n">stdout</span> <span class="o">=</span> <span class="n">orig_stdout</span>
<div class="viewcode-block" id="download_files_in_parallel"><a class="viewcode-back" href="../../../_generated/astropy.utils.data.download_files_in_parallel.html#astropy.utils.data.download_files_in_parallel">[docs]</a><span class="k">def</span> <span class="nf">download_files_in_parallel</span><span class="p">(</span><span class="n">urls</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="bp">False</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Downloads multiple files in parallel from the given URLs. Blocks until</span>
<span class="sd"> all files have downloaded. The result is a list of local file paths</span>
<span class="sd"> corresponding to the given urls.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> urls : list of str</span>
<span class="sd"> The URLs to retrieve.</span>
<span class="sd"> cache : bool, optional</span>
<span class="sd"> Whether to use the cache</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> paths : list of str</span>
<span class="sd"> The local file paths corresponding to the downloaded URLs.</span>
<span class="sd"> """</span>
<span class="kn">from</span> <span class="nn">.console</span> <span class="kn">import</span> <span class="n">ProgressBar</span>
<span class="c"># Combine duplicate URLs</span>
<span class="n">combined_urls</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="n">urls</span><span class="p">))</span>
<span class="n">combined_paths</span> <span class="o">=</span> <span class="n">ProgressBar</span><span class="o">.</span><span class="n">map</span><span class="p">(</span>
<span class="n">_do_download_files_in_parallel</span><span class="p">,</span>
<span class="p">[(</span><span class="n">x</span><span class="p">,</span> <span class="n">cache</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">combined_urls</span><span class="p">],</span>
<span class="n">multiprocess</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
<span class="n">paths</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">url</span> <span class="ow">in</span> <span class="n">urls</span><span class="p">:</span>
<span class="n">paths</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">combined_paths</span><span class="p">[</span><span class="n">combined_urls</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="n">url</span><span class="p">)])</span>
<span class="k">return</span> <span class="n">paths</span>
<span class="c"># This is used by download_file and _deltemps to determine the files to delete</span>
<span class="c"># when the interpreter exits</span></div>
<span class="n">_tempfilestodel</span> <span class="o">=</span> <span class="p">[]</span>
<span class="nd">@atexit.register</span>
<span class="k">def</span> <span class="nf">_deltemps</span><span class="p">():</span>
<span class="k">global</span> <span class="n">_tempfilestodel</span>
<span class="k">while</span> <span class="nb">len</span><span class="p">(</span><span class="n">_tempfilestodel</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
<span class="n">fn</span> <span class="o">=</span> <span class="n">_tempfilestodel</span><span class="o">.</span><span class="n">pop</span><span class="p">()</span>
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">fn</span><span class="p">):</span>
<span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">fn</span><span class="p">)</span>
<div class="viewcode-block" id="clear_download_cache"><a class="viewcode-back" href="../../../_generated/astropy.utils.data.clear_download_cache.html#astropy.utils.data.clear_download_cache">[docs]</a><span class="k">def</span> <span class="nf">clear_download_cache</span><span class="p">(</span><span class="n">hashorurl</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="sd">""" Clears the data file cache by deleting the local file(s).</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> hashorurl : str or None</span>
<span class="sd"> If None, the whole cache is cleared. Otherwise, either specifies a</span>
<span class="sd"> hash for the cached file that is supposed to be deleted, or a URL that</span>
<span class="sd"> has previously been downloaded to the cache.</span>
<span class="sd"> Raises</span>
<span class="sd"> ------</span>
<span class="sd"> OSEerror</span>
<span class="sd"> If the requested filename is not present in the data directory.</span>
<span class="sd"> """</span>
<span class="kn">from</span> <span class="nn">os</span> <span class="kn">import</span> <span class="n">unlink</span>
<span class="kn">from</span> <span class="nn">os.path</span> <span class="kn">import</span> <span class="n">join</span><span class="p">,</span> <span class="n">exists</span>
<span class="kn">from</span> <span class="nn">shutil</span> <span class="kn">import</span> <span class="n">rmtree</span>
<span class="kn">from</span> <span class="nn">warnings</span> <span class="kn">import</span> <span class="n">warn</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">dldir</span><span class="p">,</span> <span class="n">urlmapfn</span> <span class="o">=</span> <span class="n">_get_download_cache_locs</span><span class="p">()</span>
<span class="k">except</span> <span class="p">(</span><span class="ne">IOError</span><span class="p">,</span> <span class="ne">OSError</span><span class="p">)</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="n">msg</span> <span class="o">=</span> <span class="s">'Not clearing data cache - cache inacessable due to '</span>
<span class="n">estr</span> <span class="o">=</span> <span class="s">''</span> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="n">args</span><span class="p">)</span> <span class="o"><</span> <span class="mi">1</span> <span class="k">else</span> <span class="p">(</span><span class="s">': '</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
<span class="n">warn</span><span class="p">(</span><span class="n">CacheMissingWarning</span><span class="p">(</span><span class="n">msg</span> <span class="o">+</span> <span class="n">e</span><span class="o">.</span><span class="n">__class__</span><span class="o">.</span><span class="n">__name__</span> <span class="o">+</span> <span class="n">estr</span><span class="p">))</span>
<span class="k">return</span>
<span class="n">_acquire_download_cache_lock</span><span class="p">()</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">if</span> <span class="n">hashorurl</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="k">if</span> <span class="n">exists</span><span class="p">(</span><span class="n">dldir</span><span class="p">):</span>
<span class="n">rmtree</span><span class="p">(</span><span class="n">dldir</span><span class="p">)</span>
<span class="k">if</span> <span class="n">exists</span><span class="p">(</span><span class="n">urlmapfn</span><span class="p">):</span>
<span class="n">unlink</span><span class="p">(</span><span class="n">urlmapfn</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">with</span> <span class="n">_open_shelve</span><span class="p">(</span><span class="n">urlmapfn</span><span class="p">,</span> <span class="bp">True</span><span class="p">)</span> <span class="k">as</span> <span class="n">url2hash</span><span class="p">:</span>
<span class="n">filepath</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">dldir</span><span class="p">,</span> <span class="n">hashorurl</span><span class="p">)</span>
<span class="k">assert</span> <span class="n">_is_inside</span><span class="p">(</span><span class="n">filepath</span><span class="p">,</span> <span class="n">dldir</span><span class="p">),</span> \
<span class="p">(</span><span class="s">"attempted to use clear_download_cache on a location"</span> <span class="o">+</span>
<span class="s">" that's not inside the data cache directory"</span><span class="p">)</span>
<span class="k">if</span> <span class="n">exists</span><span class="p">(</span><span class="n">filepath</span><span class="p">):</span>
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">url2hash</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="k">if</span> <span class="n">v</span> <span class="o">==</span> <span class="n">filepath</span><span class="p">:</span>
<span class="k">del</span> <span class="n">url2hash</span><span class="p">[</span><span class="n">k</span><span class="p">]</span>
<span class="n">unlink</span><span class="p">(</span><span class="n">filepath</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">hashorurl</span> <span class="ow">in</span> <span class="n">url2hash</span><span class="p">:</span>
<span class="n">filepath</span> <span class="o">=</span> <span class="n">url2hash</span><span class="p">[</span><span class="n">hashorurl</span><span class="p">]</span>
<span class="k">del</span> <span class="n">url2hash</span><span class="p">[</span><span class="n">hashorurl</span><span class="p">]</span>
<span class="n">unlink</span><span class="p">(</span><span class="n">filepath</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">msg</span> <span class="o">=</span> <span class="s">'Could not find file or url {0}'</span>
<span class="k">raise</span> <span class="ne">OSError</span><span class="p">(</span><span class="n">msg</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hashorurl</span><span class="p">))</span>
<span class="k">finally</span><span class="p">:</span>
<span class="c"># the lock will be gone if rmtree was used above, but release otherwise</span>
<span class="k">if</span> <span class="n">exists</span><span class="p">(</span><span class="n">join</span><span class="p">(</span><span class="n">_get_download_cache_locs</span><span class="p">()[</span><span class="mi">0</span><span class="p">],</span> <span class="s">'lock'</span><span class="p">)):</span>
<span class="n">_release_download_cache_lock</span><span class="p">()</span>
</div>
<span class="k">def</span> <span class="nf">_get_download_cache_locs</span><span class="p">():</span>
<span class="sd">""" Finds the path to the data cache directory and makes them if</span>
<span class="sd"> they don't exist.</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> datadir : str</span>
<span class="sd"> The path to the data cache directory.</span>
<span class="sd"> shelveloc : str</span>
<span class="sd"> The path to the shelve object that stores the cache info.</span>
<span class="sd"> """</span>
<span class="kn">from</span> <span class="nn">..config.paths</span> <span class="kn">import</span> <span class="n">get_cache_dir</span>
<span class="kn">from</span> <span class="nn">os.path</span> <span class="kn">import</span> <span class="n">exists</span><span class="p">,</span> <span class="n">isdir</span><span class="p">,</span> <span class="n">join</span>
<span class="kn">from</span> <span class="nn">os</span> <span class="kn">import</span> <span class="n">mkdir</span>
<span class="n">datadir</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">get_cache_dir</span><span class="p">(),</span> <span class="s">'download'</span><span class="p">)</span>
<span class="n">shelveloc</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">get_cache_dir</span><span class="p">(),</span> <span class="s">'download_urlmap'</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">exists</span><span class="p">(</span><span class="n">datadir</span><span class="p">):</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">mkdir</span><span class="p">(</span><span class="n">datadir</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">OSError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">exists</span><span class="p">(</span><span class="n">datadir</span><span class="p">):</span>
<span class="k">raise</span>
<span class="k">elif</span> <span class="ow">not</span> <span class="n">isdir</span><span class="p">(</span><span class="n">datadir</span><span class="p">):</span>
<span class="n">msg</span> <span class="o">=</span> <span class="s">'Data cache directory {0} is not a directory'</span>
<span class="k">raise</span> <span class="ne">IOError</span><span class="p">(</span><span class="n">msg</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">datadir</span><span class="p">))</span>
<span class="k">if</span> <span class="n">isdir</span><span class="p">(</span><span class="n">shelveloc</span><span class="p">):</span>
<span class="n">msg</span> <span class="o">=</span> <span class="s">'Data cache shelve object location {0} is a directory'</span>
<span class="k">raise</span> <span class="ne">IOError</span><span class="p">(</span><span class="n">msg</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">shelveloc</span><span class="p">))</span>
<span class="k">return</span> <span class="n">datadir</span><span class="p">,</span> <span class="n">shelveloc</span>
<span class="k">def</span> <span class="nf">_open_shelve</span><span class="p">(</span><span class="n">shelffn</span><span class="p">,</span> <span class="n">withclosing</span><span class="o">=</span><span class="bp">False</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> opens a shelf in a way that is py3.x and py2.x compatible. If</span>
<span class="sd"> `withclosing` is True, it will be opened with closing, allowing use like:</span>
<span class="sd"> with _open_shelve('somefile',True) as s:</span>
<span class="sd"> ...</span>
<span class="sd"> """</span>
<span class="kn">import</span> <span class="nn">shelve</span>
<span class="kn">from</span> <span class="nn">contextlib</span> <span class="kn">import</span> <span class="n">closing</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">PY3K</span><span class="p">:</span> <span class="c"># pragma: py3</span>
<span class="n">shelf</span> <span class="o">=</span> <span class="n">shelve</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">shelffn</span><span class="p">,</span> <span class="n">protocol</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span> <span class="c"># pragma: py2</span>
<span class="n">shelf</span> <span class="o">=</span> <span class="n">shelve</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">shelffn</span> <span class="o">+</span> <span class="s">'.db'</span><span class="p">,</span> <span class="n">protocol</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="k">if</span> <span class="n">withclosing</span><span class="p">:</span>
<span class="k">return</span> <span class="n">closing</span><span class="p">(</span><span class="n">shelf</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="n">shelf</span>
<span class="c">#the cache directory must be locked before any writes are performed. Same for</span>
<span class="c">#the hash shelve, so this should be used for both.</span>
<span class="k">def</span> <span class="nf">_acquire_download_cache_lock</span><span class="p">():</span>
<span class="sd">"""</span>
<span class="sd"> Uses the lock directory method. This is good because `mkdir` is</span>
<span class="sd"> atomic at the system call level, so it's thread-safe.</span>
<span class="sd"> """</span>
<span class="kn">from</span> <span class="nn">os.path</span> <span class="kn">import</span> <span class="n">join</span>
<span class="kn">from</span> <span class="nn">os</span> <span class="kn">import</span> <span class="n">mkdir</span>
<span class="kn">from</span> <span class="nn">time</span> <span class="kn">import</span> <span class="n">sleep</span>
<span class="n">lockdir</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">_get_download_cache_locs</span><span class="p">()[</span><span class="mi">0</span><span class="p">],</span> <span class="s">'lock'</span><span class="p">)</span>
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">DOWNLOAD_CACHE_LOCK_ATTEMPTS</span><span class="p">()):</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">mkdir</span><span class="p">(</span><span class="n">lockdir</span><span class="p">)</span>
<span class="c">#write the pid of this process for informational purposes</span>
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">join</span><span class="p">(</span><span class="n">lockdir</span><span class="p">,</span> <span class="s">'pid'</span><span class="p">),</span> <span class="s">'w'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
<span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">getpid</span><span class="p">()))</span>
<span class="k">except</span> <span class="ne">OSError</span><span class="p">:</span>
<span class="n">sleep</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span>
<span class="n">msg</span> <span class="o">=</span> <span class="s">'Unable to acquire lock for cache directory ({0} exists)'</span>
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="n">msg</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">lockdir</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">_release_download_cache_lock</span><span class="p">():</span>
<span class="kn">from</span> <span class="nn">os.path</span> <span class="kn">import</span> <span class="n">join</span><span class="p">,</span> <span class="n">exists</span><span class="p">,</span> <span class="n">isdir</span>
<span class="kn">from</span> <span class="nn">os</span> <span class="kn">import</span> <span class="n">rmdir</span>
<span class="n">lockdir</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">_get_download_cache_locs</span><span class="p">()[</span><span class="mi">0</span><span class="p">],</span> <span class="s">'lock'</span><span class="p">)</span>
<span class="k">if</span> <span class="n">isdir</span><span class="p">(</span><span class="n">lockdir</span><span class="p">):</span>
<span class="c">#if the pid file is present, be sure to remove it</span>
<span class="n">pidfn</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">lockdir</span><span class="p">,</span> <span class="s">'pid'</span><span class="p">)</span>
<span class="k">if</span> <span class="n">exists</span><span class="p">(</span><span class="n">pidfn</span><span class="p">):</span>
<span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">pidfn</span><span class="p">)</span>
<span class="n">rmdir</span><span class="p">(</span><span class="n">lockdir</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">msg</span> <span class="o">=</span> <span class="s">'Error releasing lock. "{0}" either does not exist or is not '</span> <span class="o">+</span>\
<span class="s">'a directory.'</span>
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="n">msg</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">lockdir</span><span class="p">))</span>
</pre></div>
</div>
</div>
</div>
<div class="sphinxsidebar">
<div class="sphinxsidebarwrapper"><h3>Page Contents</h3>
</div>
</div>
<div class="clearer"></div>
</div>
<footer class="footer">
<p class="pull-right">
<a href="#">Back to Top</a></p>
<p>
© Copyright 2011-2013, The Astropy Developers.<br/>
Created using <a href="http://sphinx.pocoo.org/">Sphinx</a> 1.1.3.
Last built 22 Oct 2013. <br/>
</p>
</footer>
</body>
</html>
