diff options
Diffstat (limited to 'vim/bundle/YouCompleteMe/third_party/pythonfutures/crawl.py')
m--------- | vim/bundle/YouCompleteMe | 0 | ||||
-rwxr-xr-x | vim/bundle/YouCompleteMe/third_party/pythonfutures/crawl.py | 74 |
2 files changed, 0 insertions, 74 deletions
diff --git a/vim/bundle/YouCompleteMe b/vim/bundle/YouCompleteMe new file mode 160000 +Subproject 0de1c0c9bb13ce82172b472c676035cd47cf6a6 diff --git a/vim/bundle/YouCompleteMe/third_party/pythonfutures/crawl.py b/vim/bundle/YouCompleteMe/third_party/pythonfutures/crawl.py deleted file mode 100755 index 86e0af7..0000000 --- a/vim/bundle/YouCompleteMe/third_party/pythonfutures/crawl.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Compare the speed of downloading URLs sequentially vs. using futures.""" - -import functools -import time -import timeit -import sys - -try: - from urllib2 import urlopen -except ImportError: - from urllib.request import urlopen - -from concurrent.futures import (as_completed, ThreadPoolExecutor, - ProcessPoolExecutor) - -URLS = ['http://www.google.com/', - 'http://www.apple.com/', - 'http://www.ibm.com', - 'http://www.thisurlprobablydoesnotexist.com', - 'http://www.slashdot.org/', - 'http://www.python.org/', - 'http://www.bing.com/', - 'http://www.facebook.com/', - 'http://www.yahoo.com/', - 'http://www.youtube.com/', - 'http://www.blogger.com/'] - -def load_url(url, timeout): - kwargs = {'timeout': timeout} if sys.version_info >= (2, 6) else {} - return urlopen(url, **kwargs).read() - -def download_urls_sequential(urls, timeout=60): - url_to_content = {} - for url in urls: - try: - url_to_content[url] = load_url(url, timeout=timeout) - except: - pass - return url_to_content - -def download_urls_with_executor(urls, executor, timeout=60): - try: - url_to_content = {} - future_to_url = dict((executor.submit(load_url, url, timeout), url) - for url in urls) - - for future in as_completed(future_to_url): - try: - url_to_content[future_to_url[future]] = future.result() - except: - pass - return url_to_content - finally: - executor.shutdown() - -def main(): - for name, fn in [('sequential', - functools.partial(download_urls_sequential, URLS)), - ('processes', - functools.partial(download_urls_with_executor, - URLS, - ProcessPoolExecutor(10))), - ('threads', - functools.partial(download_urls_with_executor, - URLS, - ThreadPoolExecutor(10)))]: - sys.stdout.write('%s: ' % name.ljust(12)) - start = time.time() - url_map = fn() - sys.stdout.write('%.2f seconds (%d of %d downloaded)\n' % - (time.time() - start, len(url_map), len(URLS))) - -if __name__ == '__main__': - main() |