aboutsummaryrefslogtreecommitdiff
path: root/vim/bundle/YouCompleteMe/third_party/pythonfutures/crawl.py
diff options
context:
space:
mode:
authorKarel Kočí <cynerd@email.cz>2016-06-30 16:03:25 +0200
committerKarel Kočí <cynerd@email.cz>2016-06-30 16:03:25 +0200
commite573b3020c032400eed60b649a2cbf55266e6bb0 (patch)
tree8f572394ac8433529c7a8e70d160a2fbe8268b4e /vim/bundle/YouCompleteMe/third_party/pythonfutures/crawl.py
parentb8c667bd64b3edd38d56c63c5bd1db53a23b4499 (diff)
downloadmyconfigs-e573b3020c032400eed60b649a2cbf55266e6bb0.tar.gz
myconfigs-e573b3020c032400eed60b649a2cbf55266e6bb0.tar.bz2
myconfigs-e573b3020c032400eed60b649a2cbf55266e6bb0.zip
Add current configurations from old repository
Diffstat (limited to 'vim/bundle/YouCompleteMe/third_party/pythonfutures/crawl.py')
-rwxr-xr-xvim/bundle/YouCompleteMe/third_party/pythonfutures/crawl.py74
1 files changed, 74 insertions, 0 deletions
diff --git a/vim/bundle/YouCompleteMe/third_party/pythonfutures/crawl.py b/vim/bundle/YouCompleteMe/third_party/pythonfutures/crawl.py
new file mode 100755
index 0000000..86e0af7
--- /dev/null
+++ b/vim/bundle/YouCompleteMe/third_party/pythonfutures/crawl.py
@@ -0,0 +1,74 @@
+"""Compare the speed of downloading URLs sequentially vs. using futures."""
+
+import functools
+import time
+import timeit
+import sys
+
+try:
+ from urllib2 import urlopen
+except ImportError:
+ from urllib.request import urlopen
+
+from concurrent.futures import (as_completed, ThreadPoolExecutor,
+ ProcessPoolExecutor)
+
+URLS = ['http://www.google.com/',
+ 'http://www.apple.com/',
+ 'http://www.ibm.com',
+ 'http://www.thisurlprobablydoesnotexist.com',
+ 'http://www.slashdot.org/',
+ 'http://www.python.org/',
+ 'http://www.bing.com/',
+ 'http://www.facebook.com/',
+ 'http://www.yahoo.com/',
+ 'http://www.youtube.com/',
+ 'http://www.blogger.com/']
+
+def load_url(url, timeout):
+ kwargs = {'timeout': timeout} if sys.version_info >= (2, 6) else {}
+ return urlopen(url, **kwargs).read()
+
+def download_urls_sequential(urls, timeout=60):
+ url_to_content = {}
+ for url in urls:
+ try:
+ url_to_content[url] = load_url(url, timeout=timeout)
+ except:
+ pass
+ return url_to_content
+
+def download_urls_with_executor(urls, executor, timeout=60):
+ try:
+ url_to_content = {}
+ future_to_url = dict((executor.submit(load_url, url, timeout), url)
+ for url in urls)
+
+ for future in as_completed(future_to_url):
+ try:
+ url_to_content[future_to_url[future]] = future.result()
+ except:
+ pass
+ return url_to_content
+ finally:
+ executor.shutdown()
+
+def main():
+ for name, fn in [('sequential',
+ functools.partial(download_urls_sequential, URLS)),
+ ('processes',
+ functools.partial(download_urls_with_executor,
+ URLS,
+ ProcessPoolExecutor(10))),
+ ('threads',
+ functools.partial(download_urls_with_executor,
+ URLS,
+ ThreadPoolExecutor(10)))]:
+ sys.stdout.write('%s: ' % name.ljust(12))
+ start = time.time()
+ url_map = fn()
+ sys.stdout.write('%.2f seconds (%d of %d downloaded)\n' %
+ (time.time() - start, len(url_map), len(URLS)))
+
+if __name__ == '__main__':
+ main()