|
@@ -2,7 +2,7 @@ import os
|
|
|
import logging
|
|
|
import time
|
|
|
import re
|
|
|
-import urlparse
|
|
|
+import urllib.parse
|
|
|
import string
|
|
|
import random
|
|
|
|
|
@@ -20,11 +20,15 @@ class Ripper(object):
|
|
|
submission_limit = 50
|
|
|
supported_exts = [ "jpg", "jpeg", "png", "gif" ]
|
|
|
|
|
|
- def __init__(self, redis_conn, output_dir):
|
|
|
+ def __init__(self, redis_conn, output_dir, client_id, client_secret):
|
|
|
self.redis_conn = redis_conn
|
|
|
self.output_dir = os.path.abspath(output_dir)
|
|
|
|
|
|
- self.reddit = praw.Reddit(user_agent=self.agent)
|
|
|
+ self.reddit = praw.Reddit(
|
|
|
+ client_id=client_id,
|
|
|
+ client_secret=client_secret,
|
|
|
+ user_agent=self.agent,
|
|
|
+ )
|
|
|
|
|
|
def _random_name(self, size=10, chars=string.ascii_lowercase +
|
|
|
string.digits):
|
|
@@ -37,7 +41,7 @@ class Ripper(object):
|
|
|
return title
|
|
|
|
|
|
def _get_ext(self, url):
|
|
|
- url_info = urlparse.urlparse(url)
|
|
|
+ url_info = urllib.parse.urlparse(url)
|
|
|
return url_info.path.split(".")[-1].lower()
|
|
|
|
|
|
def _seen(self, sub_id, subreddit):
|
|
@@ -102,11 +106,11 @@ class Ripper(object):
|
|
|
self.redis_conn.sadd("subreddits", sub["name"])
|
|
|
|
|
|
if sub["type"] == "hot":
|
|
|
- submissions = self.reddit.get_subreddit(sub["name"]).get_hot(
|
|
|
+ submissions = self.reddit.subreddit(sub["name"]).hot(
|
|
|
limit=self.submission_limit
|
|
|
)
|
|
|
elif sub["type"] == "new":
|
|
|
- submissions = self.reddit.get_subreddit(sub["name"]).get_new(
|
|
|
+ submissions = self.reddit.subreddit(sub["name"]).new(
|
|
|
limit=self.submission_limit
|
|
|
)
|
|
|
else:
|