Mirrors/Ultimate-Facebook-Scraper
2
0
Fork 0
mirror of https://github.com/harismuneer/Ultimate-Facebook-Scraper synced 2024-11-10 06:04:17 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #104 from aayaffe/feature/page_scraper

Browse source 
Scraping group posts
This commit is contained in:
Haris Muneer ⚡️ 2020-04-17 20:55:21 +05:00 committed by GitHub
commit dd4a0541d3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 654 additions and 269 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
.gitignore vendored
View file

@ -115,8 +115,13 @@ dmypy.json
## Generated data
/data
scraper/credentials.yaml
scraper/input.txt
scraper/data/
scraper/debug.log
##misplaced configuration files
scraper/selectors.json
scraper/params.json
scraper/input.txt
## Python venv
venv
venv

5
README.md
View file

@ -115,6 +115,7 @@ A bot which scrapes almost everything about a user's Facebook profile including:
- videos
- friends list and their profile photos (including Followers, Following, Work Friends, College Friends etc)
- and all public posts/statuses available on the user's timeline.
**NEW NEW NEW** Now you can scrape Facebook group posts.
Data is scraped in an organized format to be used for educational/research purposes by researchers. This scraper does not use Facebook's Graph API meaning there are no rate limiting issues.
@ -167,12 +168,12 @@ $ pip install -e .
```
The code is multi-platform and is tested on both Windows and Linux.
The tool uses latest version of [Chrome Web Driver](http://chromedriver.chromium.org/downloads). I have placed the webdriver along with the code but if that version doesn't work then replace the chrome web driver with the latest one according to your platform and your Google Chrome version.
Chrome driver is automatically downloaded using the chromedriver_manager package.
### How to Run
- Fill your Facebook credentials into [`credentials.yaml`](credentials.yaml)
- Edit the [`input.txt`](input.txt) file and add many profiles links as you want in the following format with each link on a new line:
- Edit the [`input.txt`](input.txt) file and add profile, groups and individual group posts links as you want in the following format with each link on a new line:
Make sure the link only contains the username or id number at the end and not any other stuff. Make sure its in the format mentioned above.

1
input.txt
View file

@ -1,2 +1,3 @@
#Lines starting with # and empty lines will be ignored
https://www.facebook.com/andrew.ng.96
https://www.facebook.com/zuck

268
params.json
View file

@ -1,121 +1,151 @@
{
"Friends": {
"scan_list": [
"All",
"Mutual Friends",
"Following",
"Followers",
"Work",
"College",
"Current City",
"Hometown"
],
"section": [
"/friends",
"/friends_mutual",
"/following",
"/followers",
"/friends_work",
"/friends_college",
"/friends_current_city",
"/friends_hometown"
],
"elements_path": [
"//*[contains(@id,'pagelet_timeline_medley_friends')][1]/div[2]/div/ul/li/div/a",
"//*[contains(@id,'pagelet_timeline_medley_friends')][1]/div[2]/div/ul/li/div/a",
"//*[contains(@class,'_3i9')][1]/div/div/ul/li[1]/div[2]/div/div/div/div/div[2]/ul/li/div/a",
"//*[contains(@class,'fbProfileBrowserListItem')]/div/a",
"//*[contains(@id,'pagelet_timeline_medley_friends')][1]/div[2]/div/ul/li/div/a",
"//*[contains(@id,'pagelet_timeline_medley_friends')][1]/div[2]/div/ul/li/div/a",
"//*[contains(@id,'pagelet_timeline_medley_friends')][1]/div[2]/div/ul/li/div/a",
"//*[contains(@id,'pagelet_timeline_medley_friends')][1]/div[2]/div/ul/li/div/a"
],
"file_names": [
"All Friends.txt",
"Mutual Friends.txt",
"Following.txt",
"Followers.txt",
"Work Friends.txt",
"College Friends.txt",
"Current City Friends.txt",
"Hometown Friends.txt"
],
"save_status":0
},
"Photos": {
"scan_list": [
"'s Photos",
"Photos of"
],
"section": [
"/photos_all",
"/photos_of"
],
"elements_path": [
"//*[contains(@id, 'pic_')]",
"//*[contains(@id, 'pic_')]"
],
"file_names": [
"Uploaded Photos.txt",
"Tagged Photos.txt"
],
"save_status": 1
},
"Videos": {
"scan_list": [
"'s Videos",
"Videos of"
],
"section": [
"/videos_by",
"/videos_of"
],
"elements_path": [
"//*[contains(@id, 'pagelet_timeline_app_collection_')]/ul",
"//*[contains(@id, 'pagelet_timeline_app_collection_')]/ul"
],
"file_names": [
"Uploaded Videos.txt",
"Tagged Videos.txt"
],
"save_status": 2
},
"About": {
"scan_list": [],
"section": [
"/about?section=overview",
"/about?section=education",
"/about?section=living",
"/about?section=contact-info",
"/about?section=relationship",
"/about?section=bio",
"/about?section=year-overviews"
],
"elements_path": [
"//*[contains(@id, 'pagelet_timeline_app_collection_')]/ul/li/div/div[2]/div/div",
"//*[contains(@id, 'pagelet_timeline_app_collection_')]/ul/li/div/div[2]/div/div",
"//*[contains(@id, 'pagelet_timeline_app_collection_')]/ul/li/div/div[2]/div/div",
"//*[contains(@id, 'pagelet_timeline_app_collection_')]/ul/li/div/div[2]/div/div",
"//*[contains(@id, 'pagelet_timeline_app_collection_')]/ul/li/div/div[2]/div/div",
"//*[contains(@id, 'pagelet_timeline_app_collection_')]/ul/li/div/div[2]/div/div",
"//*[contains(@id, 'pagelet_timeline_app_collection_')]/ul/li/div/div[2]/div/div"
],
"file_names": [
"Overview.txt",
"Work and Education.txt",
"Places Lived.txt",
"Contact and Basic Info.txt",
"Family and Relationships.txt",
"Details About.txt",
"Life Events.txt"
],
"save_status": 3
},
"Posts": {
"scan_list": [],
"section": [],
"elements_path": ["//div[@class='_5pcb _4b0l _2q8l']"],
"file_names": ["Posts.txt"],
"save_status": 4
}
"Friends": {
"scan_list": [
"All",
"Mutual Friends",
"Following",
"Followers",
"Work",
"College",
"Current City",
"Hometown"
],
"section": [
"/friends",
"/friends_mutual",
"/following",
"/followers",
"/friends_work",
"/friends_college",
"/friends_current_city",
"/friends_hometown"
],
"elements_path": [
"//*[contains(@id,'pagelet_timeline_medley_friends')][1]/div[2]/div/ul/li/div/a",
"//*[contains(@id,'pagelet_timeline_medley_friends')][1]/div[2]/div/ul/li/div/a",
"//*[contains(@class,'_3i9')][1]/div/div/ul/li[1]/div[2]/div/div/div/div/div[2]/ul/li/div/a",
"//*[contains(@class,'fbProfileBrowserListItem')]/div/a",
"//*[contains(@id,'pagelet_timeline_medley_friends')][1]/div[2]/div/ul/li/div/a",
"//*[contains(@id,'pagelet_timeline_medley_friends')][1]/div[2]/div/ul/li/div/a",
"//*[contains(@id,'pagelet_timeline_medley_friends')][1]/div[2]/div/ul/li/div/a",
"//*[contains(@id,'pagelet_timeline_medley_friends')][1]/div[2]/div/ul/li/div/a"
],
"file_names": [
"All Friends.txt",
"Mutual Friends.txt",
"Following.txt",
"Followers.txt",
"Work Friends.txt",
"College Friends.txt",
"Current City Friends.txt",
"Hometown Friends.txt"
],
"save_status": 0
},
"Members": {
"scan_list": [
"All"
],
"section": [
"/members"
],
"elements_path": [
"//*[contains(@id,'pagelet_group_members')][1]/div[2]/div/ul/li/div/a"
],
"file_names": [
"All Members.txt"
],
"save_status": 0
},
"Photos": {
"scan_list": [
"'s Photos",
"Photos of"
],
"section": [
"/photos_all",
"/photos_of"
],
"elements_path": [
"//*[contains(@id, 'pic_')]",
"//*[contains(@id, 'pic_')]"
],
"file_names": [
"Uploaded Photos.txt",
"Tagged Photos.txt"
],
"save_status": 1
},
"Videos": {
"scan_list": [
"'s Videos",
"Videos of"
],
"section": [
"/videos_by",
"/videos_of"
],
"elements_path": [
"//*[contains(@id, 'pagelet_timeline_app_collection_')]/ul",
"//*[contains(@id, 'pagelet_timeline_app_collection_')]/ul"
],
"file_names": [
"Uploaded Videos.txt",
"Tagged Videos.txt"
],
"save_status": 2
},
"About": {
"scan_list": [],
"section": [
"/about?section=overview",
"/about?section=education",
"/about?section=living",
"/about?section=contact-info",
"/about?section=relationship",
"/about?section=bio",
"/about?section=year-overviews"
],
"elements_path": [
"//*[contains(@id, 'pagelet_timeline_app_collection_')]/ul/li/div/div[2]/div/div",
"//*[contains(@id, 'pagelet_timeline_app_collection_')]/ul/li/div/div[2]/div/div",
"//*[contains(@id, 'pagelet_timeline_app_collection_')]/ul/li/div/div[2]/div/div",
"//*[contains(@id, 'pagelet_timeline_app_collection_')]/ul/li/div/div[2]/div/div",
"//*[contains(@id, 'pagelet_timeline_app_collection_')]/ul/li/div/div[2]/div/div",
"//*[contains(@id, 'pagelet_timeline_app_collection_')]/ul/li/div/div[2]/div/div",
"//*[contains(@id, 'pagelet_timeline_app_collection_')]/ul/li/div/div[2]/div/div"
],
"file_names": [
"Overview.txt",
"Work and Education.txt",
"Places Lived.txt",
"Contact and Basic Info.txt",
"Family and Relationships.txt",
"Details About.txt",
"Life Events.txt"
],
"save_status": 3
},
"Posts": {
"scan_list": [],
"section": [],
"elements_path": [
"//div[@class='_5pcb _4b0l _2q8l']"
],
"file_names": [
"Posts.txt"
],
"save_status": 4
},
"GroupPosts": {
"scan_list": [],
"section": [],
"elements_path": [
"//div[@class='_4-u2 mbm _4mrt _5jmm _5pat _5v3q _7cqq _4-u8']"
],
"file_names": [
"Posts.txt"
],
"save_status": 5
}
}

3
requirements.txt
View file

@ -1,2 +1,3 @@
selenium==3.141.0
pyyaml
pyyaml
webdriver_manager

478
scraper/scraper.py
View file

@ -1,7 +1,5 @@
import calendar
import json
import os
import platform
import sys
import urllib.request
import yaml
@ -9,7 +7,7 @@ import utils
import argparse
from selenium import webdriver
from selenium.common.exceptions import TimeoutException, NoSuchElementException
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
@ -53,6 +51,12 @@ def get_facebook_images_url(img_links):
# takes a url and downloads image from that url
def image_downloader(img_links, folder_name):
"""
Download images from a list of image urls.
:param img_links:
:param folder_name:
:return: list of image names downloaded
"""
img_names = []
try:
@ -84,7 +88,6 @@ def image_downloader(img_links, folder_name):
os.chdir(parent)
except Exception:
print("Exception (image_downloader):", sys.exc_info()[0])
return img_names
@ -94,99 +97,37 @@ def image_downloader(img_links, folder_name):
def extract_and_write_posts(elements, filename):
try:
f = open(filename, "w", newline="\r\n")
f = open(filename, "w", newline="\r\n", encoding="utf-8")
f.writelines(
" TIME || TYPE || TITLE || STATUS || LINKS(Shared Posts/Shared Links etc) "
" TIME || TYPE || TITLE || STATUS || LINKS(Shared Posts/Shared Links etc) || POST_ID "
+ "\n"
+ "\n"
)
ids = []
for x in elements:
try:
title = " "
status = " "
link = ""
time = " "
# id
post_id = utils.get_post_id(x)
ids.append(post_id)
# time
time = utils.get_time(x)
# title
title = utils.get_title(x, selectors)
if title.text.find("shared a memory") != -1:
x = x.find_element_by_xpath(selectors.get("title_element"))
title = utils.get_title(x, selectors)
status = utils.get_status(x, selectors)
if (
title.text
== driver.find_element_by_id(selectors.get("title_text")).text
):
if status == "":
temp = utils.get_div_links(x, "img", selectors)
if (
temp == ""
): # no image tag which means . it is not a life event
link = utils.get_div_links(x, "a", selectors).get_attribute(
"href"
)
type = "status update without text"
else:
type = "life event"
link = utils.get_div_links(x, "a", selectors).get_attribute(
"href"
)
status = utils.get_div_links(x, "a", selectors).text
else:
type = "status update"
if utils.get_div_links(x, "a", selectors) != "":
link = utils.get_div_links(x, "a", selectors).get_attribute(
"href"
)
elif title.text.find(" shared ") != -1:
x1, link = utils.get_title_links(title)
type = "shared " + x1
elif title.text.find(" at ") != -1 or title.text.find(" in ") != -1:
if title.text.find(" at ") != -1:
x1, link = utils.get_title_links(title)
type = "check in"
elif title.text.find(" in ") != 1:
status = utils.get_div_links(x, "a", selectors).text
elif (
title.text.find(" added ") != -1 and title.text.find("photo") != -1
):
type = "added photo"
link = utils.get_div_links(x, "a", selectors).get_attribute("href")
elif (
title.text.find(" added ") != -1 and title.text.find("video") != -1
):
type = "added video"
link = utils.get_div_links(x, "a", selectors).get_attribute("href")
else:
type = "others"
if not isinstance(title, str):
title = title.text
status = status.replace("\n", " ")
title = title.replace("\n", " ")
link, status, title, post_type = get_status_and_title(link, x)
line = (
str(time)
+ " || "
+ str(type)
+ str(post_type)
+ " || "
+ str(title)
+ " || "
+ str(status)
+ " || "
+ str(link)
+ " || "
+ str(post_id)
+ "\n"
)
@ -197,12 +138,114 @@ def extract_and_write_posts(elements, filename):
except Exception:
pass
f.close()
except ValueError:
print("Exception (extract_and_write_posts)", "Status =", sys.exc_info()[0])
except Exception:
print("Exception (extract_and_write_posts)", "Status =", sys.exc_info()[0])
return
def get_status_and_title(link, x):
# title
title = utils.get_title(x, selectors)
if title.text.find("shared a memory") != -1:
x = x.find_element_by_xpath(selectors.get("title_element"))
title = utils.get_title(x, selectors)
status = utils.get_status(x, selectors)
if title.text == driver.find_element_by_id(selectors.get("title_text")).text:
if status == "":
temp = utils.get_div_links(x, "img", selectors)
if temp == "": # no image tag which means . it is not a life event
link = utils.get_div_links(x, "a", selectors).get_attribute("href")
post_type = "status update without text"
else:
post_type = "life event"
link = utils.get_div_links(x, "a", selectors).get_attribute("href")
status = utils.get_div_links(x, "a", selectors).text
else:
post_type = "status update"
if utils.get_div_links(x, "a", selectors) != "":
link = utils.get_div_links(x, "a", selectors).get_attribute("href")
elif title.text.find(" shared ") != -1:
x1, link = utils.get_title_links(title)
post_type = "shared " + x1
elif title.text.find(" at ") != -1 or title.text.find(" in ") != -1:
if title.text.find(" at ") != -1:
x1, link = utils.get_title_links(title)
post_type = "check in"
elif title.text.find(" in ") != 1:
status = utils.get_div_links(x, "a", selectors).text
elif title.text.find(" added ") != -1 and title.text.find("photo") != -1:
post_type = "added photo"
link = utils.get_div_links(x, "a", selectors).get_attribute("href")
elif title.text.find(" added ") != -1 and title.text.find("video") != -1:
post_type = "added video"
link = utils.get_div_links(x, "a", selectors).get_attribute("href")
else:
post_type = "others"
if not isinstance(title, str):
title = title.text
status = status.replace("\n", " ")
title = title.replace("\n", " ")
return link, status, title, post_type
def extract_and_write_group_posts(elements, filename):
try:
f = create_post_file(filename)
ids = []
for x in elements:
try:
# id
post_id = utils.get_group_post_id(x)
ids.append(post_id)
except Exception:
pass
total = len(ids)
i = 0
for post_id in ids:
i += 1
try:
add_group_post_to_file(f, filename, post_id, i, total, reload=True)
except ValueError:
pass
f.close()
except ValueError:
print("Exception (extract_and_write_posts)", "Status =", sys.exc_info()[0])
except Exception:
print("Exception (extract_and_write_posts)", "Status =", sys.exc_info()[0])
return
def add_group_post_to_file(f, filename, post_id, number=1, total=1, reload=False):
print("Scraping Post(" + post_id + "). " + str(number) + " of " + str(total))
photos_dir = os.path.dirname(filename)
if reload:
driver.get(utils.create_post_link(post_id, selectors))
line = get_group_post_as_line(post_id, photos_dir)
try:
f.writelines(line)
except Exception:
print("Posts: Could not map encoded characters")
def create_post_file(filename):
"""
Creates post file and header
:param filename:
:return: file
"""
f = open(filename, "w", newline="\r\n", encoding="utf-8")
f.writelines(
"TIME || TYPE || TITLE || STATUS || LINKS(Shared Posts/Shared Links etc) || POST_ID || "
"PHOTO || COMMENTS " + "\n"
)
return f
# -------------------------------------------------------------
# -------------------------------------------------------------
@ -215,11 +258,12 @@ def save_to_file(name, elements, status, current_section):
# status 2 = dealing with videos
# status 3 = dealing with about section
# status 4 = dealing with posts
# status 5 = dealing with group posts
try:
f = None # file pointer
if status != 4:
if status != 4 and status != 5:
f = open(name, "w", encoding="utf-8", newline="\r\n")
results = []
@ -363,6 +407,11 @@ def save_to_file(name, elements, status, current_section):
extract_and_write_posts(elements, name)
return
# dealing with Group Posts
elif status == 5:
extract_and_write_group_posts(elements, name)
return
"""Write results to file"""
if status == 0:
for i, _ in enumerate(results):
@ -404,14 +453,14 @@ def save_to_file(name, elements, status, current_section):
# -----------------------------------------------------------------------------
def scrape_data(user_id, scan_list, section, elements_path, save_status, file_names):
def scrape_data(url, scan_list, section, elements_path, save_status, file_names):
"""Given some parameters, this function can scrap friends/photos/videos/about/posts(statuses) of a profile"""
page = []
if save_status == 4:
page.append(user_id)
if save_status == 4 or save_status == 5:
page.append(url)
page += [user_id + s for s in section]
page += [url + s for s in section]
for i, _ in enumerate(scan_list):
try:
@ -431,6 +480,7 @@ def scrape_data(user_id, scan_list, section, elements_path, save_status, file_na
if save_status != 3:
utils.scroll(total_scrolls, driver, selectors, scroll_time)
pass
data = driver.find_elements_by_xpath(elements_path[i])
@ -477,52 +527,191 @@ def create_original_link(url):
return original_link
def scrap_profile(ids):
def scrap_profile():
data_folder = os.path.join(os.getcwd(), "data")
utils.create_folder(data_folder)
os.chdir(data_folder)
# execute for all profiles given in input.txt file
url = driver.current_url
user_id = create_original_link(url)
print("\nScraping:", user_id)
try:
target_dir = os.path.join(data_folder, user_id.split("/")[-1])
utils.create_folder(target_dir)
os.chdir(target_dir)
except Exception:
print("Some error occurred in creating the profile directory.")
os.chdir("../..")
return
to_scrap = ["Friends", "Photos", "Videos", "About", "Posts"]
for item in to_scrap:
print("----------------------------------------")
print("Scraping {}..".format(item))
if item == "Posts":
scan_list = [None]
elif item == "About":
scan_list = [None] * 7
else:
scan_list = params[item]["scan_list"]
section = params[item]["section"]
elements_path = params[item]["elements_path"]
file_names = params[item]["file_names"]
save_status = params[item]["save_status"]
scrape_data(user_id, scan_list, section, elements_path, save_status, file_names)
print("{} Done!".format(item))
print("Finished Scraping Profile " + str(user_id) + ".")
os.chdir("../..")
return
def get_comments():
comments = []
try:
data = driver.find_element_by_xpath(selectors.get("comment_section"))
reply_links = driver.find_elements_by_xpath(
selectors.get("more_comment_replies")
)
for link in reply_links:
try:
driver.execute_script("arguments[0].click();", link)
except Exception:
pass
see_more_links = driver.find_elements_by_xpath(
selectors.get("comment_see_more_link")
)
for link in see_more_links:
try:
driver.execute_script("arguments[0].click();", link)
except Exception:
pass
data = data.find_elements_by_xpath(selectors.get("comment"))
for d in data:
try:
author = d.find_element_by_xpath(selectors.get("comment_author")).text
text = d.find_element_by_xpath(selectors.get("comment_text")).text
replies = utils.get_replies(d, selectors)
comments.append([author, text, replies])
except Exception:
pass
except Exception:
pass
return comments
def get_group_post_as_line(post_id, photos_dir):
try:
data = driver.find_element_by_xpath(selectors.get("single_post"))
time = utils.get_time(data)
title = utils.get_title(data, selectors).text
# link, status, title, type = get_status_and_title(title,data)
link = utils.get_div_links(data, "a", selectors)
if link != "":
link = link.get_attribute("href")
post_type = ""
status = '"' + utils.get_status(data, selectors).replace("\r\n", " ") + '"'
photos = utils.get_post_photos_links(data, selectors, photos_small_size)
comments = get_comments()
photos = image_downloader(photos, photos_dir)
line = (
str(time)
+ "||"
+ str(post_type)
+ "||"
+ str(title)
+ "||"
+ str(status)
+ "||"
+ str(link)
+ "||"
+ str(post_id)
+ "||"
+ str(photos)
+ "||"
+ str(comments)
+ "\n"
)
return line
except Exception:
return ""
def create_folders():
"""
Creates folder for saving data (profile, post or group) according to current driver url
Changes current dir to target_dir
:return: target_dir or None in case of failure
"""
folder = os.path.join(os.getcwd(), "data")
utils.create_folder(folder)
os.chdir(folder)
try:
item_id = get_item_id(driver.current_url)
target_dir = os.path.join(folder, item_id)
utils.create_folder(target_dir)
os.chdir(target_dir)
return target_dir
except Exception:
print("Some error occurred in creating the group directory.")
os.chdir("../..")
return None
def get_item_id(url):
"""
Gets item id from url
:param url: facebook url string
:return: item id or empty string in case of failure
"""
ret = ""
try:
link = create_original_link(url)
ret = link.split("/")[-1]
if ret.strip() == "":
ret = link.split("/")[-2]
except Exception as e:
print("Failed to get id: " + format(e))
return ret
def scrape_group(url):
if create_folders() is None:
return
group_id = get_item_id(url)
# execute for all profiles given in input.txt file
for user_id in ids:
print("\nScraping:", group_id)
driver.get(user_id)
url = driver.current_url
user_id = create_original_link(url)
to_scrap = ["GroupPosts"] # , "Photos", "Videos", "About"]
for item in to_scrap:
print("----------------------------------------")
print("Scraping {}..".format(item))
print("\nScraping:", user_id)
if item == "GroupPosts":
scan_list = [None]
elif item == "About":
scan_list = [None] * 7
else:
scan_list = params[item]["scan_list"]
try:
target_dir = os.path.join(folder, user_id.split("/")[-1])
utils.create_folder(target_dir)
os.chdir(target_dir)
except Exception:
print("Some error occurred in creating the profile directory.")
continue
section = params[item]["section"]
elements_path = params[item]["elements_path"]
file_names = params[item]["file_names"]
save_status = params[item]["save_status"]
to_scrap = ["Friends", "Photos", "Videos", "About", "Posts"]
for item in to_scrap:
print("----------------------------------------")
print("Scraping {}..".format(item))
scrape_data(url, scan_list, section, elements_path, save_status, file_names)
if item == "Posts":
scan_list = [None]
elif item == "About":
scan_list = [None] * 7
else:
scan_list = params[item]["scan_list"]
print("{} Done!".format(item))
section = params[item]["section"]
elements_path = params[item]["elements_path"]
file_names = params[item]["file_names"]
save_status = params[item]["save_status"]
scrape_data(
user_id, scan_list, section, elements_path, save_status, file_names
)
print("{} Done!".format(item))
print("\nProcess Completed.")
print("Finished Scraping Group " + str(group_id) + ".")
os.chdir("../..")
return
@ -532,13 +721,6 @@ def scrap_profile(ids):
# -----------------------------------------------------------------------------
def safe_find_element_by_id(driver, elem_id):
try:
return driver.find_element_by_id(elem_id)
except NoSuchElementException:
return None
def login(email, password):
""" Logging into our own profile """
@ -554,17 +736,11 @@ def login(email, password):
# options.add_argument("headless")
try:
platform_ = platform.system().lower()
driver = webdriver.Chrome(
executable_path=ChromeDriverManager().install(), options=options
)
except Exception:
print(
"Kindly replace the Chrome Web Driver with the latest one from "
"http://chromedriver.chromium.org/downloads "
"and also make sure you have the latest Chrome Browser version."
"\nYour OS: {}".format(platform_)
)
print("Error loading chrome webdriver " + sys.exc_info()[0])
exit(1)
fb_path = facebook_https_prefix + facebook_link_body
@ -583,7 +759,7 @@ def login(email, password):
driver.find_element_by_name("login").click()
# if your account uses multi factor authentication
mfa_code_input = safe_find_element_by_id(driver, "approvals_code")
mfa_code_input = utils.safe_find_element_by_id(driver, "approvals_code")
if mfa_code_input is None:
return
@ -592,8 +768,10 @@ def login(email, password):
driver.find_element_by_id("checkpointSubmitButton").click()
# there are so many screens asking you to verify things. Just skip them all
while safe_find_element_by_id(driver, "checkpointSubmitButton") is not None:
dont_save_browser_radio = safe_find_element_by_id(driver, "u_0_3")
while (
utils.safe_find_element_by_id(driver, "checkpointSubmitButton") is not None
):
dont_save_browser_radio = utils.safe_find_element_by_id(driver, "u_0_3")
if dont_save_browser_radio is not None:
dont_save_browser_radio.click()
@ -616,17 +794,34 @@ def scraper(**kwargs):
if ("password" not in cfg) or ("email" not in cfg):
print("Your email or password is missing. Kindly write them in credentials.txt")
exit(1)
ids = [
facebook_https_prefix + facebook_link_body + line.split("/")[-1]
for line in open("input.txt", newline="\n")
urls = [
facebook_https_prefix + facebook_link_body + get_item_id(line)
for line in open("input.txt", newline="\r\n")
if not line.lstrip().startswith("#") and not line.strip() == ""
]
if len(ids) > 0:
if len(urls) > 0:
print("\nStarting Scraping...")
login(cfg["email"], cfg["password"])
scrap_profile(ids)
for url in urls:
driver.get(url)
link_type = utils.identify_url(driver.current_url)
if link_type == 0:
scrap_profile()
elif link_type == 1:
# scrap_post(url)
pass
elif link_type == 2:
scrape_group(driver.current_url)
elif link_type == 3:
file_name = params["GroupPosts"]["file_names"][0]
item_id = get_item_id(driver.current_url)
if create_folders() is None:
continue
f = create_post_file(file_name)
add_group_post_to_file(f, file_name, item_id)
f.close()
os.chdir("../..")
driver.close()
else:
print("Input file is empty.")
@ -694,7 +889,6 @@ if __name__ == "__main__":
old_height = 0
driver = None
CHROMEDRIVER_BINARIES_FOLDER = "bin"
with open("selectors.json") as a, open("params.json") as b:
selectors = json.load(a)

119
scraper/utils.py
View file

@ -1,10 +1,12 @@
import argparse
import os
import sys
from calendar import calendar
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.common.exceptions import TimeoutException, NoSuchElementException
from selenium.webdriver.support.ui import WebDriverWait
# -----------------------------------------------------------------------------
#
# -----------------------------------------------------------------------------
@ -17,6 +19,12 @@ def to_bool(x):
raise argparse.ArgumentTypeError("Boolean value expected")
def create_post_link(post_id, selectors):
return (
selectors["facebook_https_prefix"] + selectors["facebook_link_body"] + post_id
)
# -----------------------------------------------------------------------------
#
# -----------------------------------------------------------------------------
@ -75,6 +83,64 @@ def get_status(x, selectors):
return status
def get_post_id(x):
post_id = -1
try:
post_id = x.get_attribute("id")
post_id = post_id.split(":")[-1]
except Exception:
pass
return post_id
def get_group_post_id(x):
post_id = -1
try:
post_id = x.get_attribute("id")
post_id = post_id.split("_")[-1]
if ";" in post_id:
post_id = post_id.split(";")
post_id = post_id[2]
else:
post_id = post_id.split(":")[0]
except Exception:
pass
return post_id
def get_photo_link(x, selectors, small_photo):
link = ""
try:
if small_photo:
link = x.find_element_by_xpath(
selectors.get("post_photo_small")
).get_attribute("src")
else:
link = x.get_attribute("data-ploi")
except NoSuchElementException:
try:
link = x.find_element_by_xpath(
selectors.get("post_photo_small_opt1")
).get_attribute("src")
except AttributeError:
pass
except Exception:
print("Exception (get_post_photo_link):", sys.exc_info()[0])
except Exception:
print("Exception (get_post_photo_link):", sys.exc_info()[0])
return link
def get_post_photos_links(x, selectors, small_photo):
links = []
photos = safe_find_elements_by_xpath(x, selectors.get("post_photos"))
if photos is not None:
for el in photos:
links.append(get_photo_link(el, selectors, small_photo))
return links
def get_div_links(x, tag, selectors):
try:
temp = x.find_element_by_xpath(selectors.get("temp"))
@ -137,3 +203,52 @@ def get_time(x):
finally:
return time
def identify_url(url):
"""
A possible way to identify the link.
Not Exhaustive!
:param url:
:return:
0 - Profile
1 - Profile post
2 - Group
3 - Group post
"""
if "groups" in url:
if "permalink" in url:
return 3
else:
return 2
elif "posts" in url:
return 1
else:
return 0
def safe_find_elements_by_xpath(driver, xpath):
try:
return driver.find_elements_by_xpath(xpath)
except NoSuchElementException:
return None
def get_replies(comment_element, selectors):
replies = []
data = comment_element.find_elements_by_xpath(selectors.get("comment_reply"))
for d in data:
try:
author = d.find_element_by_xpath(selectors.get("comment_author")).text
text = d.find_element_by_xpath(selectors.get("comment_text")).text
replies.append([author, text])
except Exception:
pass
return replies
def safe_find_element_by_id(driver, elem_id):
try:
return driver.find_element_by_id(elem_id)
except NoSuchElementException:
return None

14
selectors.json
View file

@ -17,5 +17,17 @@
"scroll_script": "window.scrollTo(0, document.body.scrollHeight);",
"title_text": "fb-timeline-cover-name",
"profilePicThumb": "profilePicThumb",
"fb_link": "https://en-gb.facebook.com/"
"fb_link": "https://en-gb.facebook.com/",
"single_post" : ".//div[contains(@class, '_5pcr')]",
"post_photos": ".//a[contains(@class, '_5dec') or contains(@class, '_4-eo')]",
"post_photo_small" : ".//img[contains(@class, '_46-i')]",
"post_photo_small_opt1" : ".//img[contains(@class, 'scaledImageFitWidth') or contains(@class, 'scaledImageFitHeight')]",
"comment_section" : ".//*[@class='commentable_item']",
"comment" : ".//div[@aria-label='Comment']",
"comment_author" : ".//a[@class='_6qw4']",
"comment_text" : ".//span[contains(@class,'_3l3x')]",
"more_comment_replies": ".//a[contains(@class,'_4sxc _42ft')]",
"comment_see_more_link" : ".//a[contains(@class,'_5v47 fss')]",
"comment_reply" : "..//..//div[@aria-label='Comment reply']"
}

0
test/__init__.py Normal file
View file

26
test/test_utils.py Normal file
View file

@ -0,0 +1,26 @@
from unittest import TestCase
from scraper import utils
class Test(TestCase):
def test_identify_url(self):
self.assertEqual(
utils.identify_url("https://www.facebook.com/groups/123456789694/?fref=nf"),
2,
)
self.assertEqual(
utils.identify_url("https://www.facebook.com/groups/123456789694"), 2
)
self.assertEqual(
utils.identify_url(
"https://www.facebook.com/groups/12345645546/permalink/213453415513/"
),
3,
)
self.assertEqual(
utils.identify_url("https://www.facebook.com/dfsdfsdf.sdfsdfs"), 0,
)
self.assertEqual(
utils.identify_url("https://www.facebook.com/sdfsdfsd/posts/123456784684"),
1,
)