getMessage().' - '.$mdb2->getUserinfo()); } // iterate through downloaded files containing 10 random people from Facebook New York while(true) { if ($argv[1] != "") { scrape_persons_friends($argv[1], $mdb2); break; } $d = dir("."); while (false !== ($entry = $d->read())) { if (preg_match("/^vancouver-\d+\.txt/",$entry)) { if ($DEBUG) echo "Loading " . $entry . "\n"; if (filesize($entry) < 20000) { if ($DEBUG) echo $entry . " is too small. Deleting.\n"; unlink($entry); continue; } $fh = fopen($entry, "r") or die("Could not open file!"); $data = fread($fh, filesize($entry)) or die("Could not read file!"); fclose($fh); // Get all the ids on that page $pattern = "/

close(); } function scrape_persons_friends($id, $network_id, $mdb2) { $DEBUG = true; $sequential_deletes = 0; if ($DEBUG) echo "Loading " . $id . "'s friends from network " . $network_id . "\n"; mkdir("$id-1"); $page = 1; $next_relative_url = "friends/?id=$id&nk=$network_id"; // Dump this person's New York friends into a series of files while (true) { `wget --no-verbose --output-document=$id-1/vancouver_friends_$page.html --load-cookies=cookies.txt --save-cookies=cookies.txt --keep-session-cookies --no-check-certificate --user-agent=Mozilla/5.0 "http://www.facebook.com/$next_relative_url";`; // Sleep for 2 to 4 seconds. sleep(2); // Open downloaded page if (filesize("$id-1/vancouver_friends_$page.html") < 10000) { if (filesize("$id-1/vancouver_friends_$page.html") == 850) { echo "This person has no public friendships.\n"; break; } if ($DEBUG) echo "$id-1/vancouver_friends_$page.html is too small. Deleting.\n"; unlink("$id-1/vancouver_friends_$page.html"); $sequential_deletes++; if ($sequential_deletes > 5) { if ($DEBUG) echo "Logging into Facebook again."; $sequential_deletes = 0; //`./login.sh`; sleep(10); continue; } $page++; continue; } if ($DEBUG) echo "Opening $id-1/vancouver_friends_$page.html\n"; $fh2 = fopen("$id-1/vancouver_friends_$page.html", "r") or die("Could not open file!"); $this_friend_page = fread($fh2, filesize("$id-1/vancouver_friends_$page.html")) or die("Could not read file!"); fclose($fh2); // Scrape Friends' IDs scrape_friends_ids($id, $this_friend_page, $mdb2); // Scrape Total Friendships if ($page == 1) scrape_total_friendships($id, $this_friend_page, $mdb2); if ($next_relative_url = next_page_url($this_friend_page)) { $page++; } else { break; } } if ($DEBUG) echo "Deleting $id-1/\n"; deltree("$id-1"); } function deltree($path) { if (is_dir($path)) { if (version_compare(PHP_VERSION, '5.0.0') < 0) { $entries = array(); if ($handle = opendir($path)) { while (false !== ($file = readdir($handle))) $entries[] = $file; closedir($handle); } } else { $entries = scandir($path); if ($entries === false) $entries = array(); // just in case scandir fail... } foreach ($entries as $entry) { if ($entry != '.' && $entry != '..') { deltree($path.'/'.$entry); } } return rmdir($path); } else { return unlink($path); } } function id_exists($id, $mdb2) { $DEBUG = true; $sql = "SELECT * FROM friendship_summary WHERE id = $id"; $result = $mdb2->query($sql); if(PEAR::isError($result)) { if ($DEBUG) echo("Failed to issue query, error message : " . $result->getMessage() . "\n"); } $row = $result->fetchRow(MDB2_FETCHMODE_ASSOC); return $row[id]; } function next_page_url($this_friend_page) { // Check if there's another page to download. $next_pattern = "/

Next<\/a><\/li>/"; preg_match($next_pattern, $this_friend_page, &$next_link_matches); if ($DEBUG) echo "Next page link: " . $next_link_matches[1] . "\n"; return $next_link_matches[1]; } function scrape_total_friendships($id, $this_friend_page, $mdb2) { $DEBUG = 1; // Scrape out total friendships $total_friends_pattern = "/has (\d+) friends/"; preg_match($total_friends_pattern, $this_friend_page, &$total_friend_matches); // Insert total into database if ($total_friend_matches[1]) { if ($DEBUG) echo $id . " has " . $total_friend_matches[1] . " friends.\n"; $sql = "INSERT INTO friendship_summary (id, 1st_degree) VALUES ('$id', " . $total_friend_matches[1] . ")"; $result = $mdb2->query($sql); if(PEAR::isError($result)) { if ($DEBUG) echo("Failed to issue query, error message : " . $result->getMessage() . ". Trying an UPDATE instead.\n"); $sql = "UPDATE friendship_summary SET 1st_degree = " . $total_friend_matches[1] . " where id = '" . $id . "'"; $result = $mdb2->query($sql); } else { `echo "\\033"`; } } else { if ($DEBUG) echo $id . " has no friends in Vancouver.\n"; } } function scrape_friends_ids($id, $this_friend_page, $mdb2) { $DEBUG = 1; $bell = false; if ($DEBUG) echo "Scraping $id's friends' ids.\n"; // Scrape out friends' user ids //$friends_id_pattern = "/fname\\\\\"/"; $friends_id_pattern = "/id=(\d+)\\\\\" class=\\\\\"fname\\\\\">/"; // $friends_id_pattern = "//"; preg_match_all($friends_id_pattern, $this_friend_page, &$friend_id_matches); if ($DEBUG) echo sizeof($friend_id_matches[0]) . " ids found\n"; // Insert relationships into database foreach ($friend_id_matches[1] as $friend_id) { if ($DEBUG) echo $id . " and " . $friend_id . " are friends.\n"; $sql = "INSERT INTO friendships (id, friend_id) VALUES ('$id', '" . $friend_id . "')"; $result = $mdb2->query($sql); if(PEAR::isError($result)) { if ($DEBUG) echo("Failed to issue query, error message : " . $result->getMessage() . "\n"); echo "This friendship has probably already been entered into the database.\n"; } else { $bell = true; } } if ($bell) { `echo "\\033"`; `echo ^G`; } } ?>