Script to download twitter search results as csv
Step 1: Go to Twitter search results page
Make sure the url looks like
Step 2: Open Chrome developer console
Open the browser developer console (also known as the inspect element window). Use the keyboard shortcut Ctrl + Shift + J (or Command + Option + J on macOS). This will open the developer console in a separate window.
Step 3: Paste this code and execute it by pressing enter (return)
const run = async () => {
const USERS_TO_FETCH = 100;
const SEARCH_RESULTS_CONTAINER_SELECTOR =
'[aria-label="Timeline: Search timeline"]';
const searchResultsContainer = document.querySelector(
SEARCH_RESULTS_CONTAINER_SELECTOR
);
const ROW_SELECTOR = ".css-1dbjc4n.r-1adg3ll.r-1ny4l3l";
const fetchedResults = [];
function scrollAndWait() {
return new Promise((resolve, reject) => {
const startTime = Date.now();
const timeoutDuration = 5000; // Set the desired timeout duration (in milliseconds)
const checkScroll = () => {
window.scrollTo(0, document.body.scrollHeight);
previousScrollHeight = document.body.scrollHeight;
console.log(window.scrollY, previousScrollHeight);
if (
window.scrollY === previousScrollHeight - window.innerHeight ||
fetchedResults.length >= USERS_TO_FETCH
) {
resolve();
} else {
const currentTime = Date.now();
if (currentTime - startTime > timeoutDuration) {
reject();
} else {
setTimeout(checkScroll, 100); // Check every 100ms
}
}
};
checkScroll();
});
}
console.log(`Fetching ${USERS_TO_FETCH} results`);
let previousScrollHeight = 0;
let fetching = false;
const fetchResults = async () => {
if (fetchedResults.length >= USERS_TO_FETCH) {
exportResults(fetchedResults);
return;
}
const results = searchResultsContainer.querySelectorAll(ROW_SELECTOR);
for (let i = 0; i < results.length; i++) {
const user = {};
const result = results[i];
const text = result.innerText.split("\n");
user.name = JSON.stringify(text[0]);
user.username = JSON.stringify(text[1]);
user.followsYou = text[2] === "Follows you";
user.bio = JSON.stringify(text[3]);
if (
!fetchedResults.find(
(fetchedUser) => fetchedUser.username === user.username
)
) {
fetchedResults.push(user);
}
}
console.log(`Fetched ${fetchedResults.length} results so far...`);
fetching = false;
await delay(Math.floor(Math.random() * (2000 - 1000 + 1) + 1000));
previousScrollHeight = window.scrollY;
console.log(window.scrollY, previousScrollHeight);
window.scrollTo(0, document.body.scrollHeight);
if (previousScrollHeight > 0 && window.scrollY > 0) {
if (
window.scrollY <= previousScrollHeight ||
fetchedResults.length >= USERS_TO_FETCH
) {
exportResults(fetchedResults);
return;
}
}
};
fetchResults();
const observer = new MutationObserver(async () => {
if (fetchedResults.length < USERS_TO_FETCH) {
if (!fetching) {
fetching = true;
await delay(Math.floor(Math.random() * (2000 - 1000 + 1) + 1000));
await fetchResults();
}
} else {
observer.disconnect();
return;
}
});
observer.observe(searchResultsContainer, {
childList: true,
subtree: true,
});
};
Reach more people,
Reach more revenue.
"The secret of getting ahead is getting started"
Reach my revenue goals