Script to download twitter search results as csv

Script to download twitter search results as csv

Step 1: Go to Twitter search results page

Make sure the url looks like

Step 2: Open Chrome developer console

Open the browser developer console (also known as the inspect element window). Use the keyboard shortcut  Ctrl + Shift + J  (or  Command + Option + J  on macOS). This will open the developer console in a separate window.

Step 3: Paste this code and execute it by pressing enter (return)

const run = async () => {

const USERS_TO_FETCH = 100;

const SEARCH_RESULTS_CONTAINER_SELECTOR =

'[aria-label="Timeline: Search timeline"]';

const searchResultsContainer = document.querySelector(

SEARCH_RESULTS_CONTAINER_SELECTOR

);

const ROW_SELECTOR = ".css-1dbjc4n.r-1adg3ll.r-1ny4l3l";

const fetchedResults = [];

function scrollAndWait() {

return new Promise((resolve, reject) => {

const startTime = Date.now();

const timeoutDuration = 5000; // Set the desired timeout duration (in milliseconds)

const checkScroll = () => {

window.scrollTo(0, document.body.scrollHeight);

previousScrollHeight = document.body.scrollHeight;

console.log(window.scrollY, previousScrollHeight);

if (

window.scrollY === previousScrollHeight - window.innerHeight ||

fetchedResults.length >= USERS_TO_FETCH

) {

resolve();

} else {

const currentTime = Date.now();

if (currentTime - startTime > timeoutDuration) {

reject();

} else {

setTimeout(checkScroll, 100); // Check every 100ms

}

}

};

checkScroll();

});

}

console.log(`Fetching ${USERS_TO_FETCH} results`);

let previousScrollHeight = 0;

let fetching = false;

const fetchResults = async () => {

if (fetchedResults.length >= USERS_TO_FETCH) {

exportResults(fetchedResults);

return;

}

const results = searchResultsContainer.querySelectorAll(ROW_SELECTOR);

for (let i = 0; i < results.length; i++) {

const user = {};

const result = results[i];

const text = result.innerText.split("\n");

user.name = JSON.stringify(text[0]);

user.username = JSON.stringify(text[1]);

user.followsYou = text[2] === "Follows you";

user.bio = JSON.stringify(text[3]);

if (

!fetchedResults.find(

(fetchedUser) => fetchedUser.username === user.username

)

) {

fetchedResults.push(user);

}

}

console.log(`Fetched ${fetchedResults.length} results so far...`);

fetching = false;

await delay(Math.floor(Math.random() * (2000 - 1000 + 1) + 1000));

previousScrollHeight = window.scrollY;

console.log(window.scrollY, previousScrollHeight);

window.scrollTo(0, document.body.scrollHeight);

if (previousScrollHeight > 0 && window.scrollY > 0) {

if (

window.scrollY <= previousScrollHeight ||

fetchedResults.length >= USERS_TO_FETCH

) {

exportResults(fetchedResults);

return;

}

}

};

fetchResults();

const observer = new MutationObserver(async () => {

if (fetchedResults.length < USERS_TO_FETCH) {

if (!fetching) {

fetching = true;

await delay(Math.floor(Math.random() * (2000 - 1000 + 1) + 1000));

await fetchResults();

}

} else {

observer.disconnect();

return;

}

});

observer.observe(searchResultsContainer, {

childList: true,

subtree: true,

});

};

Join the community

Reach more people,
Reach more revenue.

"The secret of getting ahead is getting started"

Reach my revenue goals