How to Scrape SoundCloud for AI Training: Step-By-Step Tutorial

const puppeteer = require('puppeteer');

const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));

async function scrapePlaylistTracks() {
  let browser;
  
  try {
    browser = await puppeteer.launch({ 
      headless: true,
      args: [
        // PROXY CONFIGURATION
        '--proxy-server=http://gate.decodo.com:7000',
        '--no-sandbox',
        '--disable-setuid-sandbox',
        '--disable-dev-shm-usage',
        '--disable-gpu',
        '--mute-audio',
        '--window-size=1366,768'
      ]
    });
    
    const page = await browser.newPage();
    await page.setViewport({ width: 1366, height: 768 });
    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Safari/537.36');
    
    // PROXY AUTHENTICATION - Replace with your proxy credentials
    await page.authenticate({
      username: 'YOUR_PROXY_USERNAME',
      password: 'YOUR_PROXY_PASSWORD'
    });
    
    // Block noisy resources
    await page.setRequestInterception(true);
    page.on('request', (req) => {
      const rt = req.resourceType();
      if (['image', 'media', 'font', 'manifest'].includes(rt)) {
        return req.abort();
      }
      return req.continue();
    });
    
    // Silence console noise
    page.on('console', () => {});
    page.on('pageerror', () => {});
    
    // TARGET URL - Replace with the Soundcloud playlist you want to scrape
    await page.goto('https://soundcloud.com/music-charts-uk/sets/hip-hop', {
      waitUntil: 'domcontentloaded',
      timeout: 45000
    });
    
    await delay(5000);
    
    // Handle cookie consent
    try {
      await page.waitForSelector('#onetrust-accept-btn-handler', { timeout: 5000 });
      await page.click('#onetrust-accept-btn-handler');
      await delay(500);
    } catch (e) {}
    
    // Handle login modal
    try {
      await page.waitForSelector('.modal__closeButton', { timeout: 5000 });
      await page.click('.modal__closeButton');
      await delay(300);
    } catch (e) {}
    
    // Scroll to load all tracks
    for (let i = 0; i < 5; i++) {
      await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
      await delay(3000);
    }
    
    const itemCount = await page.evaluate(() => {
      const trackListContainer = document.querySelector('.trackList__list') || 
                                 document.querySelector('ul') || 
                                 document.querySelector('[class*="trackList"]');
      return trackListContainer ? trackListContainer.querySelectorAll('li').length : 0;
    });
    console.log(`Found ${itemCount} playlist tracks`);
    
    // Click on all track photos to reveal hidden track information
    console.log('Revealing hidden tracks by clicking...');
    
    const trackImages = await page.$$('.trackList__list li img[width="32"][height="32"]');
    
    for (let i = 0; i < trackImages.length; i++) {
      try {
        await trackImages[i].click();
        await delay(800);
      } catch (e) {
        // Continue if click fails on any particular image
      }
    }
    
    // Additional delay to ensure all clicked tracks have loaded
    await delay(2000);
    
    // Extract playlist tracks
    const tracks = await page.evaluate(() => {
      const results = [];
      
      // Find the main track list container
      const trackListContainer = document.querySelector('.trackList__list') || 
                                 document.querySelector('ul') || 
                                 document.querySelector('[class*="trackList"]');
      
      if (!trackListContainer) return [];
      
      const trackItems = trackListContainer.querySelectorAll('li');
      
      trackItems.forEach((trackItem) => {
        // Get track rank/position
        const rankEl = trackItem.querySelector('.trackItem__separator');
        if (!rankEl) return;
        
        const rank = rankEl.textContent.trim();
        
        // Get artist and track title
        const artistEl = trackItem.querySelector('.trackItem__username');
        const titleEl = trackItem.querySelector('.trackItem__trackTitle');
        
        if (!artistEl || !titleEl) return;
        
        const artist = artistEl.textContent.trim();
        const title = titleEl.textContent.trim();
        const trackUrl = 'https://soundcloud.com' + titleEl.href.replace('https://soundcloud.com', '');
        
        // Check if track is geo-blocked
        const blockMsgEl = trackItem.querySelector('.trackItem__blockMsg');
        if (blockMsgEl) {
          results.push({
            index: parseInt(rank),
            artist,
            title,
            trackUrl,
            status: `[${blockMsgEl.textContent.trim()}]`
          });
          return;
        }
        
        // Extract play count if available
        const playCountEl = trackItem.querySelector('.trackItem__playCount');
        let plays = 'N/A';
        
        if (playCountEl) {
          let playText = playCountEl.textContent.replace(/[\r\n\t]/g, ' ').replace(/\s+/g, ' ').trim();
          const playMatch = playText.match(/(\d+(?:\.\d+)?[KMB])\s*$/);
          plays = playMatch ? playMatch[1] : playText;
        }
        
        results.push({
          index: parseInt(rank),
          artist,
          title,
          trackUrl,
          plays
        });
      });
      
      // Sort tracks by rank to ensure proper order
      return results.sort((a, b) => a.index - b.index);
    });
    
    console.log(`${tracks.length} playlist tracks available:\n`);
    
    tracks.forEach((track) => {
      const playsDisplay = track.plays ? ` (${track.plays} plays)` : '';
      const statusDisplay = track.status ? ` ${track.status}` : '';
      console.log(`${track.index}. ${track.artist} - "${track.title}"${playsDisplay}${statusDisplay}`);
      console.log(`   ${track.trackUrl}\n`);
    });
    
    return tracks;
    
  } catch (error) {
    console.error('Scraping failed:', error.message);
  } finally {
    if (browser) {
      await browser.close();
    }
  }
}

scrapePlaylistTracks();

const puppeteer = require('puppeteer');

const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));

async function scrapeSearchResults() {
  let browser;
  
  try {
    browser = await puppeteer.launch({ 
      headless: true,
      args: [
        // PROXY CONFIGURATION
        '--proxy-server=http://gate.decodo.com:7000',
        '--no-sandbox',
        '--disable-setuid-sandbox',
        '--disable-dev-shm-usage',
        '--disable-gpu',
        '--mute-audio',
        '--window-size=1366,768'
      ]
    });
    
    const page = await browser.newPage();
    await page.setViewport({ width: 1366, height: 768 });
    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Safari/537.36');
    
    // PROXY AUTHENTICATION - Replace with your proxy credentials
    await page.authenticate({
      username: 'YOUR_PROXY_USERNAME',
      password: 'YOUR_PROXY_PASSWORD'
    });
    
    // Block noisy resources
    await page.setRequestInterception(true);
    page.on('request', (req) => {
      const rt = req.resourceType();
      if (['image', 'media', 'font', 'manifest'].includes(rt)) {
        return req.abort();
      }
      return req.continue();
    });
    
    // Silence console noise
    page.on('console', () => {});
    page.on('pageerror', () => {});
    
    // TARGET URL - Replace with the Soundcloud search results page you want to scrape
    await page.goto('https://soundcloud.com/search/sounds?q=creative%20commons&filter.created_at=last_year&filter.license=to_use_commercially', {
      waitUntil: 'domcontentloaded',
      timeout: 45000
    });
    
    await delay(3000);
    
    // Handle cookie consent
    try {
      await page.waitForSelector('#onetrust-accept-btn-handler', { timeout: 5000 });
      await page.click('#onetrust-accept-btn-handler');
      await delay(500);
    } catch (e) {}
    
    // Handle login modal
    try {
      await page.waitForSelector('.modal__closeButton', { timeout: 5000 });
      await page.click('.modal__closeButton');
      await delay(300);
    } catch (e) {}
    
    // Wait for search results
    await page.waitForSelector('.searchList__item', { timeout: 15000 });
    
    // Infinite scrolling logic
    console.log('Starting infinite scroll to load more results...');
    
    const maxScrollAttempts = 5; // Configurable
    const targetResults = 500; // Stop when we have enough results
    
    let previousItemCount = 0;
    let scrollAttempts = 0;
    let noNewContentCount = 0;
    
    while (scrollAttempts < maxScrollAttempts) {
      // Get current item count
      const currentItemCount = await page.evaluate(() =>
        document.querySelectorAll('.searchList__item').length
      );
      
      console.log(`Scroll attempt ${scrollAttempts + 1}: Found ${currentItemCount} items`);
      
      // If we have enough results, break
      if (currentItemCount >= targetResults) {
        console.log(`Reached target of ${targetResults} results. Stopping scroll.`);
        break;
      }
      
      // Check if new content was loaded
      if (currentItemCount > previousItemCount) {
        noNewContentCount = 0; // Reset counter if new content appeared
        console.log(`New content loaded: +${currentItemCount - previousItemCount} items`);
      } else {
        noNewContentCount++;
        console.log(`No new content loaded (attempt ${noNewContentCount})`);
        
        // If no new content for 3 consecutive attempts, we've reached the end
        if (noNewContentCount >= 3) {
          console.log('No new content loaded after 3 attempts. Reached end of results.');
          break;
        }
      }
      
      previousItemCount = currentItemCount;
      
      // Scroll down to trigger loading more content
      await page.evaluate(() => {
        window.scrollTo(0, document.body.scrollHeight);
      });
      
      await delay(2000);
      
      // Also try scrolling by viewport height as backup
      await page.evaluate(() => {
        window.scrollBy(0, window.innerHeight);
      });
      
      await delay(1500);
      
      // Check if there's a "Load more" button and click it
      try {
        const loadMoreButton = await page.$('.searchLoadMore button, .loadMore button, button[data-testid="load-more"]');
        if (loadMoreButton) {
          console.log('Found "Load more" button, clicking...');
          await loadMoreButton.click();
          await delay(3000);
        }
      } catch (e) {
        // No load more button found, continue with scrolling
      }
      
      scrollAttempts++;
    }
    
    const finalItemCount = await page.evaluate(() =>
      document.querySelectorAll('.searchList__item').length
    );
    
    // Extract downloadable tracks
    const tracks = await page.evaluate(async () => {
      const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));
      const results = [];
      const searchItems = document.querySelectorAll('.searchList__item');
      
      const maxResults = searchItems.length; // Process all found items
      
      console.log(`Processing ${Math.min(maxResults, searchItems.length)} items for download availability...`);
      
      for (let i = 0; i < Math.min(maxResults, searchItems.length); i++) {
        const item = searchItems[i];
        
        if ((i + 1) % 10 === 0) {
          console.log(`Processed ${i + 1}/${Math.min(maxResults, searchItems.length)} items...`);
        }
        
        item.scrollIntoView({ block: 'center' });
        await delay(100);
        
        const artistEl = item.querySelector('.soundTitle__usernameText') || 
                         item.querySelector('.soundTitle .usernameText') ||
                         item.querySelector('.sc-text-body') ||
                         item.querySelector('a[href*="/"]') ||
                         item.querySelector('[class*="username"]');
        const artist = artistEl ? artistEl.textContent.trim() : 'Unknown Artist';
        
        let title = 'Unknown Title';
        let trackUrl = '';
        
        const titleSelectors = [
          '.soundTitle__title > .sc-link-dark',
          '.soundTitle__title span:not(.soundTitle__usernameText)',
          'h3 a.sc-link-dark',
          'h3 a',
          '.soundTitle__title',
          '[class*="title"] a',
          'a[href*="/"]',
          '.sc-link-dark'
        ];
        for (const selector of titleSelectors) {
          const titleEl = item.querySelector(selector);
          if (titleEl && titleEl.textContent.trim()) {
            title = titleEl.textContent.trim();
            break;
          }
        }
        
        const linkSelectors = [
          '.soundTitle__title a[href]',
          'h3 a[href]',
          'a.sc-link-dark[href]',
          'a[href*="/"]',
          '[class*="title"] a[href]',
          'a[href]'
        ];
        for (const selector of linkSelectors) {
          const linkEl = item.querySelector(selector);
          if (linkEl) {
            const href = linkEl.getAttribute('href') || '';
            if (href) {
              trackUrl = href.startsWith('http') ? href : `https://soundcloud.com${href}`;
              break;
            }
          }
        }
        
        let hasDownload = false;
        const moreSelectors = [
          'button[aria-label="More"]',
          '.soundActions .sc-button-more',
          '.sc-button-more',
          'button[title="More"]',
          'button[class*="more"]',
          '[data-testid="more-button"]',
          'button[aria-label*="more"]',
          '.moreButton'
        ];
        let moreBtn = null;
        for (const selector of moreSelectors) {
          moreBtn = item.querySelector(selector);
          if (moreBtn) break;
        }
        
        if (moreBtn) {
          try {
            moreBtn.scrollIntoView({ block: 'center' });
            await delay(80);
            moreBtn.click();
            
            let menuFound = false;
            for (let attempts = 0; attempts < 20; attempts++) {
              const menus = Array.from(document.querySelectorAll('.moreActions'));
              if (menus.some(m => m.querySelector('.sc-button-label'))) {
                menuFound = true;
                break;
              }
              await delay(100);
            }
            
            if (menuFound) {
              const menus = Array.from(document.querySelectorAll('.moreActions'));
              if (menus.length) {
                const btnRect = moreBtn.getBoundingClientRect();
                let bestMenu = null;
                let bestDist = Infinity;
                
                for (const menu of menus) {
                  const menuRect = menu.getBoundingClientRect();
                  const dx = Math.abs((menuRect.left + menuRect.width / 2) - (btnRect.left + btnRect.width / 2));
                  const dy = Math.abs((menuRect.top + menuRect.height / 2) - (btnRect.top + btnRect.height / 2));
                  const distance = Math.sqrt(dx * dx + dy * dy);
                  if (distance < bestDist) {
                    bestDist = distance;
                    bestMenu = menu;
                  }
                }
                
                if (bestMenu) {
                  const downloadBtn = bestMenu.querySelector('.sc-button-download, button[aria-label*="Download"]');
                  hasDownload = !!downloadBtn;
                }
              }
              
              document.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape' }));
              await delay(50);
            }
          } catch (e) {
            console.log(`Error checking download for item ${i + 1}:`, e.message);
          }
        }
        
        if (hasDownload && title !== 'Unknown Title' && trackUrl) {
          results.push({
            index: results.length + 1,
            artist,
            title,
            trackUrl,
            hasDownload: true
          });
        }
        
        await delay(150);
      }
      
      return results;
    });
    
    // Single summary with leading blank line
    console.log(`\nTotal items found: ${finalItemCount}`);
    console.log(`\nFound ${tracks.length} downloadable tracks:\n`);
    
    tracks.forEach((track) => {
      console.log(`${track.index}. ${track.artist} - "${track.title}"`);
      console.log(`   ${track.trackUrl}\n`);
    });
    
    return tracks;
    
  } catch (error) {
    console.error('Scraping failed:', error.message);
  } finally {
    if (browser) {
      await browser.close();
    }
  }
}

scrapeSearchResults();