300+ ટોપ ગુજરાતી site નું એનાલીસીસ

const fs = require('fs');
const { chromium } = require('playwright');

(async () => {
    // Launch the browser
    const browser = await chromium.launch({ headless: false });
    const page = await browser.newPage();

    // Navigate to the URL
    await page.goto('https://topgujaratiblog.wordpress.com/2017/06/07/top-gujarati-blog/');

    // Fetch all the anchor links
    const links = await page.$$eval('a', anchors => anchors.map(anchor => anchor.href));

    // Write the links to a text file
    fs.writeFile('links.txt', links.join('\n'), (err) => {
        if (err) throw err;
        console.log('Links have been saved to links.txt');
    });

    // Close the browser
    await browser.close();
})();

const fs = require('fs');

// Read the file and get the links
fs.readFile('links.txt', 'utf8', (err, data) => {
    if (err) throw err;

    // Split the file content by newlines to get each link form as an array
    const links = data.split('\n').filter(link => link.trim() !== '');

    // Remove duplicate links using Set
    const uniqueLinks = [...new Set(links)];

    // Write the unique links back to the file
    fs.writeFile('links.txt', uniqueLinks.join('\n'), (err) => {
        if (err) throw err;
        console.log('Duplicates removed and file updated with unique links.');
    });
});

const fs = require('fs');
const { chromium } = require('playwright');

(async () => {
    // Launch the browser
    const browser = await chromium.launch({ headless: false });
    const page = await browser.newPage();

    let allLinks = [];
    let currentPage = 1;
    let lastPage = 5; // Set this manually for now based on your observation

    while (currentPage <= lastPage) {
        // Navigate to the target URL with pagination
        const url = `https://www.indiblogger.in/tag/gujarati&p=${currentPage}`;
        await page.goto(url);
        console.log(`Fetching links from page ${currentPage}`);

        // Fetch the anchor tags within h3 with class media-heading
        const links = await page.$$eval('h3.media-heading a', anchors => anchors.map(anchor => anchor.href));
        allLinks.push(...links);

        // Check for pagination to get the number of pages (could be adjusted dynamically)
        // If you need to dynamically find the last page:
        // lastPage = await page.$$eval('ul.pagination li a', items => 
        //     Math.max(...items.map(i => parseInt(i.getAttribute('data-id')) || 0))
        // );

        // Increment to the next page
        currentPage++;
    }

    // Remove duplicate links using Set
    const uniqueLinks = [...new Set(allLinks)];

    // Write the unique links to a text file
    fs.writeFile('indiblogger_all_links.txt', uniqueLinks.join('\n'), (err) => {
        if (err) throw err;
        console.log('All links have been saved to indiblogger_all_links.txt');
    });

    // Close the browser
    await browser.close();
})();

const fs = require('fs');
const readline = require('readline');
const { chromium } = require('playwright');
const path = require('path');
const csvWriter = require('csv-writer').createObjectCsvWriter;

// CSV writer setup
const writer = csvWriter({
    path: 'url_check_results2.csv',
    header: [
        { id: 'url', title: 'URL' },
        { id: 'isWorking', title: 'Is Working?' },
    ]
});

// Setup readline for user input
const rl = readline.createInterface({
    input: process.stdin,
    output: process.stdout
});

// Gujarati character range (Unicode range for Gujarati characters)
const gujaratiRegex = /[\u0A80-\u0AFF]/;

// Function to check if a URL is working and contains Gujarati characters
async function checkUrl(page, url) {
    try {
        const response = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 5000 }); // Timeout after 5 seconds
        if (!response || response.status() !== 200) {
            return false;
        }

        // Get the page content and check for at least one Gujarati character
        const pageContent = await page.content();
        return gujaratiRegex.test(pageContent); // Return true if Gujarati character is found
    } catch (error) {
        return false; // If any error occurs, mark it as not working
    }
}

// Function to get manual user input (Yes/No)
async function getUserConfirmation(url) {
    return new Promise((resolve) => {
        rl.question(`Manual check required for URL: ${url}. Is it working? (Yes/No): `, (answer) => {
            resolve(answer.trim().toLowerCase() === 'yes');
        });
    });
}

// Main script to process the URLs
(async () => {
    // Read the links from the text file
    const filePath = path.join(__dirname, 'notworking_links.txt');
    const urls = fs.readFileSync(filePath, 'utf8').split('\n').filter(Boolean); // Split by newlines and filter out empty lines

    // Launch the browser in non-headless mode
    const browser = await chromium.launch({ headless: false });
    const page = await browser.newPage();

    let results = [];

    // Iterate over each URL and check if it's working
    for (let url of urls) {
        // for (let i = 38; i < urls.length; i++) {
        // let url = urls[i];

        if (!url.startsWith('http://') && !url.startsWith('https://')) {
            url = `http://${url}`;
        }
        let isWorking = await checkUrl(page, url);

        // If the HTTP version didn't work, try HTTPS
        if (!isWorking && url.startsWith('http://')) {
            url = url.replace('http://', 'https://');
            console.log(`Retrying with HTTPS: ${url}`);
            isWorking = await checkUrl(page, url);
        }

        // If automated check failed, ask user for confirmation
        if (!isWorking) {
            console.log(`Automated check failed for URL: ${url}`);
            isWorking = await getUserConfirmation(url);
        }

        // Add the result to the list
        results.push({ url, isWorking: isWorking ? 'Yes' : 'No' });
        console.log(`URL: ${url} - Is Working: ${isWorking ? 'Yes' : 'No'}`);
    }

    // Write the results to a CSV file
    await writer.writeRecords(results)
        .then(() => console.log('Results written to url_check_results.csv'));

    // Close the browser and readline
    await browser.close();
    rl.close();
})();

const { chromium } = require('playwright');
const fs = require('fs');
const path = require('path');
const csv = require('csv-parser');
const { parse } = require('json2csv');

// Read the CSV and process URLs
const processCSV = async (csvFilePath) => {
    const rows = [];

    // Read and parse the CSV file
    fs.createReadStream(csvFilePath)
        .pipe(csv())
        .on('data', (row) => {
            if (row['Is Working?'] === 'Yes') {
                rows.push({ url: row.URL });
            }
        })
        .on('end', async () => {
            // Process URLs for sitemap.xml and /feed/
            const processedData = await checkForSitemapAndFeed(rows);

            // Append sitemap and feed columns and save the new CSV file
            const newCSVData = processedData.map(item => ({
                URL: item.url,
                'Is Working?': 'Yes',
                'Has Sitemap?': item.hasSitemap ? 'Yes' : 'No',
                'Has Feed?': item.hasFeed ? 'Yes' : 'No'
            }));

            saveCSV(newCSVData, csvFilePath);
        });
};

// Check if sitemap.xml and /feed/ exist
const checkForSitemapAndFeed = async (urls) => {
    const browser = await chromium.launch({
        headless: false
    });
    const context = await browser.newContext({
        ignoreHTTPSErrors: true
    });
    const page = await context.newPage();

    const results = [];
    for (const { url } of urls) {
        const sitemapUrl = `${url.replace(/\/$/, '')}/sitemap.xml`;
        const feedUrl = `${url.replace(/\/$/, '')}/feed/`;

        let hasSitemap = false;
        let hasFeed = false;

        // Check sitemap.xml
        try {
            const sitemapResponse = await page.goto(sitemapUrl, { timeout: 10000 });
            if (sitemapResponse && sitemapResponse.status() === 200) {
                hasSitemap = true;
            }
        } catch (err) {
            // Ignore error if sitemap doesn't exist
        }

        // Check /feed/
        try {
            const feedResponse = await page.goto(feedUrl, { timeout: 10000 });
            if (feedResponse && feedResponse.status() === 200) {
                hasFeed = true;
            }
        } catch (err) {
            // Ignore error if feed doesn't exist
        }

        // Push results
        results.push({ url, hasSitemap, hasFeed });
    }

    await browser.close();
    return results;
};

// Save the updated CSV
const saveCSV = (data, originalFilePath) => {
    const fields = ['URL', 'Is Working?', 'Has Sitemap?', 'Has Feed?'];
    const csvData = parse(data, { fields });

    const outputFilePath = path.join(path.dirname(originalFilePath), 'updated_' + path.basename(originalFilePath));
    fs.writeFileSync(outputFilePath, csvData);
    console.log(`Updated CSV saved at: ${outputFilePath}`);
};

// File path to the deduplicated_urls.csv
const csvFilePath = './deduplicated_urls.csv';

// Start the process
processCSV(csvFilePath);

const { chromium } = require('playwright');
const fs = require('fs');
const path = require('path');
const csv = require('csv-parser');
const { parse } = require('json2csv');  // Keep it as parse(data) for now
const { XMLParser } = require('fast-xml-parser');

// Read the CSV and process URLs
const processCSV = async (csvFilePath) => {
  const rows = [];

  // Read and parse the CSV file
  fs.createReadStream(csvFilePath)
    .pipe(csv())
    .on('data', (row) => {
      rows.push({
        url: row.URL,
        isWorking: row['Is Working?'],
        hasSitemap: row['Has Sitemap?'],
        hasFeed: row['Has Feed?'],
        lastUpdated: ''
      });
    })
    .on('end', async () => {
      // Process URLs where 'Has Feed?' is 'Yes'
      const processedData = await fetchLatestPostDate(rows);

      // Save the updated CSV file with the latest post date
      saveCSV(processedData, csvFilePath);
    });
};

// Fetch latest post date from /feed/ if the feed exists
const fetchLatestPostDate = async (urls) => {
  const browser = await chromium.launch({ headless: false });
  const context = await browser.newContext({
    ignoreHTTPSErrors: true  // Ignore SSL certificate errors
  });
  const page = await context.newPage();
  const parser = new XMLParser();

  const results = [];
  for (const row of urls) {
    if (row.hasFeed === 'Yes') {
      const feedUrl = `${row.url.replace(/\/$/, '')}/feed/`;
      try {
        const response = await page.goto(feedUrl, { timeout: 10000 });

        // Only proceed if we have a valid XML response (check the content-type header)
        if (response && response.status() === 200 && response.headers()['content-type'].includes('xml')) {
          const xmlData = await response.text();
          const parsedData = parser.parse(xmlData);

          // Attempt to find the latest post date (depends on feed structure, assume RSS/Atom structure)
          const latestPostDate = extractLatestPostDate(parsedData);

          if (latestPostDate) {
            // Convert date to UTC-0
            const utcDate = new Date(latestPostDate).toISOString();  // Convert to UTC
            row.lastUpdated = utcDate;
          }
        }
      } catch (err) {
        // Ignore errors and leave lastUpdated blank
      }
    }
    results.push(row);
  }

  await browser.close();
  return results;
};

// Extract latest post date from the parsed XML (RSS/Atom feed formats)
const extractLatestPostDate = (parsedData) => {
  let latestPostDate = null;

  // Check for RSS feed format: `<rss><channel><item><pubDate>`
  if (parsedData.rss && parsedData.rss.channel && parsedData.rss.channel.item) {
    const items = Array.isArray(parsedData.rss.channel.item) ? parsedData.rss.channel.item : [parsedData.rss.channel.item];
    latestPostDate = items[0].pubDate || null;
  }

  // Check for Atom feed format: `<feed><entry><updated>`
  if (parsedData.feed && parsedData.feed.entry) {
    const entries = Array.isArray(parsedData.feed.entry) ? parsedData.feed.entry : [parsedData.feed.entry];
    latestPostDate = entries[0].updated || null;
  }

  return latestPostDate;
};

// Save the updated CSV
const saveCSV = (data, originalFilePath) => {
  // Use parse(data) since the fields option may not be needed
  const csvData = parse(data);  // Keep parse(data) as it's working for you

  const outputFilePath = path.join(path.dirname(originalFilePath), 'updated_' + path.basename(originalFilePath));
  fs.writeFileSync(outputFilePath, csvData);
  console.log(`Updated CSV saved at: ${outputFilePath}`);
};

// File path to the updated_deduplicated_urls.csv
const csvFilePath = './updated_deduplicated_urls.csv';

// Start the process
processCSV(csvFilePath);

const { chromium } = require('playwright');
const fs = require('fs');
const path = require('path');
const csv = require('csv-parser');
const { parse } = require('json2csv');
const { XMLParser } = require('fast-xml-parser');

// Read the CSV and process URLs
const processCSV = async (csvFilePath) => {
    const rows = [];

    // Read and parse the CSV file
    fs.createReadStream(csvFilePath)
        .pipe(csv())
        .on('data', (row) => {
            // Only process rows that do not have a lastUpdated value and have 'Has Sitemap?' as 'Yes' and 'Has Feed?' as 'No'
            if (!row.lastUpdated && row.hasSitemap === 'Yes') {
                rows.push({
                    url: row.url,
                    isWorking: row.isWorking,
                    hasSitemap: row.hasSitemap,
                    hasFeed: row.hasFeed,
                    lastUpdated: ''
                });
            }
        })
        .on('end', async () => {
            // Process the remaining URLs to fetch the latest post date from the sitemap
            const processedData = await fetchLatestPostDateFromSitemap(rows, csvFilePath);

            // Save the updated CSV file with the latest post date from sitemap
            saveCSV(processedData, csvFilePath);
        });
};

// Fetch latest post date from sitemap if the site has one and no feed
const fetchLatestPostDateFromSitemap = async (urls, csvFilePath) => {
    const browser = await chromium.launch({ headless: false });
    const context = await browser.newContext({ ignoreHTTPSErrors: true });
    const page = await context.newPage();
    const parser = new XMLParser();

    const results = [];

    for (const row of urls) {
        const sitemapUrl = `${row.url.replace(/\/$/, '')}/sitemap.xml`;
        try {
            const response = await page.goto(sitemapUrl, { timeout: 10000 });

            // Only proceed if we have a valid XML response (check the content-type header)
            if (response && response.status() === 200 && response.headers()['content-type'].includes('xml')) {
                const xmlData = await response.text();
                const parsedData = parser.parse(xmlData);

                // Extract the latest post date from the sitemap
                const latestPostDate = extractLatestPostDateFromSitemap(parsedData);

                if (latestPostDate) {
                    const utcDate = new Date(latestPostDate).toISOString();  // Convert to UTC
                    row.lastUpdated = utcDate;
                }
            }
        } catch (err) {
            // Ignore errors and leave lastUpdated blank
        }

        results.push(row);
    }

    await browser.close();
    return results;
};

// Extract latest post date from the parsed XML sitemap
const extractLatestPostDateFromSitemap = (parsedData) => {
    let latestPostDate = null;

    // Check for <urlset> structure in sitemap and extract <lastmod> dates
    if (parsedData.urlset && parsedData.urlset.url) {
        const urls = Array.isArray(parsedData.urlset.url) ? parsedData.urlset.url : [parsedData.urlset.url];

        // Find the latest <lastmod> date
        latestPostDate = urls.reduce((latest, url) => {
            const lastmod = url.lastmod;
            return lastmod && (!latest || new Date(lastmod) > new Date(latest)) ? lastmod : latest;
        }, null);
    }

    return latestPostDate;
};

// Save the updated CSV
const saveCSV = (newData, originalFilePath) => {
    // Read the original CSV again to include all rows, including those that were skipped (no sitemap or had feed)
    const updatedData = [];
    fs.createReadStream(originalFilePath)
        .pipe(csv())
        .on('data', (row) => {
            // Update the lastUpdated field for the rows that have been processed in the sitemap
            const updatedRow = newData.find((newRow) => newRow.url === row.url);
            if (updatedRow) {
                row.lastUpdated = updatedRow.lastUpdated;
            }
            updatedData.push(row);
        })
        .on('end', () => {
            // Use parse(data) since the fields option may not be needed
            const csvData = parse(updatedData);  // Keep parse(data) as it's working for you

            const outputFilePath = path.join(path.dirname(originalFilePath), 'updated_' + path.basename(originalFilePath));
            fs.writeFileSync(outputFilePath, csvData);
            console.log(`Updated CSV saved at: ${outputFilePath}`);
        });
};

// File path to the updated_updated_deduplicated_urls.csv
const csvFilePath = './updated_updated_deduplicated_urls.csv';

// Start the process
processCSV(csvFilePath);