300+ ટોપ ગુજરાતી site નું એનાલીસીસ

const fs = require('fs');
const { chromium } = require('playwright');

(async () => {
    // Launch the browser
    const browser = await chromium.launch({ headless: false });
    const page = await browser.newPage();

    // Navigate to the URL
    await page.goto('https://topgujaratiblog.wordpress.com/2017/06/07/top-gujarati-blog/');

    // Fetch all the anchor links
    const links = await page.$$eval('a', anchors => anchors.map(anchor => anchor.href));

    // Write the links to a text file
    fs.writeFile('links.txt', links.join('\n'), (err) => {
        if (err) throw err;
        console.log('Links have been saved to links.txt');
    });

    // Close the browser
    await browser.close();
})();

const fs = require('fs');

// Read the file and get the links
fs.readFile('links.txt', 'utf8', (err, data) => {
    if (err) throw err;

    // Split the file content by newlines to get each link form as an array
    const links = data.split('\n').filter(link => link.trim() !== '');

    // Remove duplicate links using Set
    const uniqueLinks = [...new Set(links)];

    // Write the unique links back to the file
    fs.writeFile('links.txt', uniqueLinks.join('\n'), (err) => {
        if (err) throw err;
        console.log('Duplicates removed and file updated with unique links.');
    });
});

const fs = require('fs');
const { chromium } = require('playwright');

(async () => {
    // Launch the browser
    const browser = await chromium.launch({ headless: false });
    const page = await browser.newPage();

    let allLinks = [];
    let currentPage = 1;
    let lastPage = 5; // Set this manually for now based on your observation

    while (currentPage <= lastPage) {
        // Navigate to the target URL with pagination
        const url = `https://www.indiblogger.in/tag/gujarati&p=${currentPage}`;
        await page.goto(url);
        console.log(`Fetching links from page ${currentPage}`);

        // Fetch the anchor tags within h3 with class media-heading
        const links = await page.$$eval('h3.media-heading a', anchors => anchors.map(anchor => anchor.href));
        allLinks.push(...links);

        // Check for pagination to get the number of pages (could be adjusted dynamically)
        // If you need to dynamically find the last page:
        // lastPage = await page.$$eval('ul.pagination li a', items => 
        //     Math.max(...items.map(i => parseInt(i.getAttribute('data-id')) || 0))
        // );

        // Increment to the next page
        currentPage++;
    }

    // Remove duplicate links using Set
    const uniqueLinks = [...new Set(allLinks)];

    // Write the unique links to a text file
    fs.writeFile('indiblogger_all_links.txt', uniqueLinks.join('\n'), (err) => {
        if (err) throw err;
        console.log('All links have been saved to indiblogger_all_links.txt');
    });

    // Close the browser
    await browser.close();
})();

const fs = require('fs');
const readline = require('readline');
const { chromium } = require('playwright');
const path = require('path');
const csvWriter = require('csv-writer').createObjectCsvWriter;

// CSV writer setup
const writer = csvWriter({
    path: 'url_check_results2.csv',
    header: [
        { id: 'url', title: 'URL' },
        { id: 'isWorking', title: 'Is Working?' },
    ]
});

// Setup readline for user input
const rl = readline.createInterface({
    input: process.stdin,
    output: process.stdout
});

// Gujarati character range (Unicode range for Gujarati characters)
const gujaratiRegex = /[\u0A80-\u0AFF]/;

// Function to check if a URL is working and contains Gujarati characters
async function checkUrl(page, url) {
    try {
        const response = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 5000 }); // Timeout after 5 seconds
        if (!response || response.status() !== 200) {
            return false;
        }

        // Get the page content and check for at least one Gujarati character
        const pageContent = await page.content();
        return gujaratiRegex.test(pageContent); // Return true if Gujarati character is found
    } catch (error) {
        return false; // If any error occurs, mark it as not working
    }
}

// Function to get manual user input (Yes/No)
async function getUserConfirmation(url) {
    return new Promise((resolve) => {
        rl.question(`Manual check required for URL: ${url}. Is it working? (Yes/No): `, (answer) => {
            resolve(answer.trim().toLowerCase() === 'yes');
        });
    });
}

// Main script to process the URLs
(async () => {
    // Read the links from the text file
    const filePath = path.join(__dirname, 'notworking_links.txt');
    const urls = fs.readFileSync(filePath, 'utf8').split('\n').filter(Boolean); // Split by newlines and filter out empty lines

    // Launch the browser in non-headless mode
    const browser = await chromium.launch({ headless: false });
    const page = await browser.newPage();

    let results = [];

    // Iterate over each URL and check if it's working
    for (let url of urls) {
        // for (let i = 38; i < urls.length; i++) {
        // let url = urls[i];

        if (!url.startsWith('http://') && !url.startsWith('https://')) {
            url = `http://${url}`;
        }
        let isWorking = await checkUrl(page, url);

        // If the HTTP version didn't work, try HTTPS
        if (!isWorking && url.startsWith('http://')) {
            url = url.replace('http://', 'https://');
            console.log(`Retrying with HTTPS: ${url}`);
            isWorking = await checkUrl(page, url);
        }

        // If automated check failed, ask user for confirmation
        if (!isWorking) {
            console.log(`Automated check failed for URL: ${url}`);
            isWorking = await getUserConfirmation(url);
        }

        // Add the result to the list
        results.push({ url, isWorking: isWorking ? 'Yes' : 'No' });
        console.log(`URL: ${url} - Is Working: ${isWorking ? 'Yes' : 'No'}`);
    }

    // Write the results to a CSV file
    await writer.writeRecords(results)
        .then(() => console.log('Results written to url_check_results.csv'));

    // Close the browser and readline
    await browser.close();
    rl.close();
})();

const { chromium } = require('playwright');
const fs = require('fs');
const path = require('path');
const csv = require('csv-parser');
const { parse } = require('json2csv');

// Read the CSV and process URLs
const processCSV = async (csvFilePath) => {
    const rows = [];

    // Read and parse the CSV file
    fs.createReadStream(csvFilePath)
        .pipe(csv())
        .on('data', (row) => {
            if (row['Is Working?'] === 'Yes') {
                rows.push({ url: row.URL });
            }
        })
        .on('end', async () => {
            // Process URLs for sitemap.xml and /feed/
            const processedData = await checkForSitemapAndFeed(rows);

            // Append sitemap and feed columns and save the new CSV file
            const newCSVData = processedData.map(item => ({
                URL: item.url,
                'Is Working?': 'Yes',
                'Has Sitemap?': item.hasSitemap ? 'Yes' : 'No',
                'Has Feed?': item.hasFeed ? 'Yes' : 'No'
            }));

            saveCSV(newCSVData, csvFilePath);
        });
};

// Check if sitemap.xml and /feed/ exist
const checkForSitemapAndFeed = async (urls) => {
    const browser = await chromium.launch({
        headless: false
    });
    const context = await browser.newContext({
        ignoreHTTPSErrors: true
    });
    const page = await context.newPage();

    const results = [];
    for (const { url } of urls) {
        const sitemapUrl = `${url.replace(/\/$/, '')}/sitemap.xml`;
        const feedUrl = `${url.replace(/\/$/, '')}/feed/`;

        let hasSitemap = false;
        let hasFeed = false;

        // Check sitemap.xml
        try {
            const sitemapResponse = await page.goto(sitemapUrl, { timeout: 10000 });
            if (sitemapResponse && sitemapResponse.status() === 200) {
                hasSitemap = true;
            }
        } catch (err) {
            // Ignore error if sitemap doesn't exist
        }

        // Check /feed/
        try {
            const feedResponse = await page.goto(feedUrl, { timeout: 10000 });
            if (feedResponse && feedResponse.status() === 200) {
                hasFeed = true;
            }
        } catch (err) {
            // Ignore error if feed doesn't exist
        }

        // Push results
        results.push({ url, hasSitemap, hasFeed });
    }

    await browser.close();
    return results;
};

// Save the updated CSV
const saveCSV = (data, originalFilePath) => {
    const fields = ['URL', 'Is Working?', 'Has Sitemap?', 'Has Feed?'];
    const csvData = parse(data, { fields });

    const outputFilePath = path.join(path.dirname(originalFilePath), 'updated_' + path.basename(originalFilePath));
    fs.writeFileSync(outputFilePath, csvData);
    console.log(`Updated CSV saved at: ${outputFilePath}`);
};

// File path to the deduplicated_urls.csv
const csvFilePath = './deduplicated_urls.csv';

// Start the process
processCSV(csvFilePath);