Node.js Expired Domain Checker

proxyyard · Nov 22, 2020

Here's a backlink checker that checks if the domain is expired (ENOTFOUND).
Simply scrape the sitemap list from a site (in our example techcrunch.com) using

https://techcrunch.com/sitemap-index-1.xmlcopy([].slice.call(document.querySelectorAll('table tr td:nth-child(2)')).map(elem => elem.innerText))

JavaScript:

const https = require('https');
const xml2js = require('xml2js');
const parser = new xml2js.Parser({
    attrkey: "ATTR"
});

let visited = {};

const checkLink = (url) => {
    //console.log(url);
    url = url.replace('http:', 'https:');
    visited[url] = true;
    return new Promise((resolve) => {
        https.get(url, (res) => {
            let html = '';
            res.on('data', (d) => {
                html += d;
            });
            res.on('end', (d) => {
                resolve({
                    url,
                    statusCode: res.statusCode,
                    headers: res.headers,
                    html
                });
            });
        }).on('error', (e) => {
            resolve({
                url,
                statusCode: -1,
                undefined,
                undefined,
            });
        });
    });
}

const URLS = [
  "https://techcrunch.com/sitemap-2.xml",
  "https://techcrunch.com/sitemap-4.xml",
  "https://techcrunch.com/sitemap-5.xml",
  "https://techcrunch.com/sitemap-6.xml",
  "https://techcrunch.com/sitemap-7.xml",
  "https://techcrunch.com/sitemap-8.xml",
  "https://techcrunch.com/sitemap-10.xml",
  "https://techcrunch.com/sitemap-11.xml",
  "https://techcrunch.com/sitemap-12.xml",
  "https://techcrunch.com/sitemap-14.xml",
  "https://techcrunch.com/sitemap-15.xml",
  "https://techcrunch.com/sitemap-16.xml",
  "https://techcrunch.com/sitemap-17.xml",
  "https://techcrunch.com/sitemap-19.xml",
  "https://techcrunch.com/sitemap-20.xml",
  "https://techcrunch.com/sitemap-21.xml",
  "https://techcrunch.com/sitemap-22.xml",
  "https://techcrunch.com/sitemap-25.xml",
  "https://techcrunch.com/sitemap-27.xml",
  "https://techcrunch.com/sitemap-29.xml",
  "https://techcrunch.com/sitemap-31.xml",
  "https://techcrunch.com/sitemap-32.xml",
  "https://techcrunch.com/sitemap-37.xml",
  "https://techcrunch.com/sitemap-38.xml",
  "https://techcrunch.com/sitemap-42.xml",
  "https://techcrunch.com/sitemap-47.xml",
  "https://techcrunch.com/sitemap-48.xml",
  "https://techcrunch.com/sitemap-54.xml",
  "https://techcrunch.com/sitemap-55.xml",
  "https://techcrunch.com/sitemap-9.xml",
  "https://techcrunch.com/sitemap-13.xml",
  "https://techcrunch.com/sitemap-18.xml",
  "https://techcrunch.com/sitemap-24.xml",
  "https://techcrunch.com/sitemap-33.xml",
  "https://techcrunch.com/sitemap-34.xml",
  "https://techcrunch.com/sitemap-39.xml",
  "https://techcrunch.com/sitemap-41.xml",
  "https://techcrunch.com/sitemap-43.xml",
  "https://techcrunch.com/sitemap-44.xml",
  "https://techcrunch.com/sitemap-45.xml",
  "https://techcrunch.com/sitemap-46.xml",
  "https://techcrunch.com/sitemap-49.xml",
  "https://techcrunch.com/sitemap-50.xml",
  "https://techcrunch.com/sitemap-51.xml",
  "https://techcrunch.com/sitemap-52.xml",
  "https://techcrunch.com/sitemap-53.xml",
  "https://techcrunch.com/sitemap-57.xml",
  "https://techcrunch.com/sitemap-59.xml",
  "https://techcrunch.com/sitemap-62.xml",
  "https://techcrunch.com/sitemap-63.xml",
  "https://techcrunch.com/sitemap-64.xml",
  "https://techcrunch.com/sitemap-66.xml",
  "https://techcrunch.com/sitemap-67.xml",
  "https://techcrunch.com/sitemap-69.xml",
  "https://techcrunch.com/sitemap-70.xml",
  "https://techcrunch.com/sitemap-71.xml",
  "https://techcrunch.com/sitemap-72.xml",
  "https://techcrunch.com/sitemap-73.xml",
  "https://techcrunch.com/sitemap-75.xml",
  "https://techcrunch.com/sitemap-3.xml",
  "https://techcrunch.com/sitemap-35.xml",
  "https://techcrunch.com/sitemap-60.xml",
  "https://techcrunch.com/sitemap-61.xml",
  "https://techcrunch.com/sitemap-65.xml",
  "https://techcrunch.com/sitemap-1.xml",
  "https://techcrunch.com/sitemap-23.xml",
  "https://techcrunch.com/sitemap-26.xml",
  "https://techcrunch.com/sitemap-28.xml",
  "https://techcrunch.com/sitemap-30.xml",
  "https://techcrunch.com/sitemap-36.xml",
  "https://techcrunch.com/sitemap-40.xml",
  "https://techcrunch.com/sitemap-56.xml",
  "https://techcrunch.com/sitemap-58.xml",
  "https://techcrunch.com/sitemap-68.xml",
  "https://techcrunch.com/sitemap-74.xml",
  "https://techcrunch.com/sitemap-76.xml",
  "https://techcrunch.com/sitemap-77.xml",
  "https://techcrunch.com/sitemap-78.xml",
  "https://techcrunch.com/sitemap-79.xml",
  "https://techcrunch.com/sitemap-80.xml",
  "https://techcrunch.com/sitemap-81.xml",
  "https://techcrunch.com/sitemap-82.xml",
  "https://techcrunch.com/sitemap-83.xml",
  "https://techcrunch.com/sitemap-84.xml",
  "https://techcrunch.com/sitemap-85.xml",
  "https://techcrunch.com/sitemap-86.xml",
  "https://techcrunch.com/sitemap-87.xml",
  "https://techcrunch.com/sitemap-88.xml",
  "https://techcrunch.com/sitemap-89.xml",
  "https://techcrunch.com/sitemap-90.xml",
  "https://techcrunch.com/sitemap-91.xml",
  "https://techcrunch.com/sitemap-92.xml",
  "https://techcrunch.com/sitemap-93.xml",
  "https://techcrunch.com/sitemap-94.xml",
  "https://techcrunch.com/sitemap-95.xml",
  "https://techcrunch.com/sitemap-96.xml",
  "https://techcrunch.com/sitemap-97.xml",
  "https://techcrunch.com/sitemap-98.xml",
  "https://techcrunch.com/sitemap-99.xml",
  "https://techcrunch.com/sitemap-100.xml",
  "https://techcrunch.com/sitemap-101.xml",
  "https://techcrunch.com/sitemap-102.xml",
  "https://techcrunch.com/sitemap-103.xml",
  "https://techcrunch.com/sitemap-104.xml",
  "https://techcrunch.com/sitemap-105.xml"
].reverse();
const dns = require('dns');
//let URL = 'https://www.latimes.com/sitemap-201401.xml';

const rec = async(obj) => {
    if (obj.statusCode === 200 && obj.url.indexOf('techcrunch.') > 0) {
        const arr = [...obj.html.matchAll(/href\=\"[^\"]+\"/g)].map(s => s[0].substring(6, s[0].length - 1));
        for (let i = 0; i < arr.length; ++i) {
            if (arr[i].indexOf('https://') === 0 && arr[i].indexOf('techcrunch.com') === -1 && arr[i].indexOf('twitter.') === -1 && arr[i].indexOf('tumblr.') === -1 && arr[i].indexOf('facebook.') === -1 && arr[i].indexOf('.css') === -1 && arr[i].indexOf('.js') === -1 && !visited[arr[i]]) {
                //console.log(arr[i]);
                await rec(await checkLink(arr[i]));
            }
        }
    } else if (obj.statusCode === 301 || obj.statusCode === 302) {
        await rec(await checkLink('https://www.techcrunch.com' + obj.headers.location));
    } else if (obj.statusCode > 399 && obj.statusCode < 406 && obj.url.indexOf('techcrunch.') === -1) {
        let url = obj.url;
        url = url.substring(url.indexOf('://') + 3, url.substring(8).indexOf('/') > -1 ? url.substring(8).indexOf('/') + 8 : url.length);
        visited[url] = true;
        dns.resolve(url, (err, r) => {
            console.log(err, url);
        });
    }
}

const root = (url) => {
    console.log(url);
    return new Promise(async(resolve) => {
        const xml_string = (await checkLink(url)).html;
        parser.parseString(xml_string, async(error, result) => {
            if (error === null) {
                let i = 0;
                let urls = result.urlset.url;
                const interval = setInterval(async() => {
                    if (i >= urls.length) {
                        clearInterval(interval);
                        return;
                    }
                    await rec(await checkLink(urls[i++]?.loc[0]));
                }, 1000);
            } else {
                console.log(error);
                resolve();
            }
        });
    });
}

let i = 0;
setInterval(async() => {
        await root(URLS[i++]);
},5000);

Paste the urls into the "URLS" array in bl-checker.js.
Then node bl-checker.js.

esmeralda · Nov 23, 2020

Nice tool

Thanks a lot for sharing.

Node.js Expired Domain Checker

proxyyard

Newbie

esmeralda

Junior Member

Main Menu

Marketplace

Making Money

BlackHat World