cdransf/coryd.dev-eleventy
Archived
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

chore: bad bots

Browse source
This commit is contained in:
Cory Dransfeldt 2024-03-17 19:18:21 -07:00
parent 249f0fc75c
commit 85e3019ccd
No known key found for this signature in database
2 changed files with 651 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
package.json
View file

@ -1,6 +1,6 @@
{ {
"name": "coryd.dev", "name": "coryd.dev",
"version": "7.10.11", "version": "7.11.11",
"description": "The source for my personal site. Built using 11ty.", "description": "The source for my personal site. Built using 11ty.",
"type": "module", "type": "module",
"scripts": { "scripts": {

656
src/_data/json/robots.js
View file

@ -1,25 +1,669 @@
export default [ export default [
'01h4x.com',
'360Spider',
'404checker',
'404enemy',
'80legs',
'ADmantX',
'AIBOT',
'ALittle Client',
'ASPSeek',
'Abonti',
'Aboundex',
'Aboundexbot',
'Acunetix',
'AdsBot-Google', 'AdsBot-Google',
'AdsTxtCrawlerTP',
'AfD-Verbotsverfahren',
'AhrefsBot',
'AiHitBot',
'Aipbot',
'Alexibot',
'AllSubmitter',
'Alligator',
'AlphaBot',
'Amazonbot', 'Amazonbot',
'Anarchie',
'Anarchy',
'Anarchy99',
'Ankit',
'Anthill',
'anthropic-ai', 'anthropic-ai',
'Apexoo',
'Aspiegel',
'Asterias',
'Atomseobot',
'Attach',
'AwarioRssBot', 'AwarioRssBot',
'AwarioSmartBot', 'AwarioSmartBot',
'BBBike',
'BDCbot',
'BDFetch',
'BLEXBot',
'BackDoorBot',
'BackStreet',
'BackWeb',
'Backlink-Ceck',
'BacklinkCrawler',
'Badass',
'Bandit',
'Barkrowler',
'BatchFTP',
'Battleztar Bazinga',
'BetaBot',
'Bigfoot',
'Bitacle',
'BlackWidow',
'Black Hole',
'Blackboard',
'Blow',
'BlowFish',
'Boardreader',
'Bolt',
'BotALot',
'Brandprotect',
'Brandwatch',
'Buck',
'Buddy',
'BuiltBotTough',
'BuiltWith',
'Bullseye',
'BunnySlippers',
'BuzzSumo',
'Bytespider', 'Bytespider',
'CATExplorador',
'CCBot', 'CCBot',
'CODE87',
'CSHttp',
'Calculon',
'CazoodleBot',
'Cegbfeieh',
'CensysInspect',
'ChatGPT-User', 'ChatGPT-User',
'CheTeam',
'CheeseBot',
'CherryPicker',
'ChinaClaw',
'Chlooe',
'Citoid',
'Claritybot',
'ClaudeBot', 'ClaudeBot',
'Claude-Web', 'Claude-Web',
'cohere-ai', 'Cliqzbot',
'DataForSeoBot', 'Cloud mapping',
'Cocolyzebot',
'Cogentbot',
'Collector',
'Copier',
'CopyRightCheck',
'Copyscape',
'Cosmos',
'Craftbot',
'Crawling at Home Project',
'CrazyWebCrawler',
'Crescent',
'CrunchBot',
'Curious',
'Custo',
'CyotekWebCopy',
'DBLBot',
'DIIbot',
'DSearch',
'DTS Agent',
'DataCha0s',
'DatabaseDriverMysqli',
'Demon',
'Deusu',
'Devil',
'Digincore',
'DigitalPebble',
'Dirbuster',
'Disco',
'Discobot',
'Discoverybot',
'Dispatch',
'DittoSpyder',
'DnBCrawler-Analytics',
'DnyzBot',
'DomCopBot',
'DomainAppender',
'DomainCrawler',
'DomainSigmaCrawler',
'DomainStatsBot',
'Domains Project',
'Dotbot',
'Download Wonder',
'Dragonfly',
'Drip',
'ECCP/1.0',
'EMail Siphon',
'EMail Wolf',
'EasyDL',
'Ebingbong',
'Ecxi',
'EirGrabber',
'EroCrawler',
'Evil',
'Exabot',
'Express WebPictures',
'ExtLinksBot',
'Extractor',
'ExtractorPro',
'Extreme Picture Finder',
'EyeNetIE',
'Ezooms',
'FDM',
'FHscan',
'FacebookBot', 'FacebookBot',
'Google-Extended', 'FemtosearchBot',
'Fimap',
'Firefox/7.0',
'FlashGet',
'Flunky',
'Foobot',
'Freeuploader',
'FrontPage',
'Fuzz',
'FyberSpider',
'Fyrebot',
'G-i-g-a-b-o-t',
'GPTBot', 'GPTBot',
'GT::WWW',
'GalaxyBot',
'Genieo',
'GermCrawler',
'GetRight',
'GetWeb',
'Getintent',
'Gigabot',
'Go!Zilla',
'Go-Ahead-Got-It',
'GoZilla',
'Google-Extended',
'Gotit',
'GrabNet',
'Grabber',
'Grafula',
'GrapeFX',
'GrapeshotCrawler',
'GridBot',
'HEADMasterSEO',
'HMView',
'HTMLparser',
'HTTP::Lite',
'HTTrack',
'Haansoft',
'HaosouSpider',
'Harvest',
'Havij',
'Heritrix',
'Hloader',
'HonoluluBot',
'Humanlinks',
'HybridBot',
'IDBTE4M',
'IDBot',
'IRLbot',
'Iblog',
'Id-search',
'IlseBot',
'Image Fetch',
'Image Sucker',
'ImagesiftBot', 'ImagesiftBot',
'magpie-crawler', 'IndeedBot',
'omgili', 'Indy Library',
'omgilibot', 'InfoNaviRobot',
'InfoTekies',
'Intelliseek',
'InterGET',
'InternetSeer',
'Internet Ninja',
'Iria',
'Iskanie',
'IstellaBot',
'JOC Web Spider',
'JamesBOT',
'Jbrofuzz',
'JennyBot',
'JetCar',
'Jetty',
'JikeSpider',
'Joomla',
'Jorgee',
'JustView',
'Jyxobot',
'Kenjin Spider',
'Keybot Translation-Search-Machine',
'Keyword Density',
'Kinza',
'Kozmosbot',
'LNSpiderguy',
'LWP::Simple',
'Lanshanbot',
'Larbin',
'Leap',
'LeechFTP',
'LeechGet',
'LexiBot',
'Lftp',
'LibWeb',
'Libwhisker',
'LieBaoFast',
'Lightspeedsystems',
'Likse',
'LinkScan',
'LinkWalker',
'Linkbot',
'LinkextractorPro',
'LinkpadBot',
'LinksManager',
'LinqiaMetadataDownloaderBot',
'LinqiaRSSBot',
'LinqiaScrapeBot',
'Lipperhey',
'Lipperhey Spider',
'Litemage_walker',
'Lmspider',
'Ltx71',
'MFC_Tear_Sample',
'MIDown tool',
'MIIxpc',
'MJ12bot',
'MQQBrowser',
'MSFrontPage',
'MSIECrawler',
'MTRobot',
'Mag-Net',
'Magnet',
'Mail.RU_Bot',
'Majestic-SEO',
'Majestic12',
'Majestic SEO',
'MarkMonitor',
'MarkWatch',
'Mass Downloader',
'Masscan',
'Mata Hari',
'MauiBot',
'Mb2345Browser',
'MeanPath Bot',
'Meanpathbot',
'Mediatoolkitbot',
'MegaIndex.ru',
'Metauri',
'MicroMessenger',
'Microsoft Data Access',
'Microsoft URL Control',
'Minefield',
'Mister PiX',
'Moblie Safari',
'Mojeek',
'Mojolicious',
'MolokaiBot',
'Morfeus Fucking Scanner',
'Mozlila',
'Mr.4x3',
'Msrabot',
'Musobot',
'NICErsPRO',
'NPbot',
'Name Intelligence',
'Nameprotect',
'Navroad',
'NearSite',
'Needle',
'Nessus',
'NetAnts',
'NetLyzer',
'NetMechanic',
'NetSpider',
'NetZIP',
'Net Vampire',
'Netcraft',
'Nettrack',
'Netvibes',
'NextGenSearchBot',
'Nibbler',
'Niki-bot',
'Nikto',
'NimbleCrawler',
'Nimbostratus',
'Ninja',
'Nmap',
'Nuclei',
'Nutch',
'Octopus',
'Offline Explorer',
'Offline Navigator',
'OnCrawl',
'OpenLinkProfiler',
'OpenVAS',
'Openfind',
'Openvas',
'OrangeBot',
'OrangeSpider',
'OutclicksBot',
'OutfoxBot',
'PECL::HTTP',
'PHPCrawl',
'POE-Component-Client-HTTP',
'PageAnalyzer',
'PageGrabber',
'PageScorer',
'PageThing.com',
'Page Analyzer',
'Pandalytics',
'Panscient',
'Papa Foto',
'Pavuk',
'PeoplePal',
'peer39_crawler', 'peer39_crawler',
'peer39_crawler/1.0', 'peer39_crawler/1.0',
'PerplexityBot', 'PerplexityBot',
'Petalbot',
'Pi-Monster',
'Picscout',
'Picsearch',
'PictureFinder',
'Piepmatz',
'Pimonster',
'Pixray',
'PleaseCrawl',
'Pockey',
'ProPowerBot',
'ProWebWalker',
'Probethenet',
'Proximic',
'Psbot',
'Pu_iN',
'Pump',
'PxBroker',
'PyCurl',
'QueryN Metasearch',
'Quick-Crawler',
'RSSingBot',
'Rainbot',
'RankActive',
'RankActiveLinkBot',
'RankFlex',
'RankingBot',
'RankingBot2',
'Rankivabot',
'RankurBot',
'Re-re',
'ReGet',
'RealDownload',
'Reaper',
'RebelMouse',
'Recorder',
'RedesScrapy',
'RepoMonkey',
'Ripper',
'RocketCrawler',
'Rogerbot',
'SBIder',
'SEOkicks',
'SEOkicks-Robot',
'SEOlyticsCrawler',
'SEOprofiler',
'SEOstats',
'SISTRIX',
'SMTBot',
'SalesIntelligent',
'ScanAlert',
'Scanbot',
'ScoutJet',
'Scrapy',
'Screaming',
'ScreenerBot',
'ScrepyBot',
'Searchestate',
'SearchmetricsBot',
'Seekport',
'SeekportBot',
'SemanticJuice',
'Semrush',
'SemrushBot',
'SentiBot',
'SenutoBot',
'SeoSiteCheckup',
'SeobilityBot',
'Seomoz',
'Shodan',
'Siphon',
'SiteCheckerBotCrawler',
'SiteExplorer',
'SiteLockSpider',
'SiteSnagger',
'SiteSucker',
'Site Sucker',
'Sitebeam',
'Siteimprove',
'Sitevigil',
'SlySearch',
'SmartDownload',
'Snake',
'Snapbot',
'Snoopy',
'SocialRankIOBot',
'Sociscraper',
'Sogou web spider',
'Sosospider',
'Sottopop',
'SpaceBison',
'Spammen',
'SpankBot',
'Spanner',
'Spbot',
'Spinn3r',
'SputnikBot',
'Sqlmap',
'Sqlworm',
'Sqworm',
'Steeler',
'Stripper',
'Sucker',
'Sucuri',
'SuperBot',
'SuperHTTP',
'Surfbot',
'SurveyBot',
'Suzuran',
'Swiftbot',
'Szukacz',
'T0PHackTeam',
'T8Abot',
'Teleport',
'TeleportPro',
'Telesoft',
'Telesphoreo',
'Telesphorep',
'TheNomad',
'The Intraformant',
'Thumbor',
'TightTwatBot',
'TinyTestBot',
'Titan',
'Toata',
'Toweyabot',
'Tracemyfile',
'Trendiction',
'Trendictionbot',
'True_Robot',
'Turingos',
'Turnitin',
'TurnitinBot',
'TwengaBot',
'Twice',
'Typhoeus',
'URLy.Warning',
'URLy Warning',
'UnisterBot',
'Upflow',
'V-BOT',
'VB Project',
'VCI',
'Vacuum',
'Vagabondo',
'VelenPublicWebCrawler',
'VeriCiteCrawler',
'VidibleScraper',
'Virusdie',
'VoidEYE',
'Voil',
'Voltron',
'WASALive-Bot',
'WBSearchBot',
'WEBDAV',
'WISENutbot',
'WPScan',
'WWW-Collector-E',
'WWW-Mechanize',
'WWW::Mechanize',
'WWWOFFLE',
'Wallpapers',
'Wallpapers/3.0',
'WallpapersHD',
'WeSEE',
'WebAuto',
'WebBandit',
'WebCollage',
'WebCopier',
'WebEnhancer',
'WebFetch',
'WebFuck',
'WebGo IS',
'WebImageCollector',
'WebLeacher',
'WebPix',
'WebReaper',
'WebSauger',
'WebStripper',
'WebSucker',
'WebWhacker',
'WebZIP',
'Web Auto',
'Web Collage',
'Web Enhancer',
'Web Fetch',
'Web Fuck',
'Web Pix',
'Web Sauger',
'Web Sucker',
'Webalta',
'WebmasterWorldForumBot',
'Webshag',
'WebsiteExtractor',
'WebsiteQuester',
'Website Quester',
'Webster',
'Whack',
'Whacker',
'Whatweb',
'Who.is Bot',
'Widow',
'WinHTTrack',
'WiseGuys Robot',
'Wonderbot',
'Woobot',
'Wotbox',
'Wprecon',
'Xaldon WebSpider',
'Xaldon_WebSpider',
'Xenu',
'YoudaoBot',
'Zade',
'Zauba',
'Zermelo',
'Zeus',
'Zitebot',
'ZmEu',
'ZoomBot',
'ZoominfoBot',
'ZumBot',
'ZyBorg',
'adscanner',
'anthropic-ai',
'archive.org_bot',
'arquivo-web-crawler',
'arquivo.pt',
'autoemailspider',
'backlink-check',
'cah.io.community',
'check1.exe',
'clark-crawler',
'coccocbot',
'cognitiveseo',
'cohere-ai',
'com.plumanalytics',
'crawl.sogou.com',
'crawler.feedback',
'crawler4j',
'dataforseo.com',
'dataforseobot',
'demandbase-bot',
'domainsproject.org',
'eCatch',
'evc-batch',
'FacebookBot',
'facebookscraper',
'gopher',
'heritrix',
'imagesift.com',
'instabid',
'internetVista monitor',
'ips-agent',
'isitwp.com',
'iubenda-radar',
'linkdexbot',
'lwp-request',
'lwp-trivial',
'magpie-crawler',
'meanpathbot',
'mediawords',
'muhstik-scan',
'netEstate NE Crawler',
'oBot',
'omgili',
'omgilibot',
'openai',
'openai.com',
'page scorer',
'pcBrowser',
'plumanalytics',
'polaris version',
'probe-image-size',
'ripz',
'SEMrushBot',
's1z.ru',
'satoristudio.net',
'scalaj-http',
'scan.lol',
'seobility',
'seocompany.store',
'seoscanners',
'seostar',
'serpstatbot',
'sexsearcher',
'sitechecker.pro',
'siteripz',
'sogouspider',
'sp_auditbot',
'spyfu',
'sysscan',
'tAkeOut',
'trendiction.com',
'trendiction.de',
'ubermetrics-technologies.com',
'voyagerx.com',
'webgains-bot',
'webmeup-crawler',
'webpros.com',
'webprosbot',
'x09Mozilla',
'x22Mozilla',
'xpymep1.exe',
'YouBot', 'YouBot',
'zauba.io',
'zgrab',
] ]