diff --git a/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-10000.txt b/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-10000.txt index a89d7c1c..8e4c1ae7 100644 --- a/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-10000.txt +++ b/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-10000.txt @@ -28085,7 +28085,7 @@ GuruFiles/ Gurufiles/ H HACAffiliate -HBO/ +HBO/ HK/ HM HMDThankYou @@ -41081,7 +41081,7 @@ actuator actuator/http://pear.php.net/robots.txt ad ad-blocker-performance-study-in-response-to-manifest-v3-finds-that-ghosterys-ad-blocker-beats-the-competition -ad-manager/ +ad-manager/ ad-retargeting-101-online-shopping-in-the-covid-19-era ad-scans ad-test @@ -41821,7 +41821,7 @@ agile064 agile065 agile066 agile067 -agility/ +agility/ aging_surprises_getting_older/article.htm aglassie aglynn @@ -43120,7 +43120,7 @@ anewleaf anfrage anfrage-oeffnungszeiten ang -anga/ +anga/ angebot angebot-details angebote @@ -44003,7 +44003,7 @@ applicant-eligibility applicant-registration application application* -application-content/ +application-content/ application-mobile-la-poste/content application-survey/ application/ @@ -45195,7 +45195,7 @@ asosinsky asouri asp.net/ asp/ -asp/ +asp/ asp/* asp/*http://allrecipes.com/robots.txt asp/*http://art19.com/robots.txt @@ -45207,7 +45207,7 @@ aspWebCalendar/ aspnet_client aspnet_client aspnet_client/ -aspnet_client/ +aspnet_client/ aspwebcalendar/ aspwebcalendar/calendar.asp aspx$ @@ -46703,7 +46703,7 @@ banners banners-old.php? banners.aspx banners/ -banners/ +banners/ banners/link-default.html banners/obama-vs-romney.html banners2/ @@ -60712,7 +60712,7 @@ cloud-setup cloud-storage-pricing-plans/* cloud-update-instructions cloud/ -cloud/ +cloud/ cloud/$ cloud/* cloud/131 @@ -63587,7 +63587,7 @@ contentassets/*.xml$ contentfeed/getdata?type=synopsi contentisblocked contentisblocked.md -contentjam/ +contentjam/ contentonly contentpar_gridblock contentresearchtool/ @@ -70733,7 +70733,7 @@ downloadpubliclightbox.php downloads downloads.php downloads/ -downloads/ +downloads/ downloads/* downloads/*/ downloads/?q= @@ -70827,7 +70827,7 @@ dritte drive drive-and-listen/ drive-angry -drive-in/ +drive-in/ drive-with-lyft*ref= drive-with-lyft*utm drive/car-reviews/*/new @@ -76131,10 +76131,10 @@ epilepsy_pictures_slideshow/article.htm episerver episerver episode -episode-engine/ -episode-podcast/ +episode-engine/ +episode-podcast/ +episode/ episode/ -episode/ episode/*/transcript$ episode/http episodes @@ -80465,7 +80465,7 @@ filter/more filter/tip filter/tips filter/tips -filter/tips +filter/tips filter/tips/ filter= filter=* @@ -80857,7 +80857,7 @@ flightsh5/ flightstat/v1 flint-deals flip-book/pdf/ad106c9390e50a84a1f87bed55b5115b712d9c6b.pdf -flip-player/ +flip-player/ flip-preview flipboardrss flipbook @@ -92206,7 +92206,7 @@ html?pageq= html?pages= html?ref= html?user_country= -html_email/ +html_email/ html_externe html_helpers html_templates @@ -94397,7 +94397,7 @@ include_xml/* includes includes.other/ includes/ -includes/ +includes/ includes/*.js includes/_stampa_articolo.php* includes/boite/1 @@ -94588,7 +94588,7 @@ index.php/facets-block-ajax index.php/filter/tip index.php/filter/tips index.php/filter/tips -index.php/filter/tips +index.php/filter/tips index.php/filter/tips/ index.php/fusion-search index.php/game-reviews/*/user-reviews/manage/add @@ -94638,7 +94638,7 @@ index.php/search index.php/search-results index.php/search-seeds index.php/search/ -index.php/search/ +index.php/search/ index.php/search? index.php/search_content_unece index.php/site-search @@ -95732,7 +95732,7 @@ ipAcl ipad ipad-games/buy/* ipad/ -ipad/ +ipad/ ipad/4/Contents/en/searchindex ipadconsent ipaddownload @@ -100232,7 +100232,7 @@ lan lancaster land land-browse-bff -land/ +land/ land/*/reviews land/*/sites/*/photos landes/ @@ -100250,7 +100250,7 @@ landing-page/webinar/kessler-expertserieswebinar-mojo-assetdeliveryhttp://curbed landing-pages/ landing.php landing/ -landing/ +landing/ landing/CME-channel landing/VillageMD landing/athenaclinicals @@ -100476,7 +100476,7 @@ latoyaprincessjackson launch-academy launch-learn-more-vid launch.aspx -launch/ +launch/ launch2005/ launchpad launchpad/*css @@ -104656,7 +104656,7 @@ map map map-constructor/loader map-regular.xml -map-review/ +map-review/ map-videofull.xml map.ashx map.asp @@ -104813,7 +104813,7 @@ margaretjenkins margarettroyer marglin margotmoinester -mari/ +mari/ maria maria-garytrainingsite maria_acevedo @@ -108777,7 +108777,7 @@ mx/user mx/user$ mx/your mx2019.jsp -mxf-final-cut-pro/ +mxf-final-cut-pro/ mxhe mxiao my @@ -109896,7 +109896,7 @@ news-ga news-insider$ news-israel-elections/elections-2015-undecided/ news-law/crime/Article-d9f6b8163498f11004.htm -news-manager/ +news-manager/ news-marketing-collateral/press-releases/ news-military/6361323ddea5a810/Article-db536baf3d10c81027.htm news-onair/actualites @@ -129768,7 +129768,7 @@ salehizadeh salernolisa sales sales-demo-video/ -sales-kit/ +sales-kit/ sales/*?*after= sales/*?*before= sales/digital @@ -130306,7 +130306,7 @@ scratch_1.4 scratching scratchr2/static screen -screen-flow/ +screen-flow/ screen-recorder-app? screen-time-strategies-you-can-start-today screen/image @@ -130836,7 +130836,7 @@ search.py search.ref search.rl search/ -search/ +search/ search/" search/$ search/* @@ -140887,7 +140887,7 @@ test.html test.jsp test.umb.edu/ test/ -test/ +test/ test/* test/*?question* test/*?rtf* @@ -142250,7 +142250,7 @@ topicsearch?q= topo-summit-2019 toprailapp tops -topsecret/ +topsecret/ topsfr topsocial topsy @@ -142680,7 +142680,7 @@ trademgr tradeoffer traduction traffic -traffic-manager/ +traffic-manager/ traffic/status*? trafficbot.life trafficbot.live @@ -144615,7 +144615,7 @@ upcomingeventdetails.html upcomingevents.html? update update-browser -update-check/wirecast/ +update-check/wirecast/ update-guide update-settings update-your-browser @@ -146175,13 +146175,13 @@ vanity vannessajharrison vans/leasing/search vansorat -vantage-cloud/ -vantage-enterprise/ -vantage-transcoding/ -vantageCloudManager/ -vantageCloudTrial/ -vantageDomainServer/ -vantageTrancodeServer/ +vantage-cloud/ +vantage-enterprise/ +vantage-transcoding/ +vantageCloudManager/ +vantageCloudTrial/ +vantageDomainServer/ +vantageTrancodeServer/ vantage_point/archive vapors/ var @@ -146600,7 +146600,7 @@ video*search=*o=mr&page=8$ video*search=*o=mr&page=9$ video- video-alert -video-cue/ +video-cue/ video-en-directo video-en-directo/ video-gallery/ @@ -172113,7 +172113,7 @@ wip/* wip_index.php wipostad/ wire- -wire-cast/ +wire-cast/ wirecutter wirecutter/*&xid= wirecutter/*.csv$