diff --git a/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-10000.txt b/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-10000.txt index 1407cf21..800c4d06 100644 --- a/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-10000.txt +++ b/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-10000.txt @@ -2490,7 +2490,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService .pdb$ .pdf .pdf -.pdf +.pdf .pdf$ .pdf$ .pdf$http://dentx.com.tr/robots.txt @@ -7496,7 +7496,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService /feed* /feed.rss /feed/ -/feed/ +/feed/ /feed/$ /feed/* /feed/*/articles.xml @@ -27666,7 +27666,7 @@ GuruFiles/ Gurufiles/ H HACAffiliate -HBO/ +HBO/ HK/ HM HMDThankYou @@ -40748,7 +40748,7 @@ actuator actuator/http://pear.php.net/robots.txt ad ad-blocker-performance-study-in-response-to-manifest-v3-finds-that-ghosterys-ad-blocker-beats-the-competition -ad-manager/ +ad-manager/ ad-retargeting-101-online-shopping-in-the-covid-19-era ad-scans ad-test @@ -41423,7 +41423,7 @@ agile064 agile065 agile066 agile067 -agility/ +agility/ aging_surprises_getting_older/article.htm agplants agreements @@ -42712,7 +42712,7 @@ anesth/employment/faculty/cv anewleaf anfrage anfrage-oeffnungszeiten -anga/ +anga/ angebot angebot-details angebote @@ -43567,7 +43567,7 @@ appliance-delivery-installation/* applicant-eligibility application application* -application-content/ +application-content/ application-survey/ application/ application/project/css/*.css @@ -44677,7 +44677,7 @@ askquestion? asm/ asp.net/ asp/ -asp/ +asp/ asp/cgabillstatus/ asp/demo_db_edit.asp asp/menu/ @@ -44685,7 +44685,7 @@ aspSubscriptionSubmit.asp aspWebCalendar/ aspnet_client aspnet_client/ -aspnet_client/ +aspnet_client/ aspwebcalendar/ aspwebcalendar/calendar.asp aspx$ @@ -45194,9 +45194,9 @@ au*/shop/*/favoriters au*/shop/*/sold au-en/shop/*?*&*&*&* au-fr -au-job/JobBoard/SalAjxGetIbarData -au-job/apply -au-job/searchresults +au-job/JobBoard/SalAjxGetIbarData +au-job/apply +au-job/searchresults au-quotidien/*idOffre= au.html au/ @@ -45919,7 +45919,7 @@ b/order b/unsubscribe b2b b2b-gift-card-shop -b2b-resource/blogdownloadform +b2b-resource/blogdownloadform b2bauth b2c-tools b=*p=* @@ -46115,7 +46115,7 @@ banners banners-old.php? banners.aspx banners/ -banners/ +banners/ banners/link-default.html banners/obama-vs-romney.html banners2/ @@ -54399,16 +54399,16 @@ business/referral business/salaire/patron/recherche business/salaire/recherche business/sales/blog -business/search/BusinessSearch/SalAjaxGetPhoneSupportTime -business/search/BusinessSearch/SalAjaxGetServerTime -business/search/BusinessSearch/SalAjxAddPollQuestionDisplayTracking -business/search/BusinessSearch/SalAjxFloatingButtonPopupEntry -business/search/BusinessSearch/SalAjxGetBlogAIHtmlOnBLogContent -business/search/BusinessSearch/SalAjxGetGlobalBannerNewsHtml -business/search/BusinessSearch/SalAjxGetPopupFloatingBarDemoFormHtml -business/search/BusinessSearch/SalAjxPainPointsDemoForProductPagePopupEntry -business/search/BusinessSearch/SalAjxPopupEntry -business/search/BusinessSearch/SalGetProductDescByPage +business/search/BusinessSearch/SalAjaxGetPhoneSupportTime +business/search/BusinessSearch/SalAjaxGetServerTime +business/search/BusinessSearch/SalAjxAddPollQuestionDisplayTracking +business/search/BusinessSearch/SalAjxFloatingButtonPopupEntry +business/search/BusinessSearch/SalAjxGetBlogAIHtmlOnBLogContent +business/search/BusinessSearch/SalAjxGetGlobalBannerNewsHtml +business/search/BusinessSearch/SalAjxGetPopupFloatingBarDemoFormHtml +business/search/BusinessSearch/SalAjxPainPointsDemoForProductPagePopupEntry +business/search/BusinessSearch/SalAjxPopupEntry +business/search/BusinessSearch/SalGetProductDescByPage business/signup business/site business/small-business-essentials @@ -54897,10 +54897,10 @@ ca-fr/trip-companion ca-fr/trips ca-fr/user ca-fr/your -ca-job/JobBoard/SalAjxGetCANLocationsByKeyword -ca-job/JobBoard/SalAjxGetIbarData -ca-job/apply -ca-job/searchresults +ca-job/JobBoard/SalAjxGetCANLocationsByKeyword +ca-job/JobBoard/SalAjxGetIbarData +ca-job/apply +ca-job/searchresults ca-privacy-notice ca-zh ca/ @@ -62708,7 +62708,7 @@ cloud-setup cloud-storage-pricing-plans/* cloud-update-instructions cloud/ -cloud/ +cloud/ cloud/$ cloud/* cloud/131 @@ -67373,7 +67373,7 @@ contentassets/*.xml$ contentfeed/getdata?type=synopsi contentisblocked contentisblocked.md -contentjam/ +contentjam/ contentonly contentpar_gridblock contentresearchtool/ @@ -74915,7 +74915,7 @@ downloadpubliclightbox.php downloads downloads.php downloads/ -downloads/ +downloads/ downloads/$ downloads/* downloads/*/ @@ -75004,7 +75004,7 @@ dritte drive drive-and-listen/ drive-angry -drive-in/ +drive-in/ drive-with-lyft*ref= drive-with-lyft*utm drive/car-reviews/*/new @@ -81191,9 +81191,9 @@ epilepsy_pictures_slideshow/article.htm episerver episerver episode -episode-engine/ -episode-podcast/ -episode/ +episode-engine/ +episode-podcast/ +episode/ episode/*/transcript$ episode/http episodes @@ -85079,7 +85079,7 @@ feed.mhtml feed.php? feed.php?mode=list&ns= feed/ -feed/ +feed/ feed/$ feed/$ feed/3 @@ -85708,7 +85708,7 @@ filter/more filter/tip filter/tips filter/tips -filter/tips +filter/tips filter/tips/ filter= filter= @@ -86055,7 +86055,7 @@ flightsh5/ flightstat/v1 flint-deals flip-book/pdf/ad106c9390e50a84a1f87bed55b5115b712d9c6b.pdf -flip-player/ +flip-player/ flip-preview flipboardrss flipbook @@ -99120,7 +99120,7 @@ html?pageq= html?pages= html?ref= html?user_country= -html_email/ +html_email/ html_externe html_helpers html_tests @@ -101148,9 +101148,9 @@ in-en/your in-hi/tutorials/*_ga in-hi/tutorials/?s= in-hi/tutorials/search -in-job/JobBoard/SalAjxGetIbarData -in-job/apply -in-job/searchresults +in-job/JobBoard/SalAjxGetIbarData +in-job/apply +in-job/searchresults in-notes/authors in-notes/rerendered in-product @@ -101296,7 +101296,7 @@ include_xml/* includes includes.other/ includes/ -includes/ +includes/ includes/*.js includes/_stampa_articolo.php* includes/boite/1 @@ -101492,7 +101492,7 @@ index.php/fdic-search index.php/filter/tip index.php/filter/tips index.php/filter/tips -index.php/filter/tips +index.php/filter/tips index.php/filter/tips/ index.php/fusion-search index.php/game-reviews/*/user-reviews/manage/add @@ -101546,7 +101546,7 @@ index.php/search index.php/search index.php/search-seeds index.php/search/ -index.php/search/ +index.php/search/ index.php/search/global index.php/search/global? index.php/search? @@ -102624,7 +102624,7 @@ ipAcl ipad ipad-games/buy/* ipad/ -ipad/ +ipad/ ipad/4/Contents/en/searchindex ipaddownload ipc @@ -105176,8 +105176,8 @@ job-offer/edit job-offer/post job-oriented-programs/ job/* -job/JobBoard/SalAjxGetIbarData -job/JobBoard/SalAjxGetLocationsByKeyword +job/JobBoard/SalAjxGetIbarData +job/JobBoard/SalAjxGetLocationsByKeyword job/banned.html job/bidders job/editResume @@ -105190,7 +105190,7 @@ job/redirect job/rules.html job/searchResume job/searchVacance -job/searchresults +job/searchresults job/sitemap.xml job/track-apply-click job/track-pageview @@ -107563,7 +107563,7 @@ laluan/*/laluan-dari- lamda/images land land-browse-bff -land/ +land/ land/*/reviews land/*/sites/*/photos lander @@ -107582,7 +107582,7 @@ landing-page/webinar/kessler-expertserieswebinar-mojo-assetdeliveryhttp://curbed landing-pages/ landing.php landing/ -landing/ +landing/ landing/CME-channel landing/VillageMD landing/athenaclinicals @@ -107804,7 +107804,7 @@ latinamerica/wp-includes launch-academy launch-learn-more-vid launch.aspx -launch/ +launch/ launch2005/ launchpad launchpad/*css @@ -112063,7 +112063,7 @@ map map map-constructor/loader map-regular.xml -map-review/ +map-review/ map-videofull.xml map.ashx map.asp @@ -112206,7 +112206,7 @@ marcioreinaldo marcosmontes marcotebaldi margeaux-watson-appointed-senior-vice-president-media-relations-universal-motown-records -mari/ +mari/ maria-ho-appointed-vice-president-corporate-communications-universal-music-group mariah-carey-puts-finishing-touches-on-her-new-album-memoirs-of-an-imperfect-angel mariah-careys-dont-forget-about-us-hits-1 @@ -115856,9 +115856,9 @@ mx*/people/*/favorites mx*/shop/*/favoriters mx*/shop/*/sold mx-es/shop/*?*&*&*&* -mx-job/JobBoard/SalAjxGetIbarData -mx-job/apply -mx-job/searchresults +mx-job/JobBoard/SalAjxGetIbarData +mx-job/apply +mx-job/searchresults mx-mx/ mx/ mx/*?*lang=es-MX @@ -115950,7 +115950,7 @@ mx/user mx/user$ mx/your mx2019.jsp -mxf-final-cut-pro/ +mxf-final-cut-pro/ my my$ my-account @@ -117091,7 +117091,7 @@ news-events/events-calendar/2018/03/ninth-annual-african-consumer-protection-dia news-events/events-calendar/2019/05/tenth-annual-african-consumer-protection-dialogue-conference news-ga news-insider$ -news-manager/ +news-manager/ news-onair/actualites news-outlets news-post @@ -125720,7 +125720,7 @@ pasaj/siparis-sorgulama pascal-negre-extends-contract-for-nine-years pasha2 pass -passages/ +passages/ passengerFirstName= passengerLastName= passport @@ -135240,7 +135240,7 @@ research/community-driven-research/global-inclusion-and-social-development/ research/configure research/configure/preview.jsp research/content/article-printable-popup.jsp -research/cost-of-living/CostOfLiving/SalAjxGetLocationsByKeyword +research/cost-of-living/CostOfLiving/SalAjxGetLocationsByKeyword research/economists/RSG_SeminarsCalendar.pdf research/economists/RSG_VisitorsCalendar.pdf research/enqueue_tracking @@ -135265,11 +135265,11 @@ research/request-broadcast-email/thank-you/ research/research-sponsored-programs/ research/research-sponsored-programs/grant-lifecycle/ research/research/SearchResults.jsp -research/salary/JobSalary/SalAjaxGetEducationLeavelsSvg -research/salary/JobSalary/SalAjaxGetExperienceLeavelsSvg -research/salary/JobSalary/SalAjxGetIbarData -research/salary/JobSalary/SalAjxGetLocationsByKeyword -research/salary/JobSalary/SalAjxGetMapSalry +research/salary/JobSalary/SalAjaxGetEducationLeavelsSvg +research/salary/JobSalary/SalAjaxGetExperienceLeavelsSvg +research/salary/JobSalary/SalAjxGetIbarData +research/salary/JobSalary/SalAjxGetLocationsByKeyword +research/salary/JobSalary/SalAjxGetMapSalry research/search research/shared/article.jsp research/thankyou.aspx @@ -135634,9 +135634,9 @@ resources/ftptransfer resources/global/player/embed resources/gocodersdk/54f8721827aeb3bebffcca5e3b1a0d43/ resources/gocodersdk/docs/1.0/54f8721827aeb3bebffcca5e3b1a0d43/ -resources/how-to/HowTo/SalAjxGetAssetDownloadForm -resources/how-to/HowTo/SalAjxGetRandomHighConversionArticleContent -resources/how-to/HowTo/SalAjxGetRandomHighTrafficArticleContent +resources/how-to/HowTo/SalAjxGetAssetDownloadForm +resources/how-to/HowTo/SalAjxGetRandomHighConversionArticleContent +resources/how-to/HowTo/SalAjxGetRandomHighTrafficArticleContent resources/html/error/404.html resources/html/infographics/2018/ceo-activism-desktop/index.html resources/html5 @@ -137639,7 +137639,7 @@ sale= sale_type= sales sales-demo-video -sales-kit/ +sales-kit/ sales/*?*after= sales/*?*before= sales/digital @@ -138042,7 +138042,7 @@ scratch_1.4 scratching scratchr2/static screen -screen-flow/ +screen-flow/ screen-recorder-app? screen-time-strategies-you-can-start-today screen/image @@ -138520,7 +138520,7 @@ search.py search.ref search.rl search/ -search/ +search/ search/" search/$ search/* @@ -146512,7 +146512,7 @@ switcher= swp_analytics swr swsbm -swz-request-demo/ +swz-request-demo/ sx sxi/servlet/sxibrowse sxi/servlet/sximovie @@ -148866,7 +148866,7 @@ test.html test.jsp test.umb.edu/ test/ -test/ +test/ test/* test/*?question* test/*?rtf* @@ -150029,7 +150029,7 @@ tools/android-sdk tools/apache tools/api/agents/agentMarket tools/backdoor-factory -tools/benefits-calculator/ +tools/benefits-calculator/ tools/blogs tools/bluez-firmware tools/browseresults*?*threshold= @@ -150044,7 +150044,7 @@ tools/desktop-video-editor-activity tools/dream-big-savings-calculator.html tools/email.asp tools/en/styleguide/ -tools/executive-compensation-calculator/ +tools/executive-compensation-calculator/ tools/fake-hwclock tools/faraday-client tools/find-a-doctor/list/export @@ -150052,7 +150052,7 @@ tools/firmware-sof tools/footer-navigation tools/forms tools/freerdp2 -tools/global-salary-calculator/ +tools/global-salary-calculator/ tools/ical.cfm tools/index.html? tools/initramfs-tools @@ -150097,7 +150097,7 @@ tools/reports-and-studies/h-2b-employer-data-hub tools/reports-and-studies/h-2b-employer-data-hub/export tools/reports-and-studies/h-2b-employer-data-hub/export-csv tools/rtlsdr-scanner -tools/salary-calculator/ +tools/salary-calculator/ tools/search tools/searchresults*?*threshold= tools/smali @@ -150225,7 +150225,7 @@ topla-kazan-kuponlari topnav toprailapp tops -topsecret/ +topsecret/ topsfr topsocial topsy @@ -150803,7 +150803,7 @@ trading_simulator_staging traduction traffic traffic-aggregation -traffic-manager/ +traffic-manager/ traffic/ traffic/status*? trafficbot.life @@ -152045,9 +152045,9 @@ uk-corporate uk-corporate/blackfriday uk-en uk-en/ -uk-job/JobBoard/SalAjxGetIbarData -uk-job/apply -uk-job/searchresults +uk-job/JobBoard/SalAjxGetIbarData +uk-job/apply +uk-job/searchresults uk-join uk-ua uk-ua/ @@ -153001,7 +153001,7 @@ upcomingevents upcomingevents.html? update update-browser -update-check/wirecast/ +update-check/wirecast/ update-guide update-settings update-your-browser @@ -154478,13 +154478,13 @@ vanessagrazziotin vanilla vanity vans/leasing/search -vantage-cloud/ -vantage-enterprise/ -vantage-transcoding/ -vantageCloudManager/ -vantageCloudTrial/ -vantageDomainServer/ -vantageTrancodeServer/ +vantage-cloud/ +vantage-enterprise/ +vantage-transcoding/ +vantageCloudManager/ +vantageCloudTrial/ +vantageDomainServer/ +vantageTrancodeServer/ vantage_point/archive vantagepoint vapors/ @@ -154877,7 +154877,7 @@ video*search=*o=mr&page=8$ video*search=*o=mr&page=9$ video- video-alert -video-cue/ +video-cue/ video-en-directo video-gallery/ video-keywords @@ -180981,7 +180981,7 @@ wip/* wip_index.php wipostad/ wire- -wire-cast/ +wire-cast/ wirecutter wirecutter/ wirecutter/*&xid= @@ -181406,7 +181406,7 @@ wp-admin wp-admin wp-admin.php wp-admin/ -wp-admin/ +wp-admin/ wp-admin/$ wp-admin/* wp-admin/about.php